From 4da66fdfb3d0686b5ed917e947869b9c2cef14a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Bene=C5=A1?= Date: Wed, 5 Jun 2024 11:09:11 +0100 Subject: [PATCH] feat: processing outgoing (#6766) --- yarn-project/aztec.js/src/utils/account.ts | 4 +- .../src/interfaces/sync-status.ts | 2 +- .../logs/l1_note_payload/l1_note_payload.ts | 14 +- .../src/logs/l1_note_payload/note.ts | 4 + .../src/logs/l1_note_payload/tagged_note.ts | 16 +- yarn-project/circuit-types/src/stats/stats.ts | 10 +- .../end-to-end/src/benchmarks/utils.ts | 6 +- .../e2e_pending_note_hashes_contract.test.ts | 7 +- yarn-project/key-store/src/key_store.ts | 30 --- .../src/database/deferred_note_dao.test.ts | 4 +- .../pxe/src/database/deferred_note_dao.ts | 9 +- ..._dao.test.ts => incoming_note_dao.test.ts} | 18 +- .../{note_dao.ts => incoming_note_dao.ts} | 12 +- .../pxe/src/database/kv_pxe_database.ts | 48 ++-- .../src/database/outgoing_note_dao.test.ts | 23 ++ .../pxe/src/database/outgoing_note_dao.ts | 64 +++++ yarn-project/pxe/src/database/pxe_database.ts | 14 +- .../src/database/pxe_database_test_suite.ts | 36 +-- .../src/note_processor/note_processor.test.ts | 182 +++++++++----- .../pxe/src/note_processor/note_processor.ts | 235 +++++++++++------- .../src/note_processor/produce_note_dao.ts | 158 ++++++++---- .../pxe/src/pxe_service/pxe_service.ts | 19 +- .../pxe/src/synchronizer/synchronizer.test.ts | 2 +- .../pxe/src/synchronizer/synchronizer.ts | 63 +++-- .../scripts/src/benchmarks/aggregate.ts | 4 +- 25 files changed, 614 insertions(+), 370 deletions(-) rename yarn-project/pxe/src/database/{note_dao.test.ts => incoming_note_dao.test.ts} (64%) rename yarn-project/pxe/src/database/{note_dao.ts => incoming_note_dao.ts} (91%) create mode 100644 yarn-project/pxe/src/database/outgoing_note_dao.test.ts create mode 100644 yarn-project/pxe/src/database/outgoing_note_dao.ts diff --git a/yarn-project/aztec.js/src/utils/account.ts b/yarn-project/aztec.js/src/utils/account.ts index 26ceb9e599d..92701d5ebe9 100644 --- a/yarn-project/aztec.js/src/utils/account.ts +++ b/yarn-project/aztec.js/src/utils/account.ts @@ -14,11 +14,11 @@ export async function waitForAccountSynch( address: CompleteAddress, { interval, timeout }: WaitOpts = DefaultWaitOpts, ): Promise { - const publicKey = address.publicKeys.masterIncomingViewingPublicKey.toString(); + const accountAddress = address.address.toString(); await retryUntil( async () => { const status = await pxe.getSyncStatus(); - const accountSynchedToBlock = status.notes[publicKey]; + const accountSynchedToBlock = status.notes[accountAddress]; if (typeof accountSynchedToBlock === 'undefined') { return false; } else { diff --git a/yarn-project/circuit-types/src/interfaces/sync-status.ts b/yarn-project/circuit-types/src/interfaces/sync-status.ts index 75770ffe14b..0e453b41bfd 100644 --- a/yarn-project/circuit-types/src/interfaces/sync-status.ts +++ b/yarn-project/circuit-types/src/interfaces/sync-status.ts @@ -2,6 +2,6 @@ export type SyncStatus = { /** Up to which block has been synched for blocks and txs. */ blocks: number; - /** Up to which block has been synched for notes, indexed by each public key being monitored. */ + /** Up to which block has been synched for notes, indexed by each account address being monitored. */ notes: Record; }; diff --git a/yarn-project/circuit-types/src/logs/l1_note_payload/l1_note_payload.ts b/yarn-project/circuit-types/src/logs/l1_note_payload/l1_note_payload.ts index 89551558d73..a4e7db0f50c 100644 --- a/yarn-project/circuit-types/src/logs/l1_note_payload/l1_note_payload.ts +++ b/yarn-project/circuit-types/src/logs/l1_note_payload/l1_note_payload.ts @@ -73,10 +73,11 @@ export class L1NotePayload { /** * Create a random L1NotePayload object (useful for testing purposes). + * @param contract - The address of a contract the note was emitted from. * @returns A random L1NotePayload object. */ - static random() { - return new L1NotePayload(Note.random(), AztecAddress.random(), Fr.random(), Fr.random()); + static random(contract = AztecAddress.random()) { + return new L1NotePayload(Note.random(), contract, Fr.random(), Fr.random()); } /** @@ -208,4 +209,13 @@ export class L1NotePayload { incomingBody.noteTypeId, ); } + + public equals(other: L1NotePayload) { + return ( + this.note.equals(other.note) && + this.contractAddress.equals(other.contractAddress) && + this.storageSlot.equals(other.storageSlot) && + this.noteTypeId.equals(other.noteTypeId) + ); + } } diff --git a/yarn-project/circuit-types/src/logs/l1_note_payload/note.ts b/yarn-project/circuit-types/src/logs/l1_note_payload/note.ts index 9562a277e48..ea59a372d71 100644 --- a/yarn-project/circuit-types/src/logs/l1_note_payload/note.ts +++ b/yarn-project/circuit-types/src/logs/l1_note_payload/note.ts @@ -56,4 +56,8 @@ export class Note extends Vector { get length() { return this.items.length; } + + equals(other: Note) { + return this.items.every((item, index) => item.equals(other.items[index])); + } } diff --git a/yarn-project/circuit-types/src/logs/l1_note_payload/tagged_note.ts b/yarn-project/circuit-types/src/logs/l1_note_payload/tagged_note.ts index defe4a1c757..4889b5cab25 100644 --- a/yarn-project/circuit-types/src/logs/l1_note_payload/tagged_note.ts +++ b/yarn-project/circuit-types/src/logs/l1_note_payload/tagged_note.ts @@ -1,9 +1,4 @@ -import { - type AztecAddress, - type GrumpkinPrivateKey, - type KeyValidationRequest, - type PublicKey, -} from '@aztec/circuits.js'; +import { AztecAddress, type GrumpkinPrivateKey, type KeyValidationRequest, type PublicKey } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; @@ -43,8 +38,13 @@ export class TaggedNote { return serializeToBuffer(this.incomingTag, this.outgoingTag, this.notePayload); } - static random(): TaggedNote { - return new TaggedNote(L1NotePayload.random()); + /** + * Create a random TaggedNote (useful for testing purposes). + * @param contract - The address of a contract the note was emitted from. + * @returns A random TaggedNote object. + */ + static random(contract = AztecAddress.random()): TaggedNote { + return new TaggedNote(L1NotePayload.random(contract)); } public encrypt( diff --git a/yarn-project/circuit-types/src/stats/stats.ts b/yarn-project/circuit-types/src/stats/stats.ts index 03d6545d70f..ca403a47d44 100644 --- a/yarn-project/circuit-types/src/stats/stats.ts +++ b/yarn-project/circuit-types/src/stats/stats.ts @@ -161,8 +161,8 @@ export type L2BlockHandledStats = { export type NoteProcessorCaughtUpStats = { /** Name of the event. */ eventName: 'note-processor-caught-up'; - /** Public key of the note processor. */ - publicKey: string; + /** Account the note processor belongs to. */ + account: string; /** Total time to catch up with the tip of the chain from scratch in ms. */ duration: number; /** Size of the notes db. */ @@ -175,8 +175,10 @@ export type NoteProcessorStats = { seen: number; /** How many notes had decryption deferred due to a missing contract */ deferred: number; - /** How many notes were successfully decrypted. */ - decrypted: number; + /** How many incoming notes were successfully decrypted. */ + decryptedIncoming: number; + /** How many outgoing notes were successfully decrypted. */ + decryptedOutgoing: number; /** How many notes failed processing. */ failed: number; /** How many blocks were spanned. */ diff --git a/yarn-project/end-to-end/src/benchmarks/utils.ts b/yarn-project/end-to-end/src/benchmarks/utils.ts index f761ada9c32..d2925adaede 100644 --- a/yarn-project/end-to-end/src/benchmarks/utils.ts +++ b/yarn-project/end-to-end/src/benchmarks/utils.ts @@ -129,9 +129,7 @@ export async function waitNewPXESynced( */ export async function waitRegisteredAccountSynced(pxe: PXE, secretKey: Fr, partialAddress: PartialAddress) { const l2Block = await pxe.getBlockNumber(); - const masterIncomingViewingPublicKey = (await pxe.registerAccount(secretKey, partialAddress)).publicKeys - .masterIncomingViewingPublicKey; - const isAccountSynced = async () => - (await pxe.getSyncStatus()).notes[masterIncomingViewingPublicKey.toString()] === l2Block; + const accountAddress = (await pxe.registerAccount(secretKey, partialAddress)).address; + const isAccountSynced = async () => (await pxe.getSyncStatus()).notes[accountAddress.toString()] === l2Block; await retryUntil(isAccountSynced, 'pxe-notes-sync'); } diff --git a/yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts b/yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts index 43b7e2c9bb8..46df722fee2 100644 --- a/yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts +++ b/yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts @@ -306,11 +306,10 @@ describe('e2e_pending_note_hashes_contract', () => { // Then emit another note log with the same counter as the one above, but with value 5 await deployedContract.methods.test_emit_bad_note_log(owner, outgoingViewer).send().wait(); - const mIVPK = wallet.getCompleteAddress().publicKeys.masterIncomingViewingPublicKey.toString(); const syncStats = await wallet.getSyncStats(); - // Expect two decryptable note logs to be emitted - expect(syncStats[mIVPK].decrypted).toEqual(2); + // Expect two incoming decryptable note logs to be emitted + expect(syncStats[owner.toString()].decryptedIncoming).toEqual(2); // Expect one note log to be dropped - expect(syncStats[mIVPK].failed).toEqual(1); + expect(syncStats[owner.toString()].failed).toEqual(1); }); }); diff --git a/yarn-project/key-store/src/key_store.ts b/yarn-project/key-store/src/key_store.ts index 36f1d904d6a..f3705a92425 100644 --- a/yarn-project/key-store/src/key_store.ts +++ b/yarn-project/key-store/src/key_store.ts @@ -306,36 +306,6 @@ export class KeyStore { return Promise.resolve(skM); } - /** - * Retrieves the master incoming viewing secret key (ivsk_m) corresponding to the specified master incoming viewing - * public key (Ivpk_m). - * @throws If the provided public key is not associated with any of the registered accounts. - * @param masterIncomingViewingPublicKey - The master nullifier public key to get secret key for. - * @returns A Promise that resolves to the master nullifier secret key. - * @dev Used when feeding the master nullifier secret key to the kernel circuit for nullifier keys verification. - */ - public getMasterIncomingViewingSecretKeyForPublicKey( - masterIncomingViewingPublicKey: PublicKey, - ): Promise { - // We iterate over the map keys to find the account address that corresponds to the provided public key - for (const [key, value] of this.#keys.entries()) { - if (value.equals(masterIncomingViewingPublicKey.toBuffer())) { - // We extract the account address from the map key - const account = key.split('-')[0]; - // We fetch the secret key and return it - const masterIncomingViewingSecretKeyBuffer = this.#keys.get(`${account.toString()}-ivsk_m`); - if (!masterIncomingViewingSecretKeyBuffer) { - throw new Error(`Could not find master incoming viewing secret key for account ${account.toString()}`); - } - return Promise.resolve(GrumpkinScalar.fromBuffer(masterIncomingViewingSecretKeyBuffer)); - } - } - - throw new Error( - `Could not find master incoming viewing secret key for public key ${masterIncomingViewingPublicKey.toString()}`, - ); - } - /** * Rotates the master nullifier key for the specified account. * diff --git a/yarn-project/pxe/src/database/deferred_note_dao.test.ts b/yarn-project/pxe/src/database/deferred_note_dao.test.ts index d3c1e5d520b..00ede18ca9c 100644 --- a/yarn-project/pxe/src/database/deferred_note_dao.test.ts +++ b/yarn-project/pxe/src/database/deferred_note_dao.test.ts @@ -5,7 +5,7 @@ import { randomInt } from '@aztec/foundation/crypto'; import { DeferredNoteDao } from './deferred_note_dao.js'; export const randomDeferredNoteDao = ({ - publicKey = Point.random(), + ivpkM = Point.random(), note = Note.random(), contractAddress = AztecAddress.random(), txHash = randomTxHash(), @@ -15,7 +15,7 @@ export const randomDeferredNoteDao = ({ dataStartIndexForTx = randomInt(100), }: Partial = {}) => { return new DeferredNoteDao( - publicKey, + ivpkM, note, contractAddress, storageSlot, diff --git a/yarn-project/pxe/src/database/deferred_note_dao.ts b/yarn-project/pxe/src/database/deferred_note_dao.ts index d45d179e5b2..0851f4dadcf 100644 --- a/yarn-project/pxe/src/database/deferred_note_dao.ts +++ b/yarn-project/pxe/src/database/deferred_note_dao.ts @@ -9,8 +9,11 @@ import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; */ export class DeferredNoteDao { constructor( - /** The public key associated with this note */ - public publicKey: PublicKey, + /** + * The incoming viewing public key the note was encrypted with. + * @dev Will never be ovpkM because there are no deferred notes for outgoing. + */ + public ivpkM: PublicKey, /** The note as emitted from the Noir contract. */ public note: Note, /** The contract address this note is created in. */ @@ -29,7 +32,7 @@ export class DeferredNoteDao { toBuffer(): Buffer { return serializeToBuffer( - this.publicKey, + this.ivpkM, this.note, this.contractAddress, this.storageSlot, diff --git a/yarn-project/pxe/src/database/note_dao.test.ts b/yarn-project/pxe/src/database/incoming_note_dao.test.ts similarity index 64% rename from yarn-project/pxe/src/database/note_dao.test.ts rename to yarn-project/pxe/src/database/incoming_note_dao.test.ts index 5b499eb73d9..ae8d562a381 100644 --- a/yarn-project/pxe/src/database/note_dao.test.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.test.ts @@ -1,9 +1,9 @@ import { Note, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteDao } from './note_dao.js'; +import { IncomingNoteDao } from './incoming_note_dao.js'; -export const randomNoteDao = ({ +export const randomIncomingNoteDao = ({ note = Note.random(), contractAddress = AztecAddress.random(), txHash = randomTxHash(), @@ -13,9 +13,9 @@ export const randomNoteDao = ({ innerNoteHash = Fr.random(), siloedNullifier = Fr.random(), index = Fr.random().toBigInt(), - publicKey = Point.random(), -}: Partial = {}) => { - return new NoteDao( + ivpkM = Point.random(), +}: Partial = {}) => { + return new IncomingNoteDao( note, contractAddress, storageSlot, @@ -25,14 +25,14 @@ export const randomNoteDao = ({ innerNoteHash, siloedNullifier, index, - publicKey, + ivpkM, ); }; -describe('Note DAO', () => { +describe('Incoming Note DAO', () => { it('convert to and from buffer', () => { - const note = randomNoteDao(); + const note = randomIncomingNoteDao(); const buf = note.toBuffer(); - expect(NoteDao.fromBuffer(buf)).toEqual(note); + expect(IncomingNoteDao.fromBuffer(buf)).toEqual(note); }); }); diff --git a/yarn-project/pxe/src/database/note_dao.ts b/yarn-project/pxe/src/database/incoming_note_dao.ts similarity index 91% rename from yarn-project/pxe/src/database/note_dao.ts rename to yarn-project/pxe/src/database/incoming_note_dao.ts index 5717a668451..85684c7cc6a 100644 --- a/yarn-project/pxe/src/database/note_dao.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.ts @@ -5,9 +5,9 @@ import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; import { type NoteData } from '@aztec/simulator'; /** - * A note with contextual data. + * A note with contextual data which was decrypted as incoming. */ -export class NoteDao implements NoteData { +export class IncomingNoteDao implements NoteData { constructor( /** The note as emitted from the Noir contract. */ public note: Note, @@ -31,7 +31,7 @@ export class NoteDao implements NoteData { /** The location of the relevant note in the note hash tree. */ public index: bigint, /** The public key with which the note was encrypted. */ - public publicKey: PublicKey, + public ivpkM: PublicKey, ) {} toBuffer(): Buffer { @@ -45,7 +45,7 @@ export class NoteDao implements NoteData { this.innerNoteHash, this.siloedNullifier, this.index, - this.publicKey, + this.ivpkM, ]); } static fromBuffer(buffer: Buffer | BufferReader) { @@ -62,7 +62,7 @@ export class NoteDao implements NoteData { const index = toBigIntBE(reader.readBytes(32)); const publicKey = Point.fromBuffer(reader); - return new NoteDao( + return new IncomingNoteDao( note, contractAddress, storageSlot, @@ -82,7 +82,7 @@ export class NoteDao implements NoteData { static fromString(str: string) { const hex = str.replace(/^0x/, ''); - return NoteDao.fromBuffer(Buffer.from(hex, 'hex')); + return IncomingNoteDao.fromBuffer(Buffer.from(hex, 'hex')); } /** diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index f17bc651823..5d6bfb528b0 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -14,7 +14,8 @@ import { contractArtifactFromBuffer, contractArtifactToBuffer } from '@aztec/typ import { type ContractInstanceWithAddress, SerializableContractInstance } from '@aztec/types/contracts'; import { DeferredNoteDao } from './deferred_note_dao.js'; -import { NoteDao } from './note_dao.js'; +import { IncomingNoteDao } from './incoming_note_dao.js'; +import { type OutgoingNoteDao } from './outgoing_note_dao.js'; import { type PxeDatabase } from './pxe_database.js'; /** @@ -32,11 +33,11 @@ export class KVPxeDatabase implements PxeDatabase { #notesByContract: AztecMultiMap; #notesByStorageSlot: AztecMultiMap; #notesByTxHash: AztecMultiMap; - #notesByOwner: AztecMultiMap; + #notesByIvpkM: AztecMultiMap; #nullifiedNotesByContract: AztecMultiMap; #nullifiedNotesByStorageSlot: AztecMultiMap; #nullifiedNotesByTxHash: AztecMultiMap; - #nullifiedNotesByOwner: AztecMultiMap; + #nullifiedNotesByIvpkM: AztecMultiMap; #deferredNotes: AztecArray; #deferredNotesByContract: AztecMultiMap; #syncedBlockPerPublicKey: AztecMap; @@ -66,12 +67,12 @@ export class KVPxeDatabase implements PxeDatabase { this.#notesByContract = db.openMultiMap('notes_by_contract'); this.#notesByStorageSlot = db.openMultiMap('notes_by_storage_slot'); this.#notesByTxHash = db.openMultiMap('notes_by_tx_hash'); - this.#notesByOwner = db.openMultiMap('notes_by_owner'); + this.#notesByIvpkM = db.openMultiMap('notes_by_owner'); this.#nullifiedNotesByContract = db.openMultiMap('nullified_notes_by_contract'); this.#nullifiedNotesByStorageSlot = db.openMultiMap('nullified_notes_by_storage_slot'); this.#nullifiedNotesByTxHash = db.openMultiMap('nullified_notes_by_tx_hash'); - this.#nullifiedNotesByOwner = db.openMultiMap('nullified_notes_by_owner'); + this.#nullifiedNotesByIvpkM = db.openMultiMap('nullified_notes_by_owner'); this.#deferredNotes = db.openArray('deferred_notes'); this.#deferredNotesByContract = db.openMultiMap('deferred_notes_by_contract'); @@ -135,13 +136,14 @@ export class KVPxeDatabase implements PxeDatabase { return val?.map(b => Fr.fromBuffer(b)); } - async addNote(note: NoteDao): Promise { - await this.addNotes([note]); + async addNote(note: IncomingNoteDao): Promise { + await this.addNotes([note], []); } - addNotes(notes: NoteDao[]): Promise { + addNotes(incomingNotes: IncomingNoteDao[], _outgoingNotes: OutgoingNoteDao[]): Promise { + // TODO(#6867): store the outgoing note return this.db.transaction(() => { - for (const dao of notes) { + for (const dao of incomingNotes) { // store notes by their index in the notes hash tree // this provides the uniqueness we need to store individual notes // and should also return notes in the order that they were created. @@ -152,7 +154,7 @@ export class KVPxeDatabase implements PxeDatabase { void this.#notesByContract.set(dao.contractAddress.toString(), noteIndex); void this.#notesByStorageSlot.set(dao.storageSlot.toString(), noteIndex); void this.#notesByTxHash.set(dao.txHash.toString(), noteIndex); - void this.#notesByOwner.set(dao.publicKey.toString(), noteIndex); + void this.#notesByIvpkM.set(dao.ivpkM.toString(), noteIndex); } }); } @@ -207,7 +209,7 @@ export class KVPxeDatabase implements PxeDatabase { }); } - #getNotes(filter: NoteFilter): NoteDao[] { + #getNotes(filter: NoteFilter): IncomingNoteDao[] { const publicKey: PublicKey | undefined = filter.owner ? this.#getCompleteAddress(filter.owner)?.publicKeys.masterIncomingViewingPublicKey : undefined; @@ -218,7 +220,7 @@ export class KVPxeDatabase implements PxeDatabase { candidateNoteSources.push({ ids: publicKey - ? this.#notesByOwner.getValues(publicKey.toString()) + ? this.#notesByIvpkM.getValues(publicKey.toString()) : filter.txHash ? this.#notesByTxHash.getValues(filter.txHash.toString()) : filter.contractAddress @@ -232,7 +234,7 @@ export class KVPxeDatabase implements PxeDatabase { if (filter.status == NoteStatus.ACTIVE_OR_NULLIFIED) { candidateNoteSources.push({ ids: publicKey - ? this.#nullifiedNotesByOwner.getValues(publicKey.toString()) + ? this.#nullifiedNotesByIvpkM.getValues(publicKey.toString()) : filter.txHash ? this.#nullifiedNotesByTxHash.getValues(filter.txHash.toString()) : filter.contractAddress @@ -244,7 +246,7 @@ export class KVPxeDatabase implements PxeDatabase { }); } - const result: NoteDao[] = []; + const result: IncomingNoteDao[] = []; for (const { ids, notes } of candidateNoteSources) { for (const id of ids) { const serializedNote = notes.get(id); @@ -252,7 +254,7 @@ export class KVPxeDatabase implements PxeDatabase { continue; } - const note = NoteDao.fromBuffer(serializedNote); + const note = IncomingNoteDao.fromBuffer(serializedNote); if (filter.contractAddress && !note.contractAddress.equals(filter.contractAddress)) { continue; } @@ -265,7 +267,7 @@ export class KVPxeDatabase implements PxeDatabase { continue; } - if (publicKey && !note.publicKey.equals(publicKey)) { + if (publicKey && !note.ivpkM.equals(publicKey)) { continue; } @@ -276,17 +278,17 @@ export class KVPxeDatabase implements PxeDatabase { return result; } - getNotes(filter: NoteFilter): Promise { + getNotes(filter: NoteFilter): Promise { return Promise.resolve(this.#getNotes(filter)); } - removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise { + removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise { if (nullifiers.length === 0) { return Promise.resolve([]); } return this.#db.transaction(() => { - const nullifiedNotes: NoteDao[] = []; + const nullifiedNotes: IncomingNoteDao[] = []; for (const nullifier of nullifiers) { const noteIndex = this.#nullifierToNoteId.get(nullifier.toString()); @@ -301,8 +303,8 @@ export class KVPxeDatabase implements PxeDatabase { continue; } - const note = NoteDao.fromBuffer(noteBuffer); - if (!note.publicKey.equals(account)) { + const note = IncomingNoteDao.fromBuffer(noteBuffer); + if (!note.ivpkM.equals(account)) { // tried to nullify someone else's note continue; } @@ -310,7 +312,7 @@ export class KVPxeDatabase implements PxeDatabase { nullifiedNotes.push(note); void this.#notes.delete(noteIndex); - void this.#notesByOwner.deleteValue(account.toString(), noteIndex); + void this.#notesByIvpkM.deleteValue(account.toString(), noteIndex); void this.#notesByTxHash.deleteValue(note.txHash.toString(), noteIndex); void this.#notesByContract.deleteValue(note.contractAddress.toString(), noteIndex); void this.#notesByStorageSlot.deleteValue(note.storageSlot.toString(), noteIndex); @@ -319,7 +321,7 @@ export class KVPxeDatabase implements PxeDatabase { void this.#nullifiedNotesByContract.set(note.contractAddress.toString(), noteIndex); void this.#nullifiedNotesByStorageSlot.set(note.storageSlot.toString(), noteIndex); void this.#nullifiedNotesByTxHash.set(note.txHash.toString(), noteIndex); - void this.#nullifiedNotesByOwner.set(note.publicKey.toString(), noteIndex); + void this.#nullifiedNotesByIvpkM.set(note.ivpkM.toString(), noteIndex); void this.#nullifierToNoteId.delete(nullifier.toString()); } diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts new file mode 100644 index 00000000000..f2a52ca58af --- /dev/null +++ b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts @@ -0,0 +1,23 @@ +import { Note, randomTxHash } from '@aztec/circuit-types'; +import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; + +import { OutgoingNoteDao } from './outgoing_note_dao.js'; + +export const randomOutgoingNoteDao = ({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = Fr.random(), + ovpkM = Point.random(), +}: Partial = {}) => { + return new OutgoingNoteDao(note, contractAddress, storageSlot, noteTypeId, txHash, ovpkM); +}; + +describe('Outgoing Note DAO', () => { + it('convert to and from buffer', () => { + const note = randomOutgoingNoteDao(); + const buf = note.toBuffer(); + expect(OutgoingNoteDao.fromBuffer(buf)).toEqual(note); + }); +}); diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.ts b/yarn-project/pxe/src/database/outgoing_note_dao.ts new file mode 100644 index 00000000000..1a8befa043f --- /dev/null +++ b/yarn-project/pxe/src/database/outgoing_note_dao.ts @@ -0,0 +1,64 @@ +import { Note, TxHash } from '@aztec/circuit-types'; +import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; +import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; + +/** + * A note with contextual data which was decrypted as outgoing. + */ +export class OutgoingNoteDao { + constructor( + /** The note as emitted from the Noir contract. */ + public note: Note, + /** The contract address this note is created in. */ + public contractAddress: AztecAddress, + /** The specific storage location of the note on the contract. */ + public storageSlot: Fr, + /** The note type identifier for the contract. */ + public noteTypeId: Fr, + /** The hash of the tx the note was created in. */ + public txHash: TxHash, + /** The public key with which the note was encrypted. */ + public ovpkM: PublicKey, + ) {} + + toBuffer(): Buffer { + return serializeToBuffer([ + this.note, + this.contractAddress, + this.storageSlot, + this.noteTypeId, + this.txHash.buffer, + this.ovpkM, + ]); + } + static fromBuffer(buffer: Buffer | BufferReader) { + const reader = BufferReader.asReader(buffer); + + const note = Note.fromBuffer(reader); + const contractAddress = AztecAddress.fromBuffer(reader); + const storageSlot = Fr.fromBuffer(reader); + const noteTypeId = Fr.fromBuffer(reader); + const txHash = new TxHash(reader.readBytes(TxHash.SIZE)); + const publicKey = Point.fromBuffer(reader); + + return new OutgoingNoteDao(note, contractAddress, storageSlot, noteTypeId, txHash, publicKey); + } + + toString() { + return '0x' + this.toBuffer().toString('hex'); + } + + static fromString(str: string) { + const hex = str.replace(/^0x/, ''); + return OutgoingNoteDao.fromBuffer(Buffer.from(hex, 'hex')); + } + + /** + * Returns the size in bytes of the Note Dao. + * @returns - Its size in bytes. + */ + public getSize() { + const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; + return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 2 + TxHash.SIZE + Point.SIZE_IN_BYTES; + } +} diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index e37d8fc7069..f40a11334e9 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -8,7 +8,8 @@ import { type ContractInstanceWithAddress } from '@aztec/types/contracts'; import { type ContractArtifactDatabase } from './contracts/contract_artifact_db.js'; import { type ContractInstanceDatabase } from './contracts/contract_instance_db.js'; import { type DeferredNoteDao } from './deferred_note_dao.js'; -import { type NoteDao } from './note_dao.js'; +import { type IncomingNoteDao } from './incoming_note_dao.js'; +import { type OutgoingNoteDao } from './outgoing_note_dao.js'; /** * A database interface that provides methods for retrieving, adding, and removing transactional data related to Aztec @@ -50,22 +51,23 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param filter - The filter to apply to the notes. * @returns The requested notes. */ - getNotes(filter: NoteFilter): Promise; + getNotes(filter: NoteFilter): Promise; /** * Adds a note to DB. * @param note - The note to add. */ - addNote(note: NoteDao): Promise; + addNote(note: IncomingNoteDao): Promise; /** * Adds an array of notes to DB. * This function is used to insert multiple notes to the database at once, * which can improve performance when dealing with large numbers of transactions. * - * @param notes - An array of notes. + * @param incomingNotes - An array of notes which were decrypted as incoming. + * @param outgoingNotes - An array of notes which were decrypted as outgoing. */ - addNotes(notes: NoteDao[]): Promise; + addNotes(incomingNotes: IncomingNoteDao[], outgoingNotes: OutgoingNoteDao[]): Promise; /** * Add notes to the database that are intended for us, but we don't yet have the contract. @@ -93,7 +95,7 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param account - A PublicKey instance representing the account for which the records are being removed. * @returns Removed notes. */ - removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise; + removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise; /** * Gets the most recently processed block number. diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index df865b2b2ba..3095494580a 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -6,8 +6,8 @@ import { Fr, Point } from '@aztec/foundation/fields'; import { BenchmarkingContractArtifact } from '@aztec/noir-contracts.js/Benchmarking'; import { SerializableContractInstance } from '@aztec/types/contracts'; -import { type NoteDao } from './note_dao.js'; -import { randomNoteDao } from './note_dao.test.js'; +import { type IncomingNoteDao } from './incoming_note_dao.js'; +import { randomIncomingNoteDao } from './incoming_note_dao.test.js'; import { type PxeDatabase } from './pxe_database.js'; /** @@ -68,13 +68,13 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }); }); - describe('notes', () => { + describe('incoming notes', () => { let owners: CompleteAddress[]; let contractAddresses: AztecAddress[]; let storageSlots: Fr[]; - let notes: NoteDao[]; + let notes: IncomingNoteDao[]; - const filteringTests: [() => NoteFilter, () => NoteDao[]][] = [ + const filteringTests: [() => NoteFilter, () => IncomingNoteDao[]][] = [ [() => ({}), () => notes], [ @@ -94,7 +94,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { [ () => ({ owner: owners[0].address }), - () => notes.filter(note => note.publicKey.equals(owners[0].publicKeys.masterIncomingViewingPublicKey)), + () => notes.filter(note => note.ivpkM.equals(owners[0].publicKeys.masterIncomingViewingPublicKey)), ], [ @@ -113,10 +113,10 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomNoteDao({ + randomIncomingNoteDao({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], - publicKey: owners[i % owners.length].publicKeys.masterIncomingViewingPublicKey, + ivpkM: owners[i % owners.length].publicKeys.masterIncomingViewingPublicKey, index: BigInt(i), }), ); @@ -129,7 +129,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }); it.each(filteringTests)('stores notes in bulk and retrieves notes', async (getFilter, getExpected) => { - await database.addNotes(notes); + await database.addNotes(notes, []); await expect(database.getNotes(getFilter())).resolves.toEqual(getExpected()); }); @@ -141,12 +141,12 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }); it.each(filteringTests)('retrieves nullified notes', async (getFilter, getExpected) => { - await database.addNotes(notes); + await database.addNotes(notes, []); // Nullify all notes and use the same filter as other test cases for (const owner of owners) { const notesToNullify = notes.filter(note => - note.publicKey.equals(owner.publicKeys.masterIncomingViewingPublicKey), + note.ivpkM.equals(owner.publicKeys.masterIncomingViewingPublicKey), ); const nullifiers = notesToNullify.map(note => note.siloedNullifier); await expect( @@ -160,13 +160,13 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }); it('skips nullified notes by default or when requesting active', async () => { - await database.addNotes(notes); + await database.addNotes(notes, []); const notesToNullify = notes.filter(note => - note.publicKey.equals(owners[0].publicKeys.masterIncomingViewingPublicKey), + note.ivpkM.equals(owners[0].publicKeys.masterIncomingViewingPublicKey), ); const nullifiers = notesToNullify.map(note => note.siloedNullifier); - await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].publicKey)).resolves.toEqual( + await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].ivpkM)).resolves.toEqual( notesToNullify, ); @@ -178,13 +178,13 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }); it('returns active and nullified notes when requesting either', async () => { - await database.addNotes(notes); + await database.addNotes(notes, []); const notesToNullify = notes.filter(note => - note.publicKey.equals(owners[0].publicKeys.masterIncomingViewingPublicKey), + note.ivpkM.equals(owners[0].publicKeys.masterIncomingViewingPublicKey), ); const nullifiers = notesToNullify.map(note => note.siloedNullifier); - await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].publicKey)).resolves.toEqual( + await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].ivpkM)).resolves.toEqual( notesToNullify, ); @@ -198,6 +198,8 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }); }); + // TODO(#6867): Add tests for outgoing notes + describe('block header', () => { it('stores and retrieves the block header', async () => { const header = makeHeader(randomInt(1000), INITIAL_L2_BLOCK_NUM); diff --git a/yarn-project/pxe/src/note_processor/note_processor.test.ts b/yarn-project/pxe/src/note_processor/note_processor.test.ts index ef62c2ac4d8..ea2b2ce8bc8 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.test.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.test.ts @@ -1,6 +1,7 @@ import { type AztecNode, EncryptedL2NoteLog, L2Block, TaggedNote } from '@aztec/circuit-types'; import { AztecAddress, + CompleteAddress, Fr, type GrumpkinPrivateKey, INITIAL_L2_BLOCK_NUM, @@ -19,15 +20,16 @@ import { type AcirSimulator } from '@aztec/simulator'; import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; +import { type IncomingNoteDao } from '../database/incoming_note_dao.js'; import { type PxeDatabase } from '../database/index.js'; import { KVPxeDatabase } from '../database/kv_pxe_database.js'; -import { type NoteDao } from '../database/note_dao.js'; +import { type OutgoingNoteDao } from '../database/outgoing_note_dao.js'; import { NoteProcessor } from './note_processor.js'; const TXS_PER_BLOCK = 4; const NUM_NOTE_HASHES_PER_BLOCK = TXS_PER_BLOCK * MAX_NEW_NOTE_HASHES_PER_TX; -/** A wrapper containing info about a note we want to mock and insert into a block */ +/** A wrapper containing info about a note we want to mock and insert into a block. */ class MockNoteRequest { constructor( /** Note we want to insert into a block. */ @@ -78,9 +80,13 @@ describe('Note Processor', () => { let keyStore: MockProxy; let simulator: MockProxy; + const app = AztecAddress.random(); + let ownerIvskM: GrumpkinPrivateKey; let ownerIvpkM: PublicKey; + let ownerOvskM: GrumpkinPrivateKey; let ownerOvKeys: KeyValidationRequest; + let account: CompleteAddress; function mockBlocks(requests: MockNoteRequest[]) { const blocks = []; @@ -118,26 +124,48 @@ describe('Note Processor', () => { beforeAll(() => { const ownerSk = Fr.random(); - const allOwnerKeys = deriveKeys(ownerSk); - const app = AztecAddress.random(); + const partialAddress = Fr.random(); + + account = CompleteAddress.fromSecretKeyAndPartialAddress(ownerSk, partialAddress); + ownerIvpkM = account.publicKeys.masterIncomingViewingPublicKey; + + ({ masterIncomingViewingSecretKey: ownerIvskM, masterOutgoingViewingSecretKey: ownerOvskM } = deriveKeys(ownerSk)); - ownerIvskM = allOwnerKeys.masterIncomingViewingSecretKey; - ownerIvpkM = allOwnerKeys.publicKeys.masterIncomingViewingPublicKey; ownerOvKeys = new KeyValidationRequest( - allOwnerKeys.publicKeys.masterOutgoingViewingPublicKey, - computeOvskApp(allOwnerKeys.masterOutgoingViewingSecretKey, app), + account.publicKeys.masterOutgoingViewingPublicKey, + computeOvskApp(ownerOvskM, app), ); }); - beforeEach(() => { + beforeEach(async () => { database = new KVPxeDatabase(openTmpStore()); addNotesSpy = jest.spyOn(database, 'addNotes'); aztecNode = mock(); keyStore = mock(); simulator = mock(); - keyStore.getMasterIncomingViewingSecretKeyForPublicKey.mockResolvedValue(ownerIvskM); - noteProcessor = new NoteProcessor(ownerIvpkM, keyStore, database, aztecNode, INITIAL_L2_BLOCK_NUM, simulator); + + keyStore.getMasterSecretKey.mockImplementation((pkM: PublicKey) => { + if (pkM.equals(ownerIvpkM)) { + return Promise.resolve(ownerIvskM); + } + if (pkM.equals(ownerOvKeys.pkM)) { + return Promise.resolve(ownerOvskM); + } + throw new Error(`Unknown public key: ${pkM}`); + }); + + keyStore.getMasterIncomingViewingPublicKey.mockResolvedValue(account.publicKeys.masterIncomingViewingPublicKey); + keyStore.getMasterOutgoingViewingPublicKey.mockResolvedValue(account.publicKeys.masterOutgoingViewingPublicKey); + + noteProcessor = await NoteProcessor.create( + account.address, + keyStore, + database, + aztecNode, + INITIAL_L2_BLOCK_NUM, + simulator, + ); simulator.computeNoteHashAndNullifier.mockImplementation((...args) => Promise.resolve({ @@ -153,8 +181,8 @@ describe('Note Processor', () => { addNotesSpy.mockReset(); }); - it('should store a note that belongs to us', async () => { - const request = new MockNoteRequest(TaggedNote.random(), 4, 0, 2, ownerIvpkM, ownerOvKeys); + it('should store an incoming note that belongs to us', async () => { + const request = new MockNoteRequest(TaggedNote.random(app), 4, 0, 2, ownerIvpkM, KeyValidationRequest.random()); const blocks = mockBlocks([request]); @@ -163,19 +191,38 @@ describe('Note Processor', () => { await noteProcessor.process(blocks, encryptedLogs); expect(addNotesSpy).toHaveBeenCalledTimes(1); - expect(addNotesSpy).toHaveBeenCalledWith([ - expect.objectContaining({ - ...request.note.notePayload, - index: request.indexWithinNoteHashTree, - }), - ]); + expect(addNotesSpy).toHaveBeenCalledWith( + [ + expect.objectContaining({ + ...request.note.notePayload, + index: request.indexWithinNoteHashTree, + }), + ], + [], + ); + }, 25_000); + + it('should store an outgoing note that belongs to us', async () => { + const request = new MockNoteRequest(TaggedNote.random(app), 4, 0, 2, Point.random(), ownerOvKeys); + + const blocks = mockBlocks([request]); + + // TODO(#6830): pass in only the blocks + const encryptedLogs = blocks.flatMap(block => block.body.noteEncryptedLogs); + await noteProcessor.process(blocks, encryptedLogs); + + expect(addNotesSpy).toHaveBeenCalledTimes(1); + // For outgoing notes, the resulting DAO does not contain index. + expect(addNotesSpy).toHaveBeenCalledWith([], [expect.objectContaining(request.note.notePayload)]); }, 25_000); it('should store multiple notes that belong to us', async () => { const requests = [ - new MockNoteRequest(TaggedNote.random(), 1, 1, 1, ownerIvpkM, ownerOvKeys), - new MockNoteRequest(TaggedNote.random(), 2, 3, 0, ownerIvpkM, ownerOvKeys), - new MockNoteRequest(TaggedNote.random(), 6, 3, 2, ownerIvpkM, ownerOvKeys), + new MockNoteRequest(TaggedNote.random(app), 1, 1, 1, ownerIvpkM, ownerOvKeys), + new MockNoteRequest(TaggedNote.random(app), 2, 3, 0, Point.random(), ownerOvKeys), + new MockNoteRequest(TaggedNote.random(app), 6, 3, 2, ownerIvpkM, KeyValidationRequest.random()), + new MockNoteRequest(TaggedNote.random(app), 9, 3, 2, Point.random(), KeyValidationRequest.random()), + new MockNoteRequest(TaggedNote.random(app), 12, 3, 2, ownerIvpkM, ownerOvKeys), ]; const blocks = mockBlocks(requests); @@ -185,23 +232,29 @@ describe('Note Processor', () => { await noteProcessor.process(blocks, encryptedLogs); expect(addNotesSpy).toHaveBeenCalledTimes(1); - expect(addNotesSpy).toHaveBeenCalledWith([ - expect.objectContaining({ - ...requests[0].note.notePayload, - // Index 1 log in the 2nd tx. - index: requests[0].indexWithinNoteHashTree, - }), - expect.objectContaining({ - ...requests[1].note.notePayload, - // Index 0 log in the 4th tx. - index: requests[1].indexWithinNoteHashTree, - }), - expect.objectContaining({ - ...requests[2].note.notePayload, - // Index 2 log in the 4th tx. - index: requests[2].indexWithinNoteHashTree, - }), - ]); + expect(addNotesSpy).toHaveBeenCalledWith( + // Incoming should contain notes from requests 0, 2, 4 because in those requests we set owner ivpk. + [ + expect.objectContaining({ + ...requests[0].note.notePayload, + index: requests[0].indexWithinNoteHashTree, + }), + expect.objectContaining({ + ...requests[2].note.notePayload, + index: requests[2].indexWithinNoteHashTree, + }), + expect.objectContaining({ + ...requests[4].note.notePayload, + index: requests[4].indexWithinNoteHashTree, + }), + ], + // Outgoing should contain notes from requests 0, 1, 4 because in those requests we set owner ovKeys. + [ + expect.objectContaining(requests[0].note.notePayload), + expect.objectContaining(requests[1].note.notePayload), + expect.objectContaining(requests[4].note.notePayload), + ], + ); }, 30_000); it('should not store notes that do not belong to us', async () => { @@ -219,8 +272,8 @@ describe('Note Processor', () => { }); it('should be able to recover two note payloads containing the same note', async () => { - const note = TaggedNote.random(); - const note2 = TaggedNote.random(); + const note = TaggedNote.random(app); + const note2 = TaggedNote.random(app); // All note payloads except one have the same contract address, storage slot, and the actual note. const requests = [ new MockNoteRequest(note, 3, 0, 0, ownerIvpkM, ownerOvKeys), @@ -236,19 +289,36 @@ describe('Note Processor', () => { const encryptedLogs = blocks.flatMap(block => block.body.noteEncryptedLogs); await noteProcessor.process(blocks, encryptedLogs); - const addedNoteDaos: NoteDao[] = addNotesSpy.mock.calls[0][0]; - expect(addedNoteDaos.map(dao => dao)).toEqual([ - expect.objectContaining({ ...requests[0].note.notePayload }), - expect.objectContaining({ ...requests[1].note.notePayload }), - expect.objectContaining({ ...requests[2].note.notePayload }), - expect.objectContaining({ ...requests[3].note.notePayload }), - expect.objectContaining({ ...requests[4].note.notePayload }), - ]); + // First we check incoming + { + const addedIncoming: IncomingNoteDao[] = addNotesSpy.mock.calls[0][0]; + expect(addedIncoming.map(dao => dao)).toEqual([ + expect.objectContaining({ ...requests[0].note.notePayload, index: requests[0].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[1].note.notePayload, index: requests[1].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[2].note.notePayload, index: requests[2].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[3].note.notePayload, index: requests[3].indexWithinNoteHashTree }), + expect.objectContaining({ ...requests[4].note.notePayload, index: requests[4].indexWithinNoteHashTree }), + ]); + + // Check that every note has a different nonce. + const nonceSet = new Set(); + addedIncoming.forEach(info => nonceSet.add(info.nonce.value)); + expect(nonceSet.size).toBe(requests.length); + } - // Check that every note has a different nonce. - const nonceSet = new Set(); - addedNoteDaos.forEach(info => nonceSet.add(info.nonce.value)); - expect(nonceSet.size).toBe(requests.length); + // Then we check outgoing + { + const addedOutgoing: OutgoingNoteDao[] = addNotesSpy.mock.calls[0][1]; + expect(addedOutgoing.map(dao => dao)).toEqual([ + expect.objectContaining(requests[0].note.notePayload), + expect.objectContaining(requests[1].note.notePayload), + expect.objectContaining(requests[2].note.notePayload), + expect.objectContaining(requests[3].note.notePayload), + expect.objectContaining(requests[4].note.notePayload), + ]); + + // Outgoing note daos do not have a nonce so we don't check it. + } }); it('advances the block number', async () => { @@ -264,7 +334,7 @@ describe('Note Processor', () => { }); it('should restore the last block number processed and ignore the starting block', async () => { - const request = new MockNoteRequest(TaggedNote.random(), 6, 0, 2, ownerIvpkM, ownerOvKeys); + const request = new MockNoteRequest(TaggedNote.random(), 6, 0, 2, Point.random(), KeyValidationRequest.random()); const blocks = mockBlocks([request]); @@ -272,8 +342,8 @@ describe('Note Processor', () => { const encryptedLogs = blocks.flatMap(block => block.body.noteEncryptedLogs); await noteProcessor.process(blocks, encryptedLogs); - const newNoteProcessor = new NoteProcessor( - ownerIvpkM, + const newNoteProcessor = await NoteProcessor.create( + account.address, keyStore, database, aztecNode, diff --git a/yarn-project/pxe/src/note_processor/note_processor.ts b/yarn-project/pxe/src/note_processor/note_processor.ts index 7c9b37f9423..9a7145b5dc1 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.ts @@ -6,31 +6,35 @@ import { TaggedNote, } from '@aztec/circuit-types'; import { type NoteProcessorStats } from '@aztec/circuit-types/stats'; -import { INITIAL_L2_BLOCK_NUM, MAX_NEW_NOTE_HASHES_PER_TX, type PublicKey } from '@aztec/circuits.js'; +import { + type AztecAddress, + INITIAL_L2_BLOCK_NUM, + MAX_NEW_NOTE_HASHES_PER_TX, + type PublicKey, +} from '@aztec/circuits.js'; import { type Fr } from '@aztec/foundation/fields'; -import { createDebugLogger } from '@aztec/foundation/log'; +import { type Logger, createDebugLogger } from '@aztec/foundation/log'; import { Timer } from '@aztec/foundation/timer'; import { type KeyStore } from '@aztec/key-store'; -import { ContractNotFoundError } from '@aztec/simulator'; +import { type AcirSimulator } from '@aztec/simulator'; -import { DeferredNoteDao } from '../database/deferred_note_dao.js'; +import { type DeferredNoteDao } from '../database/deferred_note_dao.js'; +import { type IncomingNoteDao } from '../database/incoming_note_dao.js'; import { type PxeDatabase } from '../database/index.js'; -import { type NoteDao } from '../database/note_dao.js'; +import { type OutgoingNoteDao } from '../database/outgoing_note_dao.js'; import { getAcirSimulator } from '../simulator/index.js'; -import { produceNoteDao } from './produce_note_dao.js'; +import { produceNoteDaos } from './produce_note_dao.js'; /** * Contains all the decrypted data in this array so that we can later batch insert it all into the database. */ interface ProcessedData { - /** - * Holds L2 block. - */ + /** Holds L2 block. */ block: L2Block; - /** - * DAOs of processed notes. - */ - noteDaos: NoteDao[]; + /** DAOs of processed incoming notes. */ + incomingNotes: IncomingNoteDao[]; + /** DAOs of processed outgoing notes. */ + outgoingNotes: OutgoingNoteDao[]; } /** @@ -42,21 +46,45 @@ export class NoteProcessor { public readonly timer: Timer = new Timer(); /** Stats accumulated for this processor. */ - public readonly stats: NoteProcessorStats = { seen: 0, decrypted: 0, deferred: 0, failed: 0, blocks: 0, txs: 0 }; + public readonly stats: NoteProcessorStats = { + seen: 0, + decryptedIncoming: 0, + decryptedOutgoing: 0, + deferred: 0, + failed: 0, + blocks: 0, + txs: 0, + }; - constructor( - /** - * The public counterpart to the private key to be used in note decryption. - */ - public readonly masterIncomingViewingPublicKey: PublicKey, + private constructor( + public readonly account: AztecAddress, + /** The public counterpart to the secret key to be used in the decryption of incoming note logs. */ + private readonly ivpkM: PublicKey, + /** The public counterpart to the secret key to be used in the decryption of outgoing note logs. */ + private readonly ovpkM: PublicKey, private keyStore: KeyStore, private db: PxeDatabase, private node: AztecNode, - private startingBlock: number = INITIAL_L2_BLOCK_NUM, - private simulator = getAcirSimulator(db, node, keyStore), - private log = createDebugLogger('aztec:note_processor'), + private startingBlock: number, + private simulator: AcirSimulator, + private log: Logger, ) {} + public static async create( + account: AztecAddress, + keyStore: KeyStore, + db: PxeDatabase, + node: AztecNode, + startingBlock: number = INITIAL_L2_BLOCK_NUM, + simulator = getAcirSimulator(db, node, keyStore), + log = createDebugLogger('aztec:note_processor'), + ) { + const ivpkM = await keyStore.getMasterIncomingViewingPublicKey(account); + const ovpkM = await keyStore.getMasterOutgoingViewingPublicKey(account); + + return new NoteProcessor(account, ivpkM, ovpkM, keyStore, db, node, startingBlock, simulator, log); + } + /** * Check if the NoteProcessor is synchronized with the remote block number. * The function queries the remote block number from the AztecNode and compares it with the syncedToBlock value in the NoteProcessor. @@ -77,7 +105,7 @@ export class NoteProcessor { } private getSyncedToBlock(): number { - return this.db.getSynchedBlockNumberForPublicKey(this.masterIncomingViewingPublicKey) ?? this.startingBlock - 1; + return this.db.getSynchedBlockNumberForPublicKey(this.ivpkM) ?? this.startingBlock - 1; } /** @@ -100,7 +128,12 @@ export class NoteProcessor { const blocksAndNotes: ProcessedData[] = []; // Keep track of notes that we couldn't process because the contract was not found. - const deferredNoteDaos: DeferredNoteDao[] = []; + // Note that there are no deferred outgoing notes because we don't need the contract there for anything since we + // are not attempting to derive a nullifier. + const deferredNoteDaosIncoming: DeferredNoteDao[] = []; + + const ivskM = await this.keyStore.getMasterSecretKey(this.ivpkM); + const ovskM = await this.keyStore.getMasterSecretKey(this.ovpkM); // Iterate over both blocks and encrypted logs. for (let blockIndex = 0; blockIndex < encryptedL2BlockLogs.length; ++blockIndex) { @@ -113,10 +146,8 @@ export class NoteProcessor { // We are using set for `userPertainingTxIndices` to avoid duplicates. This would happen in case there were // multiple encrypted logs in a tx pertaining to a user. - const noteDaos: NoteDao[] = []; - const secretKey = await this.keyStore.getMasterIncomingViewingSecretKeyForPublicKey( - this.masterIncomingViewingPublicKey, - ); + const incomingNotes: IncomingNoteDao[] = []; + const outgoingNotes: OutgoingNoteDao[] = []; // Iterate over all the encrypted logs and try decrypting them. If successful, store the note. for (let indexOfTxInABlock = 0; indexOfTxInABlock < txLogs.length; ++indexOfTxInABlock) { @@ -130,43 +161,48 @@ export class NoteProcessor { for (const functionLogs of txFunctionLogs) { for (const log of functionLogs.logs) { this.stats.seen++; - // @todo Issue(#6410) We should also try decrypting as outgoing if this fails. - const taggedNote = TaggedNote.decryptAsIncoming(log.data, secretKey); - if (taggedNote?.notePayload) { - const { notePayload: payload } = taggedNote; - // We have successfully decrypted the data. + const incomingTaggedNote = TaggedNote.decryptAsIncoming(log.data, ivskM)!; + const outgoingTaggedNote = TaggedNote.decryptAsOutgoing(log.data, ovskM)!; + + if (incomingTaggedNote || outgoingTaggedNote) { + if ( + incomingTaggedNote && + outgoingTaggedNote && + !incomingTaggedNote.notePayload.equals(outgoingTaggedNote.notePayload) + ) { + throw new Error('Incoming and outgoing note payloads do not match.'); + } + + const payload = incomingTaggedNote?.notePayload || outgoingTaggedNote?.notePayload; + const txHash = block.body.txEffects[indexOfTxInABlock].txHash; - try { - const noteDao = await produceNoteDao( - this.simulator, - this.masterIncomingViewingPublicKey, - payload, - txHash, - newNoteHashes, - dataStartIndexForTx, - excludedIndices, - ); - noteDaos.push(noteDao); - this.stats.decrypted++; - } catch (e) { - if (e instanceof ContractNotFoundError) { - this.stats.deferred++; - this.log.warn(e.message); - const deferredNoteDao = new DeferredNoteDao( - this.masterIncomingViewingPublicKey, - payload.note, - payload.contractAddress, - payload.storageSlot, - payload.noteTypeId, - txHash, - newNoteHashes, - dataStartIndexForTx, - ); - deferredNoteDaos.push(deferredNoteDao); - } else { - this.stats.failed++; - this.log.error(`Could not process note because of "${e}". Discarding note...`); - } + const { incomingNote, outgoingNote, incomingDeferredNote } = await produceNoteDaos( + this.simulator, + incomingTaggedNote ? this.ivpkM : undefined, + outgoingTaggedNote ? this.ovpkM : undefined, + payload, + txHash, + newNoteHashes, + dataStartIndexForTx, + excludedIndices, + this.log, + ); + + if (incomingNote) { + incomingNotes.push(incomingNote); + this.stats.decryptedIncoming++; + } + if (outgoingNote) { + outgoingNotes.push(outgoingNote); + this.stats.decryptedOutgoing++; + } + if (incomingDeferredNote) { + deferredNoteDaosIncoming.push(incomingDeferredNote); + this.stats.deferred++; + } + + if (incomingNote == undefined && outgoingNote == undefined && incomingDeferredNote == undefined) { + this.stats.failed++; } } } @@ -175,15 +211,16 @@ export class NoteProcessor { blocksAndNotes.push({ block: l2Blocks[blockIndex], - noteDaos, + incomingNotes, + outgoingNotes, }); } await this.processBlocksAndNotes(blocksAndNotes); - await this.processDeferredNotes(deferredNoteDaos); + await this.processDeferredNotes(deferredNoteDaosIncoming); const syncedToBlock = l2Blocks[l2Blocks.length - 1].number; - await this.db.setSynchedBlockNumberForPublicKey(this.masterIncomingViewingPublicKey, syncedToBlock); + await this.db.setSynchedBlockNumberForPublicKey(this.ivpkM, syncedToBlock); this.log.debug(`Synched block ${syncedToBlock}`); } @@ -198,22 +235,26 @@ export class NoteProcessor { * @param blocksAndNotes - Array of objects containing L2 blocks, user-pertaining transaction indices, and NoteDaos. */ private async processBlocksAndNotes(blocksAndNotes: ProcessedData[]) { - const noteDaos = blocksAndNotes.flatMap(b => b.noteDaos); - if (noteDaos.length) { - await this.db.addNotes(noteDaos); - noteDaos.forEach(noteDao => { + const incomingNotes = blocksAndNotes.flatMap(b => b.incomingNotes); + const outgoingNotes = blocksAndNotes.flatMap(b => b.outgoingNotes); + if (incomingNotes.length || outgoingNotes.length) { + await this.db.addNotes(incomingNotes, outgoingNotes); + incomingNotes.forEach(noteDao => { this.log.verbose( - `Added note for contract ${noteDao.contractAddress} at slot ${ + `Added incoming note for contract ${noteDao.contractAddress} at slot ${ noteDao.storageSlot } with nullifier ${noteDao.siloedNullifier.toString()}`, ); }); + outgoingNotes.forEach(noteDao => { + this.log.verbose(`Added outgoing note for contract ${noteDao.contractAddress} at slot ${noteDao.storageSlot}`); + }); } const newNullifiers: Fr[] = blocksAndNotes.flatMap(b => b.block.body.txEffects.flatMap(txEffect => txEffect.nullifiers), ); - const removedNotes = await this.db.removeNullifiedNotes(newNullifiers, this.masterIncomingViewingPublicKey); + const removedNotes = await this.db.removeNullifiedNotes(newNullifiers, this.ivpkM); removedNotes.forEach(noteDao => { this.log.verbose( `Removed note for contract ${noteDao.contractAddress} at slot ${ @@ -245,37 +286,45 @@ export class NoteProcessor { * Retry decoding the given deferred notes because we now have the contract code. * * @param deferredNoteDaos - notes that we have previously deferred because the contract was not found - * @returns An array of NoteDaos that were successfully decoded. + * @returns An array of incoming notes that were successfully decoded. * * @remarks Caller is responsible for making sure that we have the contract for the * deferred notes provided: we will not retry notes that fail again. */ - public async decodeDeferredNotes(deferredNoteDaos: DeferredNoteDao[]): Promise { + public async decodeDeferredNotes(deferredNoteDaos: DeferredNoteDao[]): Promise { const excludedIndices: Set = new Set(); - const noteDaos: NoteDao[] = []; + const incomingNotes: IncomingNoteDao[] = []; + for (const deferredNote of deferredNoteDaos) { - const { note, contractAddress, storageSlot, noteTypeId, txHash, newNoteHashes, dataStartIndexForTx } = + const { ivpkM, note, contractAddress, storageSlot, noteTypeId, txHash, newNoteHashes, dataStartIndexForTx } = deferredNote; const payload = new L1NotePayload(note, contractAddress, storageSlot, noteTypeId); - try { - const noteDao = await produceNoteDao( - this.simulator, - this.masterIncomingViewingPublicKey, - payload, - txHash, - newNoteHashes, - dataStartIndexForTx, - excludedIndices, - ); - noteDaos.push(noteDao); - this.stats.decrypted++; - } catch (e) { - this.stats.failed++; - this.log.warn(`Could not process deferred note because of "${e}". Discarding note...`); + if (!ivpkM.equals(this.ivpkM)) { + // The note is not for this account, so we skip it. + continue; + } + + const { incomingNote } = await produceNoteDaos( + this.simulator, + this.ivpkM, + undefined, + payload, + txHash, + newNoteHashes, + dataStartIndexForTx, + excludedIndices, + this.log, + ); + + if (!incomingNote) { + throw new Error('Deferred note could not be decoded.'); } + + incomingNotes.push(incomingNote); + this.stats.decryptedIncoming++; } - return noteDaos; + return incomingNotes; } } diff --git a/yarn-project/pxe/src/note_processor/produce_note_dao.ts b/yarn-project/pxe/src/note_processor/produce_note_dao.ts index d5de1d4dea9..4eaedb79b08 100644 --- a/yarn-project/pxe/src/note_processor/produce_note_dao.ts +++ b/yarn-project/pxe/src/note_processor/produce_note_dao.ts @@ -1,9 +1,12 @@ import { type L1NotePayload, type TxHash } from '@aztec/circuit-types'; import { Fr, type PublicKey } from '@aztec/circuits.js'; import { computeNoteHashNonce, siloNullifier } from '@aztec/circuits.js/hash'; -import { type AcirSimulator } from '@aztec/simulator'; +import { type Logger } from '@aztec/foundation/log'; +import { type AcirSimulator, ContractNotFoundError } from '@aztec/simulator'; -import { NoteDao } from '../database/note_dao.js'; +import { DeferredNoteDao } from '../database/deferred_note_dao.js'; +import { IncomingNoteDao } from '../database/incoming_note_dao.js'; +import { OutgoingNoteDao } from '../database/outgoing_note_dao.js'; /** * Decodes a note from a transaction that we know was intended for us. @@ -11,87 +14,144 @@ import { NoteDao } from '../database/note_dao.js'; * Accepts a set of excluded indices, which are indices that have been assigned a note in the same tx. * Inserts the index of the note into the excludedIndices set if the note is successfully decoded. * - * @param publicKey - The public counterpart to the private key to be used in note decryption. + * @param ivpkM - The public counterpart to the secret key to be used in the decryption of incoming note logs. + * @param ovpkM - The public counterpart to the secret key to be used in the decryption of outgoing note logs. * @param payload - An instance of l1NotePayload. * @param txHash - The hash of the transaction that created the note. Equivalent to the first nullifier of the transaction. * @param newNoteHashes - New note hashes in this transaction, one of which belongs to this note. * @param dataStartIndexForTx - The next available leaf index for the note hash tree for this transaction. * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same l1NotePayload, we need to find a different index for each replicate. * @param simulator - An instance of AcirSimulator. - * @returns an instance of NoteDao, or throws. inserts the index of the note into the excludedIndices set. + * @returns An object containing the incoming note, outgoing note, and deferred note. */ -export async function produceNoteDao( +export async function produceNoteDaos( simulator: AcirSimulator, - publicKey: PublicKey, + ivpkM: PublicKey | undefined, + ovpkM: PublicKey | undefined, payload: L1NotePayload, txHash: TxHash, newNoteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, -): Promise { - const { commitmentIndex, nonce, innerNoteHash, siloedNullifier } = await findNoteIndexAndNullifier( - simulator, - newNoteHashes, - txHash, - payload, - excludedIndices, - ); - const index = BigInt(dataStartIndexForTx + commitmentIndex); - excludedIndices?.add(commitmentIndex); - return new NoteDao( - payload.note, - payload.contractAddress, - payload.storageSlot, - payload.noteTypeId, - txHash, - nonce, - innerNoteHash, - siloedNullifier, - index, - publicKey, - ); + log: Logger, +): Promise<{ + incomingNote: IncomingNoteDao | undefined; + outgoingNote: OutgoingNoteDao | undefined; + incomingDeferredNote: DeferredNoteDao | undefined; +}> { + if (!ivpkM && !ovpkM) { + throw new Error('Both ivpkM and ovpkM are undefined. Cannot create note.'); + } + + let incomingNote: IncomingNoteDao | undefined; + let outgoingNote: OutgoingNoteDao | undefined; + + // Note that there are no deferred outgoing notes because we don't need the contract there for anything since we + // are not attempting to derive a nullifier. + let incomingDeferredNote: DeferredNoteDao | undefined; + + if (ovpkM) { + outgoingNote = new OutgoingNoteDao( + payload.note, + payload.contractAddress, + payload.storageSlot, + payload.noteTypeId, + txHash, + ovpkM, + ); + } + + try { + if (ivpkM) { + const { noteHashIndex, nonce, innerNoteHash, siloedNullifier } = await findNoteIndexAndNullifier( + simulator, + newNoteHashes, + txHash, + payload, + excludedIndices, + ); + const index = BigInt(dataStartIndexForTx + noteHashIndex); + excludedIndices?.add(noteHashIndex); + + incomingNote = new IncomingNoteDao( + payload.note, + payload.contractAddress, + payload.storageSlot, + payload.noteTypeId, + txHash, + nonce, + innerNoteHash, + siloedNullifier, + index, + ivpkM, + ); + } + } catch (e) { + if (e instanceof ContractNotFoundError) { + log.warn(e.message); + + if (ivpkM) { + incomingDeferredNote = new DeferredNoteDao( + ivpkM, + payload.note, + payload.contractAddress, + payload.storageSlot, + payload.noteTypeId, + txHash, + newNoteHashes, + dataStartIndexForTx, + ); + } + } else { + log.error(`Could not process note because of "${e}". Discarding note...`); + } + } + + return { + incomingNote, + outgoingNote, + incomingDeferredNote, + }; } /** - * Find the index of the note in the note hash tree by computing the note hash with different nonce and see which - * commitment for the current tx matches this value. - * Compute a nullifier for a given l1NotePayload. - * The nullifier is calculated using the private key of the account, - * contract address, and the note associated with the l1NotePayload. - * This method assists in identifying spent commitments in the private state. - * @param commitments - Commitments in the tx. One of them should be the note's commitment. - * @param txHash - First nullifier in the tx. - * @param l1NotePayload - An instance of l1NotePayload. + * Finds nonce, index, inner hash and siloed nullifier for a given note. + * @dev Finds the index in the note hash tree by computing the note hash with different nonce and see which hash for + * the current tx matches this value. + * @remarks This method assists in identifying spent notes in the note hash tree. + * @param noteHashes - Note hashes in the tx. One of them should correspond to the note we are looking for + * @param txHash - Hash of a tx the note was emitted in. + * @param l1NotePayload - The note payload. * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same * l1NotePayload. We need to find a different index for each replicate. - * @returns Information for a decrypted note, including the index of its commitment, nonce, inner note - * hash, and the siloed nullifier. Throw if cannot find the nonce for the note. + * @returns Nonce, index, inner hash and siloed nullifier for a given note. + * @throws If cannot find the nonce for the note. */ async function findNoteIndexAndNullifier( simulator: AcirSimulator, - commitments: Fr[], + noteHashes: Fr[], txHash: TxHash, { contractAddress, storageSlot, noteTypeId, note }: L1NotePayload, excludedIndices: Set, ) { - let commitmentIndex = 0; + let noteHashIndex = 0; let nonce: Fr | undefined; let innerNoteHash: Fr | undefined; let siloedNoteHash: Fr | undefined; let innerNullifier: Fr | undefined; const firstNullifier = Fr.fromBuffer(txHash.toBuffer()); - for (; commitmentIndex < commitments.length; ++commitmentIndex) { - if (excludedIndices.has(commitmentIndex)) { + for (; noteHashIndex < noteHashes.length; ++noteHashIndex) { + if (excludedIndices.has(noteHashIndex)) { continue; } - const commitment = commitments[commitmentIndex]; - if (commitment.equals(Fr.ZERO)) { + const noteHash = noteHashes[noteHashIndex]; + if (noteHash.equals(Fr.ZERO)) { break; } - const expectedNonce = computeNoteHashNonce(firstNullifier, commitmentIndex); + const expectedNonce = computeNoteHashNonce(firstNullifier, noteHashIndex); ({ innerNoteHash, siloedNoteHash, innerNullifier } = await simulator.computeNoteHashAndNullifier( contractAddress, expectedNonce, @@ -100,7 +160,7 @@ async function findNoteIndexAndNullifier( note, )); - if (commitment.equals(siloedNoteHash)) { + if (noteHash.equals(siloedNoteHash)) { nonce = expectedNonce; break; } @@ -109,11 +169,11 @@ async function findNoteIndexAndNullifier( if (!nonce) { // NB: this used to warn the user that a decrypted log didn't match any notes. // This was previously fine as we didn't chop transient note logs, but now we do (#1641 complete). - throw new Error('Cannot find a matching commitment for the note.'); + throw new Error('Cannot find a matching note hash for the note.'); } return { - commitmentIndex, + noteHashIndex, nonce, innerNoteHash: innerNoteHash!, siloedNullifier: siloNullifier(contractAddress, innerNullifier!), diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 268d16d00e8..f7d6b8b02c5 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -57,8 +57,8 @@ import { type NodeInfo } from '@aztec/types/interfaces'; import { type PXEServiceConfig, getPackageInfo } from '../config/index.js'; import { ContractDataOracle } from '../contract_data_oracle/index.js'; +import { IncomingNoteDao } from '../database/incoming_note_dao.js'; import { type PxeDatabase } from '../database/index.js'; -import { NoteDao } from '../database/note_dao.js'; import { KernelOracle } from '../kernel_oracle/index.js'; import { KernelProver } from '../kernel_prover/kernel_prover.js'; import { getAcirSimulator } from '../simulator/index.js'; @@ -121,11 +121,7 @@ export class PXEService implements PXE { } count++; - this.synchronizer.addAccount( - address.publicKeys.masterIncomingViewingPublicKey, - this.keyStore, - this.config.l2StartingBlock, - ); + await this.synchronizer.addAccount(address.address, this.keyStore, this.config.l2StartingBlock); } if (count > 0) { @@ -184,10 +180,7 @@ export class PXEService implements PXE { this.log.info(`Account:\n "${accountCompleteAddress.address.toString()}"\n already registered.`); return accountCompleteAddress; } else { - const masterIncomingViewingPublicKey = await this.keyStore.getMasterIncomingViewingPublicKey( - accountCompleteAddress.address, - ); - this.synchronizer.addAccount(masterIncomingViewingPublicKey, this.keyStore, this.config.l2StartingBlock); + await this.synchronizer.addAccount(accountCompleteAddress.address, this.keyStore, this.config.l2StartingBlock); this.log.info(`Registered account ${accountCompleteAddress.address.toString()}`); this.log.debug(`Registered account\n ${accountCompleteAddress.toReadableString()}`); } @@ -293,10 +286,10 @@ export class PXEService implements PXE { let owner = filter.owner; if (owner === undefined) { const completeAddresses = (await this.db.getCompleteAddresses()).find(address => - address.publicKeys.masterIncomingViewingPublicKey.equals(dao.publicKey), + address.publicKeys.masterIncomingViewingPublicKey.equals(dao.ivpkM), ); if (completeAddresses === undefined) { - throw new Error(`Cannot find complete address for public key ${dao.publicKey.toString()}`); + throw new Error(`Cannot find complete address for IvpkM ${dao.ivpkM.toString()}`); } owner = completeAddresses.address; } @@ -337,7 +330,7 @@ export class PXEService implements PXE { } await this.db.addNote( - new NoteDao( + new IncomingNoteDao( note.note, note.contractAddress, note.storageSlot, diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts index 36c0590a51e..d919548ee4b 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts @@ -132,7 +132,7 @@ describe('Synchronizer', () => { const partialAddress = Fr.random(); const completeAddress = await keyStore.addAccount(secretKey, partialAddress); await database.addCompleteAddress(completeAddress); - synchronizer.addAccount(completeAddress.publicKeys.masterIncomingViewingPublicKey, keyStore, startingBlockNum); + await synchronizer.addAccount(completeAddress.address, keyStore, startingBlockNum); return completeAddress; }; diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index d37c63eeb20..ceccd37527d 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -7,8 +7,8 @@ import { RunningPromise } from '@aztec/foundation/running-promise'; import { type KeyStore } from '@aztec/key-store'; import { type DeferredNoteDao } from '../database/deferred_note_dao.js'; +import { type IncomingNoteDao } from '../database/incoming_note_dao.js'; import { type PxeDatabase } from '../database/index.js'; -import { type NoteDao } from '../database/note_dao.js'; import { NoteProcessor } from '../note_processor/index.js'; /** @@ -193,7 +193,7 @@ export class Synchronizer { } this.log.debug( - `Catching up note processor ${noteProcessor.masterIncomingViewingPublicKey.toString()} by processing ${ + `Catching up note processor ${noteProcessor.account.toString()} by processing ${ blocks.length - index } blocks`, ); @@ -201,19 +201,16 @@ export class Synchronizer { if (noteProcessor.status.syncedToBlock === toBlockNumber) { // Note processor caught up, move it to `noteProcessors` from `noteProcessorsToCatchUp`. - this.log.debug( - `Note processor for ${noteProcessor.masterIncomingViewingPublicKey.toString()} has caught up`, - { - eventName: 'note-processor-caught-up', - publicKey: noteProcessor.masterIncomingViewingPublicKey.toString(), - duration: noteProcessor.timer.ms(), - dbSize: this.db.estimateSize(), - ...noteProcessor.stats, - } satisfies NoteProcessorCaughtUpStats, - ); + this.log.debug(`Note processor for ${noteProcessor.account.toString()} has caught up`, { + eventName: 'note-processor-caught-up', + account: noteProcessor.account.toString(), + duration: noteProcessor.timer.ms(), + dbSize: this.db.estimateSize(), + ...noteProcessor.stats, + } satisfies NoteProcessorCaughtUpStats); this.noteProcessorsToCatchUp = this.noteProcessorsToCatchUp.filter( - np => !np.masterIncomingViewingPublicKey.equals(noteProcessor.masterIncomingViewingPublicKey), + np => !np.account.equals(noteProcessor.account), ); this.noteProcessors.push(noteProcessor); } @@ -257,14 +254,14 @@ export class Synchronizer { * @param startingBlock - The block where to start scanning for notes for this accounts. * @returns A promise that resolves once the account is added to the Synchronizer. */ - public addAccount(publicKey: PublicKey, keyStore: KeyStore, startingBlock: number) { - const predicate = (x: NoteProcessor) => x.masterIncomingViewingPublicKey.equals(publicKey); + public async addAccount(account: AztecAddress, keyStore: KeyStore, startingBlock: number) { + const predicate = (x: NoteProcessor) => x.account.equals(account); const processor = this.noteProcessors.find(predicate) ?? this.noteProcessorsToCatchUp.find(predicate); if (processor) { return; } - this.noteProcessorsToCatchUp.push(new NoteProcessor(publicKey, keyStore, this.db, this.node, startingBlock)); + this.noteProcessorsToCatchUp.push(await NoteProcessor.create(account, keyStore, this.db, this.node, startingBlock)); } /** @@ -280,9 +277,9 @@ export class Synchronizer { if (!completeAddress) { throw new Error(`Checking if account is synched is not possible for ${account} because it is not registered.`); } - const findByPublicKey = (x: NoteProcessor) => - x.masterIncomingViewingPublicKey.equals(completeAddress.publicKeys.masterIncomingViewingPublicKey); - const processor = this.noteProcessors.find(findByPublicKey) ?? this.noteProcessorsToCatchUp.find(findByPublicKey); + const findByAccountAddress = (x: NoteProcessor) => x.account.equals(completeAddress.address); + const processor = + this.noteProcessors.find(findByAccountAddress) ?? this.noteProcessorsToCatchUp.find(findByAccountAddress); if (!processor) { throw new Error( `Checking if account is synched is not possible for ${account} because it is only registered as a recipient.`, @@ -314,9 +311,7 @@ export class Synchronizer { const lastBlockNumber = this.getSynchedBlockNumber(); return { blocks: lastBlockNumber, - notes: Object.fromEntries( - this.noteProcessors.map(n => [n.masterIncomingViewingPublicKey.toString(), n.status.syncedToBlock]), - ), + notes: Object.fromEntries(this.noteProcessors.map(n => [n.account.toString(), n.status.syncedToBlock])), }; } @@ -325,7 +320,7 @@ export class Synchronizer { * @returns The note processor stats for notes for each public key being tracked. */ public getSyncStats() { - return Object.fromEntries(this.noteProcessors.map(n => [n.masterIncomingViewingPublicKey.toString(), n.stats])); + return Object.fromEntries(this.noteProcessors.map(n => [n.account.toString(), n.stats])); } /** @@ -348,23 +343,21 @@ export class Synchronizer { } // keep track of decoded notes - const newNotes: NoteDao[] = []; + const incomingNotes: IncomingNoteDao[] = []; // now process each txHash for (const deferredNotes of txHashToDeferredNotes.values()) { // to be safe, try each note processor in case the deferred notes are for different accounts. for (const processor of this.noteProcessors) { - const decodedNotes = await processor.decodeDeferredNotes( - deferredNotes.filter(n => n.publicKey.equals(processor.masterIncomingViewingPublicKey)), - ); - newNotes.push(...decodedNotes); + const notes = await processor.decodeDeferredNotes(deferredNotes); + incomingNotes.push(...notes); } } // now drop the deferred notes, and add the decoded notes await this.db.removeDeferredNotesByContract(contractAddress); - await this.db.addNotes(newNotes); + await this.db.addNotes(incomingNotes, []); - newNotes.forEach(noteDao => { + incomingNotes.forEach(noteDao => { this.log.debug( `Decoded deferred note for contract ${noteDao.contractAddress} at slot ${ noteDao.storageSlot @@ -372,12 +365,12 @@ export class Synchronizer { ); }); - // now group the decoded notes by public key - const publicKeyToNotes: Map = new Map(); - for (const noteDao of newNotes) { - const notesForPublicKey = publicKeyToNotes.get(noteDao.publicKey) ?? []; + // now group the decoded incoming notes by public key + const publicKeyToNotes: Map = new Map(); + for (const noteDao of incomingNotes) { + const notesForPublicKey = publicKeyToNotes.get(noteDao.ivpkM) ?? []; notesForPublicKey.push(noteDao); - publicKeyToNotes.set(noteDao.publicKey, notesForPublicKey); + publicKeyToNotes.set(noteDao.ivpkM, notesForPublicKey); } // now for each group, look for the nullifiers in the nullifier tree diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index 0d171769cc5..e11a1d60d44 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -171,8 +171,8 @@ function processCircuitWitnessGeneration(entry: CircuitWitnessGenerationStats, r * Processes an entry with event name 'note-processor-caught-up' and updates results */ function processNoteProcessorCaughtUp(entry: NoteProcessorCaughtUpStats, results: BenchmarkCollectedResults) { - const { decrypted, blocks, dbSize } = entry; - if (BENCHMARK_HISTORY_CHAIN_LENGTHS.includes(blocks) && decrypted > 0) { + const { decryptedIncoming, decryptedOutgoing, blocks, dbSize } = entry; + if (BENCHMARK_HISTORY_CHAIN_LENGTHS.includes(blocks) && (decryptedIncoming > 0 || decryptedOutgoing > 0)) { append(results, 'pxe_database_size_in_bytes', blocks, dbSize); } }