Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(sync): Sync latest globals within merkle tree ops #1612

Merged
merged 16 commits into from
Aug 18, 2023
Merged
4 changes: 1 addition & 3 deletions yarn-project/aztec-node/src/aztec-node/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -338,8 +338,6 @@ export class AztecNodeService implements AztecNode {
const getTreeRoot = async (id: MerkleTreeId) =>
Fr.fromBuffer((await this.merkleTreeDB.getTreeInfo(id, false)).root);

const globalsHash = this.worldStateSynchroniser.latestGlobalVariablesHash;

return new HistoricBlockData(
await getTreeRoot(MerkleTreeId.PRIVATE_DATA_TREE),
await getTreeRoot(MerkleTreeId.NULLIFIER_TREE),
Expand All @@ -348,7 +346,7 @@ export class AztecNodeService implements AztecNode {
await getTreeRoot(MerkleTreeId.BLOCKS_TREE),
Fr.ZERO,
await getTreeRoot(MerkleTreeId.PUBLIC_DATA_TREE),
globalsHash,
await this.merkleTreeDB.getLatestGlobalVariablesHash(),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm is there any chance this would advance such that the global variable hash is too new? Like if the system moves on after await getTreeRoot(MerkleTreeId.PUBLIC_DATA_TREE),

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't fully understand the code, but if we're fetching data it would be good to fetch it all at once representing one snapshot, 'latest' makes me nervous when used with other async calls

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These changes have unfortunately broken the p2p tests, this could be the culprit

);
}
}
2 changes: 1 addition & 1 deletion yarn-project/aztec-rpc/src/synchroniser/synchroniser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ export class Synchroniser {
protected async initialSync() {
const [blockNumber, historicBlockData] = await Promise.all([
this.node.getBlockHeight(),
Promise.resolve(this.node.getHistoricBlockData()),
this.node.getHistoricBlockData(),
]);
this.initialSyncBlockHeight = blockNumber;
this.synchedToBlock = this.initialSyncBlockHeight;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import { Fr } from '@aztec/foundation/fields';
import { createDebugLogger } from '@aztec/foundation/log';
import { assertLength } from '@aztec/foundation/serialize';
import { ContractData, L2Block, L2BlockL2Logs, MerkleTreeId, PublicDataWrite, TxL2Logs } from '@aztec/types';
import { MerkleTreeOperations } from '@aztec/world-state';
import { MerkleTreeOperations, computeGlobalVariablesHash } from '@aztec/world-state';

import chunk from 'lodash.chunk';
import flatMap from 'lodash.flatmap';
Expand Down Expand Up @@ -295,7 +295,9 @@ export class SoloBlockBuilder implements BlockBuilder {
// Update the root trees with the latest data and contract tree roots,
// and validate them against the output of the root circuit simulation
this.debug(`Updating and validating root trees`);
await this.db.updateHistoricBlocksTree(left[0].constants.globalVariables);
const globalVariablesHash = await computeGlobalVariablesHash(left[0].constants.globalVariables);
await this.db.updateLatestGlobalVariablesHash(globalVariablesHash);
await this.db.updateHistoricBlocksTree(globalVariablesHash);

await this.validateRootOutput(rootOutput);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { GlobalVariables } from '@aztec/circuits.js';
import { Fr } from '@aztec/foundation/fields';
import { LowLeafWitnessData } from '@aztec/merkle-tree';
import { L2Block, MerkleTreeId, SiblingPath } from '@aztec/types';

Expand Down Expand Up @@ -116,11 +116,26 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations {
/**
* Inserts into the roots trees (CONTRACT_TREE_ROOTS_TREE, PRIVATE_DATA_TREE_ROOTS_TREE)
* the current roots of the corresponding trees (CONTRACT_TREE, PRIVATE_DATA_TREE).
* @param globalVariables - The current global variables to include in the block hash.
* @param globalVariablesHash - The hash of the current global variables to include in the block hash.
* @returns Empty promise.
*/
public updateHistoricBlocksTree(globalVariables: GlobalVariables): Promise<void> {
return this.trees.updateHistoricBlocksTree(globalVariables, this.includeUncommitted);
public updateHistoricBlocksTree(globalVariablesHash: Fr): Promise<void> {
return this.trees.updateHistoricBlocksTree(globalVariablesHash, this.includeUncommitted);
}

/**
* Updates the latest global variables hash
* @param globalVariablesHash - The latest global variables hash
*/
public updateLatestGlobalVariablesHash(globalVariablesHash: Fr): Promise<void> {
return this.trees.updateLatestGlobalVariablesHash(globalVariablesHash, this.includeUncommitted);
}

/**
* Gets the global variables hash from the previous block
*/
public getLatestGlobalVariablesHash(): Promise<Fr> {
return this.trees.getLatestGlobalVariablesHash(this.includeUncommitted);
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import { Fr } from '@aztec/foundation/fields';
import { createDebugLogger } from '@aztec/foundation/log';
import { L2Block, L2BlockDownloader, L2BlockSource } from '@aztec/types';

import { MerkleTreeDb, MerkleTreeOperations, computeGlobalVariablesHash } from '../index.js';
import { MerkleTreeDb, MerkleTreeOperations } from '../index.js';
import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js';
import { getConfigEnvVars } from './config.js';
import { WorldStateRunningState, WorldStateStatus, WorldStateSynchroniser } from './world_state_synchroniser.js';
Expand All @@ -23,9 +22,6 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
private runningPromise: Promise<void> = Promise.resolve();
private currentState: WorldStateRunningState = WorldStateRunningState.IDLE;

/** The latest Global Variables hash for the HEAD of the chain. */
public latestGlobalVariablesHash: Fr = Fr.ZERO;

constructor(
private merkleTreeDb: MerkleTreeDb,
private l2BlockSource: L2BlockSource,
Expand Down Expand Up @@ -57,7 +53,6 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {

// get the current latest block number
this.latestBlockNumberAtStart = await this.l2BlockSource.getBlockHeight();
this.latestGlobalVariablesHash = await computeGlobalVariablesHash();

const blockToDownloadFrom = this.currentL2BlockNum + 1;

Expand Down Expand Up @@ -121,8 +116,6 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser {
private async handleL2Block(l2Block: L2Block) {
await this.merkleTreeDb.handleL2Block(l2Block);
this.currentL2BlockNum = l2Block.number;
this.latestGlobalVariablesHash = await computeGlobalVariablesHash(l2Block.globalVariables);
this.log(`Synced global variables with hash ${this.latestGlobalVariablesHash.toString()}`);
if (
this.currentState === WorldStateRunningState.SYNCHING &&
this.currentL2BlockNum >= this.latestBlockNumberAtStart
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import { Fr } from '@aztec/circuits.js';

import { MerkleTreeOperations } from '../index.js';

/**
Expand Down Expand Up @@ -58,9 +56,4 @@ export interface WorldStateSynchroniser {
* @returns An instance of MerkleTreeOperations that will not include uncommitted data.
*/
getCommitted(): MerkleTreeOperations;

/**
* The latest Global Variables hash for the HEAD of the chain.
*/
latestGlobalVariablesHash: Fr;
}
18 changes: 15 additions & 3 deletions yarn-project/world-state/src/world-state-db/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { GlobalVariables, MAX_NEW_NULLIFIERS_PER_TX } from '@aztec/circuits.js';
import { MAX_NEW_NULLIFIERS_PER_TX } from '@aztec/circuits.js';
import { Fr } from '@aztec/foundation/fields';
import { createDebugLogger } from '@aztec/foundation/log';
import { LeafData, LowLeafWitnessData } from '@aztec/merkle-tree';
import { L2Block, MerkleTreeId, SiblingPath } from '@aztec/types';
Expand Down Expand Up @@ -175,9 +176,20 @@ export interface MerkleTreeOperations {
/**
* Inserts the new block hash into the new block hashes tree.
* This includes all of the current roots of all of the data trees and the current blocks global vars.
* @param globalVariables - The global variables to insert into the block hash.
* @param globalVariablesHash - The global variables hash to insert into the block hash.
*/
updateHistoricBlocksTree(globalVariables: GlobalVariables): Promise<void>;
updateHistoricBlocksTree(globalVariablesHash: Fr): Promise<void>;

/**
* Updates the latest global variables hash
* @param globalVariablesHash - The latest global variables hash
*/
updateLatestGlobalVariablesHash(globalVariablesHash: Fr): Promise<void>;

/**
* Gets the global variables hash from the previous block
*/
getLatestGlobalVariablesHash(): Promise<Fr>;

/**
* Batch insert multiple leaves into the tree.
Expand Down
43 changes: 35 additions & 8 deletions yarn-project/world-state/src/world-state-db/merkle_trees.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
PRIVATE_DATA_TREE_HEIGHT,
PUBLIC_DATA_TREE_HEIGHT,
} from '@aztec/circuits.js';
import { computeBlockHashWithGlobals } from '@aztec/circuits.js/abis';
import { computeBlockHash } from '@aztec/circuits.js/abis';
import { SerialQueue } from '@aztec/foundation/fifo';
import { createDebugLogger } from '@aztec/foundation/log';
import { IWasmModule } from '@aztec/foundation/wasm';
Expand All @@ -31,6 +31,7 @@ import { L2Block, MerkleTreeId, SiblingPath, merkleTreeIds } from '@aztec/types'
import { default as levelup } from 'levelup';

import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js';
import { computeGlobalVariablesHash } from '../utils.js';
import {
CurrentTreeRoots,
INITIAL_NULLIFIER_TREE_SIZE,
Expand All @@ -46,6 +47,7 @@ import {
*/
export class MerkleTrees implements MerkleTreeDb {
private trees: (AppendOnlyTree | UpdateOnlyTree)[] = [];
private latestGlobalVariablesHash: Fr = Fr.ZERO;
private jobQueue = new SerialQueue();

constructor(private db: levelup.LevelUp, private log = createDebugLogger('aztec:merkle_trees')) {}
Expand Down Expand Up @@ -105,7 +107,8 @@ export class MerkleTrees implements MerkleTreeDb {
this.jobQueue.start();

// The first leaf in the blocks tree contains the empty roots of the other trees and empty global variables.
await this.updateHistoricBlocksTree(GlobalVariables.empty(), true);
this.latestGlobalVariablesHash = await computeGlobalVariablesHash(GlobalVariables.empty());
await this.updateHistoricBlocksTree(this.latestGlobalVariablesHash, true);
await historicBlocksTree.commit();
}

Expand Down Expand Up @@ -147,14 +150,29 @@ export class MerkleTrees implements MerkleTreeDb {
/**
* Inserts into the roots trees (CONTRACT_TREE_ROOTS_TREE, PRIVATE_DATA_TREE_ROOTS_TREE, L1_TO_L2_MESSAGES_TREE_ROOTS_TREE)
* the current roots of the corresponding trees (CONTRACT_TREE, PRIVATE_DATA_TREE, L1_TO_L2_MESSAGES_TREE).
* @param globals - The global variables to use for hashing.
* @param globalsHash - The current global variables hash.
* @param includeUncommitted - Indicates whether to include uncommitted data.
*/
public async updateHistoricBlocksTree(globals: GlobalVariables, includeUncommitted: boolean) {
const blockHash = await this.getCurrentBlockHash(globals, includeUncommitted);
public async updateHistoricBlocksTree(globalsHash: Fr, includeUncommitted: boolean) {
const blockHash = await this.getCurrentBlockHash(globalsHash, includeUncommitted);
await this.appendLeaves(MerkleTreeId.BLOCKS_TREE, [blockHash.toBuffer()]);
}

/**
* Updates the latest global variables hash
* @param globalVariablesHash - The latest global variables hash
*/
public async updateLatestGlobalVariablesHash(globalVariablesHash: Fr) {
await this.synchronise(() => this._updateLatestGlobalVariablesHash(globalVariablesHash));
}

/**
* Gets the global variables hash from the previous block
*/
getLatestGlobalVariablesHash(): Promise<Fr> {
return Promise.resolve(this.latestGlobalVariablesHash);
}

/**
* Gets the tree info for the specified tree.
* @param treeId - Id of the tree to get information from.
Expand Down Expand Up @@ -183,10 +201,10 @@ export class MerkleTrees implements MerkleTreeDb {
};
}

async getCurrentBlockHash(globals: GlobalVariables, includeUncommitted: boolean): Promise<Fr> {
async getCurrentBlockHash(globalsHash: Fr, includeUncommitted: boolean): Promise<Fr> {
const roots = this.getAllTreeRoots(includeUncommitted).map(root => Fr.fromBuffer(root));
const wasm = await CircuitsWasm.get();
return computeBlockHashWithGlobals(wasm, globals, roots[0], roots[1], roots[2], roots[3], roots[4]);
return computeBlockHash(wasm, globalsHash, roots[0], roots[1], roots[2], roots[3], roots[4]);
}

getAllTreeRoots(includeUncommitted: boolean): Buffer[] {
Expand Down Expand Up @@ -377,6 +395,11 @@ export class MerkleTrees implements MerkleTreeDb {
return await this.jobQueue.put(fn);
}

private _updateLatestGlobalVariablesHash(globalVariablesHash: Fr): Promise<void> {
this.latestGlobalVariablesHash = globalVariablesHash;
return Promise.resolve();
}

/**
* Returns the tree info for the specified tree id.
* @param treeId - Id of the tree to get information from.
Expand Down Expand Up @@ -514,7 +537,11 @@ export class MerkleTrees implements MerkleTreeDb {
}

// Sync and add the block to the historic blocks tree
const blockHash = await this.getCurrentBlockHash(l2Block.globalVariables, true);
const globalVariablesHash = await computeGlobalVariablesHash(l2Block.globalVariables);
await this.updateLatestGlobalVariablesHash(globalVariablesHash);
this.log(`Synced global variables with hash ${this.latestGlobalVariablesHash.toString()}`);

const blockHash = await this.getCurrentBlockHash(globalVariablesHash, true);
await this._appendLeaves(MerkleTreeId.BLOCKS_TREE, [blockHash.toBuffer()]);

await this._commit();
Expand Down