Merge branch 'master' into revert-9846-lx/ipa-accumulator-builder

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/callbacks.cpp

@@ -4,10 +4,17 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
-#include <endian.h>
 #include <functional>
 #include <vector>
 
+#ifdef __APPLE__
+#include <libkern/OSByteOrder.h>
+#define htole64(x) OSSwapHostToLittleInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
+#else
+#include <sys/types.h>
+#endif
+
 namespace bb::crypto::merkle_tree {
 void throw_error(const std::string& errorString, int error)
 {

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp

@@ -102,6 +102,86 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_meta_data)
     }
 }
 
+TEST_F(LMDBTreeStoreTest, can_write_and_read_multiple_blocks_with_meta)
+{
+    LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders);
+    uint64_t start_block = 647810461952355;
+    uint64_t num_blocks = 1000;
+    for (size_t i = 0; i < num_blocks; i++) {
+        BlockPayload blockData;
+        blockData.blockNumber = i + start_block;
+        blockData.root = VALUES[i];
+        blockData.size = 45 + (i * 97);
+        LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction();
+        store.write_block_data(i + start_block, blockData, *transaction);
+
+        TreeMeta meta;
+        meta.committedSize = blockData.size;
+        meta.size = blockData.size;
+        meta.root = blockData.root;
+        meta.depth = 32;
+        meta.unfinalisedBlockHeight = i + start_block;
+        meta.name = "NullifierTree";
+        store.write_meta_data(meta, *transaction);
+        transaction->commit();
+    }
+
+    BlockPayload blockData;
+    for (size_t i = 0; i < num_blocks; i++) {
+        LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction();
+        BlockPayload readBack;
+        bool success = store.read_block_data(i + start_block, readBack, *transaction);
+        EXPECT_TRUE(success);
+
+        blockData.blockNumber = i + start_block;
+        blockData.root = VALUES[i];
+        blockData.size = 45 + (i * 97);
+        EXPECT_EQ(readBack, blockData);
+    }
+
+    {
+        TreeMeta meta;
+        LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction();
+        store.read_meta_data(meta, *transaction);
+
+        EXPECT_EQ(meta.committedSize, blockData.size);
+        EXPECT_EQ(meta.size, blockData.size);
+        EXPECT_EQ(meta.root, blockData.root);
+        EXPECT_EQ(meta.depth, 32);
+        EXPECT_EQ(meta.unfinalisedBlockHeight, blockData.blockNumber);
+        EXPECT_EQ(meta.name, "NullifierTree");
+    }
+}
+
+uint64_t serde_value(uint64_t value)
+{
+    std::vector<uint8_t> data = serialise_key(value);
+    uint64_t return_value = 0;
+    deserialise_key(data.data(), return_value);
+    return return_value;
+}
+
+TEST_F(LMDBTreeStoreTest, can_serde_64bit_values)
+{
+    union mapped {
+        uint64_t u64;
+        uint8_t arr[8];
+    };
+    mapped value1;
+    mapped value2;
+    value1.arr[0] = value2.arr[7] = 0x11;
+    value1.arr[1] = value2.arr[6] = 0x22;
+    value1.arr[2] = value2.arr[5] = 0x33;
+    value1.arr[3] = value2.arr[4] = 0x44;
+    value1.arr[4] = value2.arr[3] = 0x55;
+    value1.arr[5] = value2.arr[2] = 0x66;
+    value1.arr[6] = value2.arr[1] = 0x77;
+    value1.arr[7] = value2.arr[0] = 0x88;
+
+    EXPECT_EQ(value1.u64, serde_value(value1.u64));
+    EXPECT_EQ(value2.u64, serde_value(value2.u64));
+}
+
 TEST_F(LMDBTreeStoreTest, can_write_and_read_leaf_indices)
 {
     Indices indices;

yarn-project/world-state/src/native/native_world_state.test.ts

@@ -23,7 +23,8 @@ import { join } from 'path';
 import { assertSameState, compareChains, mockBlock } from '../test/utils.js';
 import { INITIAL_NULLIFIER_TREE_SIZE, INITIAL_PUBLIC_DATA_TREE_SIZE } from '../world-state-db/merkle_tree_db.js';
 import { type WorldStateStatusSummary } from './message.js';
-import { NativeWorldStateService } from './native_world_state.js';
+import { NativeWorldStateService, WORLD_STATE_VERSION_FILE } from './native_world_state.js';
+import { WorldStateVersion } from './world_state_version.js';
 
 describe('NativeWorldState', () => {
   let dataDir: string;
@@ -58,6 +59,8 @@ describe('NativeWorldState', () => {
       await expect(
         ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]),
       ).resolves.toBeDefined();
+      const status = await ws.getStatusSummary();
+      expect(status.unfinalisedBlockNumber).toBe(1n);
       await ws.close();
     });
 
@@ -77,6 +80,41 @@ describe('NativeWorldState', () => {
       await expect(
         ws.getCommitted().findLeafIndex(MerkleTreeId.NOTE_HASH_TREE, block.body.txEffects[0].noteHashes[0]),
       ).resolves.toBeUndefined();
+      const status = await ws.getStatusSummary();
+      expect(status.unfinalisedBlockNumber).toBe(0n);
+      await ws.close();
+    });
+
+    it('clears the database if the world state version is different', async () => {
+      // open ws against the data again
+      let ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize);
+      // db should be empty
+      let emptyStatus = await ws.getStatusSummary();
+      expect(emptyStatus.unfinalisedBlockNumber).toBe(0n);
+
+      // populate it and then close it
+      const fork = await ws.fork();
+      ({ block, messages } = await mockBlock(1, 2, fork));
+      await fork.close();
+
+      const status = await ws.handleL2BlockAndMessages(block, messages);
+      expect(status.summary.unfinalisedBlockNumber).toBe(1n);
+      await ws.close();
+      // we open up the version file that was created and modify the version to be older
+      const fullPath = join(dataDir, 'world_state', WORLD_STATE_VERSION_FILE);
+      const storedWorldStateVersion = await WorldStateVersion.readVersion(fullPath);
+      expect(storedWorldStateVersion).toBeDefined();
+      const modifiedVersion = new WorldStateVersion(
+        storedWorldStateVersion!.version - 1,
+        storedWorldStateVersion!.rollupAddress,
+      );
+      await modifiedVersion.writeVersionFile(fullPath);
+
+      // Open the world state again and it should be empty
+      ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize);
+      // db should be empty
+      emptyStatus = await ws.getStatusSummary();
+      expect(emptyStatus.unfinalisedBlockNumber).toBe(0n);
       await ws.close();
     });
 

yarn-project/world-state/src/native/native_world_state.ts

@@ -24,7 +24,7 @@ import { padArrayEnd } from '@aztec/foundation/collection';
 import { createDebugLogger } from '@aztec/foundation/log';
 
 import assert from 'assert/strict';
-import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises';
+import { mkdir, mkdtemp, rm } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
@@ -40,8 +40,16 @@ import {
   worldStateRevision,
 } from './message.js';
 import { NativeWorldState } from './native_world_state_instance.js';
+import { WorldStateVersion } from './world_state_version.js';
 
-const ROLLUP_ADDRESS_FILE = 'rollup_address';
+export const WORLD_STATE_VERSION_FILE = 'version';
+
+// A crude way of maintaining DB versioning
+// We don't currently have any method of performing data migrations
+// should the world state db structure change
+// For now we will track versions using this hardcoded value and delete
+// the state if a change is detected
+export const WORLD_STATE_DB_VERSION = 1; // The initial version
 
 export class NativeWorldStateService implements MerkleTreeDatabase {
   protected initialHeader: Header | undefined;
@@ -60,17 +68,24 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
     cleanup = () => Promise.resolve(),
   ): Promise<NativeWorldStateService> {
     const worldStateDirectory = join(dataDir, 'world_state');
-    const rollupAddressFile = join(worldStateDirectory, ROLLUP_ADDRESS_FILE);
-    const currentRollupStr = await readFile(rollupAddressFile, 'utf8').catch(() => undefined);
-    const currentRollupAddress = currentRollupStr ? EthAddress.fromString(currentRollupStr.trim()) : undefined;
+    const versionFile = join(worldStateDirectory, WORLD_STATE_VERSION_FILE);
+    const storedWorldStateVersion = await WorldStateVersion.readVersion(versionFile);
 
-    if (currentRollupAddress && !rollupAddress.equals(currentRollupAddress)) {
-      log.warn('Rollup address changed, deleting database');
+    if (!storedWorldStateVersion) {
+      log.warn('No world state version found, deleting world state directory');
+      await rm(worldStateDirectory, { recursive: true, force: true });
+    } else if (!rollupAddress.equals(storedWorldStateVersion.rollupAddress)) {
+      log.warn('Rollup address changed, deleting world state directory');
+      await rm(worldStateDirectory, { recursive: true, force: true });
+    } else if (storedWorldStateVersion.version != WORLD_STATE_DB_VERSION) {
+      log.warn('World state version change detected, deleting world state directory');
       await rm(worldStateDirectory, { recursive: true, force: true });
     }
 
+    const newWorldStateVersion = new WorldStateVersion(WORLD_STATE_DB_VERSION, rollupAddress);
+
     await mkdir(worldStateDirectory, { recursive: true });
-    await writeFile(rollupAddressFile, rollupAddress.toString(), 'utf8');
+    await newWorldStateVersion.writeVersionFile(versionFile);
 
     const instance = new NativeWorldState(worldStateDirectory, dbMapSizeKb);
     const worldState = new this(instance, log, cleanup);

yarn-project/world-state/src/native/world_state_version.ts

@@ -0,0 +1,41 @@
+import { EthAddress } from '@aztec/circuits.js';
+
+import { readFile, writeFile } from 'fs/promises';
+
+export class WorldStateVersion {
+  constructor(readonly version: number, readonly rollupAddress: EthAddress) {}
+
+  static async readVersion(filename: string) {
+    const versionData = await readFile(filename, 'utf-8').catch(() => undefined);
+    if (versionData === undefined) {
+      return undefined;
+    }
+    const versionJSON = JSON.parse(versionData);
+    if (versionJSON.version === undefined || versionJSON.rollupAddress === undefined) {
+      return undefined;
+    }
+    return WorldStateVersion.fromJSON(versionJSON);
+  }
+
+  public async writeVersionFile(filename: string) {
+    const data = JSON.stringify(this.toJSON());
+    await writeFile(filename, data, 'utf-8');
+  }
+
+  toJSON() {
+    return {
+      version: this.version,
+      rollupAddress: this.rollupAddress.toChecksumString(),
+    };
+  }
+
+  static fromJSON(obj: any): WorldStateVersion {
+    const version = obj.version;
+    const rollupAddress = EthAddress.fromString(obj.rollupAddress);
+    return new WorldStateVersion(version, rollupAddress);
+  }
+
+  static empty() {
+    return new WorldStateVersion(0, EthAddress.ZERO);
+  }
+}