test(store-sync): add query benchmark, refactor test data scripts (#2228

)

.gitignore

@@ -17,6 +17,5 @@ templates/*/pnpm-lock.yaml
 
 .env
 
-test-data/world-logs-10.json
-test-data/world-logs-100.json
-test-data/world-logs-1000.json
+test-data/world-logs-bulk-*.json
+test-data/world-logs-query.json

e2e/packages/contracts/src/systems/VectorSystem.sol

@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: MIT
+pragma solidity >=0.8.24;
+
+import { System } from "@latticexyz/world/src/System.sol";
+import { Vector } from "../codegen/index.sol";
+
+contract VectorSystem is System {
+  function setVector(uint32 key, int32 x, int32 y) public {
+    Vector.set(key, x, y);
+  }
+}

e2e/packages/test-data/generate-test-data-records.ts → e2e/packages/test-data/generate-bench-data-bulk.ts

@@ -22,11 +22,19 @@ for (let i = 0; i < NUM_RECORDS.length; i++) {
   const rpc = `http://127.0.0.1:8545/${numRecords}`;
 
   console.log(`generating logs for ${numRecords} records`);
-  const logs = await generateLogs(numRecords, rpc);
+  const logs = await generateLogs(rpc, async (worldContract) => {
+    console.log("calling setNumber");
+    for (let i = 0; i < numRecords - 1; i++) {
+      await worldContract.write.setNumber([i, i]);
+    }
+
+    const lastTx = await worldContract.write.setNumber([numRecords - 1, numRecords - 1]);
+    return lastTx;
+  });
 
   const logsFilename = path.join(
     path.dirname(fileURLToPath(import.meta.url)),
-    `../../../test-data/world-logs-${numRecords}.json`
+    `../../../test-data/world-logs-bulk-${numRecords}.json`
   );
 
   console.log("writing", logs.length, "logs to", logsFilename);

e2e/packages/test-data/generate-bench-data-query.ts

@@ -0,0 +1,42 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { createAnvil } from "@viem/anvil";
+import { generateLogs } from "./generateLogs";
+
+const anvil = createAnvil({
+  blockTime: 1,
+  blockBaseFeePerGas: 0,
+  gasLimit: 20_000_000,
+});
+
+console.log("starting anvil");
+await anvil.start();
+const rpc = `http://${anvil.host}:${anvil.port}`;
+
+const logs = await generateLogs(rpc, async (worldContract) => {
+  for (let i = 0; i < 100; i++) {
+    await worldContract.write.setNumber([i, i]);
+  }
+
+  for (let i = 0; i < 50; i++) {
+    await worldContract.write.setVector([i, i, i]);
+  }
+
+  const lastTx = await worldContract.write.set([[0]]);
+
+  return lastTx;
+});
+
+const logsFilename = path.join(
+  path.dirname(fileURLToPath(import.meta.url)),
+  `../../../test-data/world-logs-query.json`
+);
+
+console.log("writing", logs.length, "logs to", logsFilename);
+await fs.writeFile(logsFilename, JSON.stringify(logs, null, 2));
+
+// TODO: figure out why anvil doesn't stop immediately
+// console.log("stopping anvil");
+// await anvil.stop();
+process.exit(0);

e2e/packages/test-data/generate-test-data.ts

@@ -2,23 +2,7 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { createAnvil } from "@viem/anvil";
-import { execa } from "execa";
-import {
-  ClientConfig,
-  createPublicClient,
-  createWalletClient,
-  encodeEventTopics,
-  http,
-  isHex,
-  numberToHex,
-} from "viem";
-import { mudFoundry } from "@latticexyz/common/chains";
-import { getContract } from "@latticexyz/common";
-import { storeEventsAbi } from "@latticexyz/store";
-import { privateKeyToAccount } from "viem/accounts";
-import IWorldAbi from "../contracts/out/IWorld.sol/IWorld.abi.json";
-
-const logsFilename = path.join(path.dirname(fileURLToPath(import.meta.url)), `../../../test-data/world-logs.json`);
+import { generateLogs } from "./generateLogs";
 
 const anvil = createAnvil({
   blockTime: 1,
@@ -30,72 +14,16 @@ console.log("starting anvil");
 await anvil.start();
 const rpc = `http://${anvil.host}:${anvil.port}`;
 
-console.log("deploying world");
-const { stdout, stderr } = await execa("pnpm", ["mud", "deploy", "--rpc", rpc, "--saveDeployment", "false"], {
-  cwd: "../contracts",
-  stdio: "pipe",
-  env: {
-    DEBUG: "mud:*",
-  },
-});
-if (stderr) console.error(stderr);
-if (stdout) console.log(stdout);
-
-const [, worldAddress] = stdout.match(/worldAddress: '(0x[0-9a-f]+)'/i) ?? [];
-if (!isHex(worldAddress)) {
-  throw new Error("world address not found in output, did the deploy fail?");
-}
-console.log("got world address", worldAddress);
-
-const clientOptions = {
-  chain: mudFoundry,
-  transport: http(rpc),
-  pollingInterval: 1000,
-} as const satisfies ClientConfig;
-
-const publicClient = createPublicClient(clientOptions);
+const logs = await generateLogs(rpc, async (worldContract) => {
+  console.log("calling set");
+  await worldContract.write.set([[420]]);
+  console.log("calling push");
+  const lastTx = await worldContract.write.push([69]);
 
-// anvil default private key
-const account = privateKeyToAccount("0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80");
-const walletClient = createWalletClient({
-  ...clientOptions,
-  account,
+  return lastTx;
 });
 
-const worldContract = getContract({
-  address: worldAddress,
-  abi: IWorldAbi,
-  publicClient,
-  walletClient,
-});
-
-console.log("calling set");
-await worldContract.write.set([[420]]);
-console.log("calling push");
-const lastTx = await worldContract.write.push([69]);
-
-console.log("waiting for tx");
-const receipt = await publicClient.waitForTransactionReceipt({ hash: lastTx });
-
-console.log("fetching logs", receipt.blockNumber);
-const logs = await publicClient.request({
-  method: "eth_getLogs",
-  params: [
-    {
-      address: worldAddress,
-      topics: [
-        storeEventsAbi.flatMap((event) =>
-          encodeEventTopics({
-            abi: [event],
-            eventName: event.name,
-          })
-        ),
-      ],
-      fromBlock: numberToHex(0n),
-      toBlock: numberToHex(receipt.blockNumber),
-    },
-  ],
-});
+const logsFilename = path.join(path.dirname(fileURLToPath(import.meta.url)), `../../../test-data/world-logs.json`);
 
 console.log("writing", logs.length, "logs to", logsFilename);
 await fs.writeFile(logsFilename, JSON.stringify(logs, null, 2));

e2e/packages/test-data/generateLogs.ts

@@ -1,20 +1,38 @@
 import { execa } from "execa";
 import {
   ClientConfig,
+  GetContractReturnType,
+  Hex,
+  PublicClient,
+  RpcLog,
+  WalletClient,
   createPublicClient,
   createWalletClient,
   encodeEventTopics,
-  getContract,
   http,
   isHex,
   numberToHex,
+  getContract,
+  Transport,
+  Chain,
 } from "viem";
 import { mudFoundry } from "@latticexyz/common/chains";
 import { storeEventsAbi } from "@latticexyz/store";
-import { privateKeyToAccount } from "viem/accounts";
+import { Account, privateKeyToAccount } from "viem/accounts";
 import IWorldAbi from "../contracts/out/IWorld.sol/IWorld.abi.json";
 
-export async function generateLogs(numRecords: number, rpc: string) {
+type WorldAbi = typeof IWorldAbi;
+
+type WorldContract = GetContractReturnType<
+  WorldAbi,
+  PublicClient<Transport, Chain>,
+  WalletClient<Transport, Chain, Account>
+>;
+
+export async function generateLogs(
+  rpc: string,
+  transactionHook: (worldContract: WorldContract) => Promise<Hex>
+): Promise<RpcLog[]> {
   console.log("deploying world");
   const { stdout, stderr } = await execa("pnpm", ["mud", "deploy", "--rpc", rpc, "--saveDeployment", "false"], {
     cwd: "../contracts",
@@ -55,12 +73,7 @@ export async function generateLogs(numRecords: number, rpc: string) {
     walletClient,
   });
 
-  console.log("calling setNumber");
-  for (let i = 0; i < numRecords - 1; i++) {
-    await worldContract.write.setNumber([i, i]);
-  }
-
-  const lastTx = await worldContract.write.setNumber([numRecords, numRecords]);
+  const lastTx = await transactionHook(worldContract);
 
   console.log("waiting for tx");
   const receipt = await publicClient.waitForTransactionReceipt({ hash: lastTx });

packages/store-sync/benchmarks/getRecord.bench.ts

@@ -5,9 +5,9 @@ import { encodeEntity } from "../src/recs";
 import { logsToBlocks } from "../test/logsToBlocks";
 import { buildTable, getTables } from "../src/sqlite";
 import { eq } from "drizzle-orm";
-import worldRpcLogs10 from "../../../test-data/world-logs-10.json";
-import worldRpcLogs100 from "../../../test-data/world-logs-100.json";
-import worldRpcLogs1000 from "../../../test-data/world-logs-1000.json";
+import worldRpcLogs10 from "../../../test-data/world-logs-bulk-10.json";
+import worldRpcLogs100 from "../../../test-data/world-logs-bulk-100.json";
+import worldRpcLogs1000 from "../../../test-data/world-logs-bulk-1000.json";
 
 describe.each([
   { numRecords: 10, logs: worldRpcLogs10 },

packages/store-sync/benchmarks/getRecords.bench.ts

@@ -3,9 +3,9 @@ import { getComponentEntities, getComponentValue } from "@latticexyz/recs";
 import { buildTable, getTables } from "../src/sqlite";
 import { logsToBlocks } from "../test/logsToBlocks";
 import { createRecsStorage, createSqliteStorage, createZustandStorage, tables } from "../test/utils";
-import worldRpcLogs10 from "../../../test-data/world-logs-10.json";
-import worldRpcLogs100 from "../../../test-data/world-logs-100.json";
-import worldRpcLogs1000 from "../../../test-data/world-logs-1000.json";
+import worldRpcLogs10 from "../../../test-data/world-logs-bulk-10.json";
+import worldRpcLogs100 from "../../../test-data/world-logs-bulk-100.json";
+import worldRpcLogs1000 from "../../../test-data/world-logs-bulk-1000.json";
 
 describe.each([
   { numRecords: 10, logs: worldRpcLogs10 },

packages/store-sync/benchmarks/query.bench.ts

@@ -0,0 +1,36 @@
+import { bench, describe } from "vitest";
+import { Has, runQuery } from "@latticexyz/recs";
+import { createRecsStorage, createSqliteStorage, createZustandStorage, tables } from "../test/utils";
+import { logsToBlocks } from "../test/logsToBlocks";
+import worldRpcLogs from "../../../test-data/world-logs-query.json";
+import { buildTable, getTables } from "../src/sqlite";
+import { eq } from "drizzle-orm";
+
+const blocks = logsToBlocks(worldRpcLogs);
+
+const { components, storageAdapter: recsStorageAdapter } = createRecsStorage();
+const { useStore, storageAdapter: zustandStorageAdapter } = createZustandStorage();
+const { database, storageAdapter: sqliteStorageAdapter } = await createSqliteStorage();
+
+for (const block of blocks) {
+  await Promise.all([recsStorageAdapter(block), zustandStorageAdapter(block), sqliteStorageAdapter(block)]);
+}
+
+describe("Get records with query", async () => {
+  bench("recs: `runQuery`", async () => {
+    runQuery([Has(components.Number), Has(components.Vector)]);
+  });
+
+  bench("zustand: `getRecords`", async () => {
+    const records = useStore.getState().getRecords(tables.Number);
+
+    Object.keys(useStore.getState().getRecords(tables.Vector)).filter((id) => id in records);
+  });
+
+  bench("sqlite: `innerJoin`", async () => {
+    const numberTable = buildTable(getTables(database).filter((table) => table.name === "Number")[0]);
+    const vectorTable = buildTable(getTables(database).filter((table) => table.name === "Vector")[0]);
+
+    await database.select().from(numberTable).innerJoin(vectorTable, eq(numberTable.key, vectorTable.key));
+  });
+});

packages/store-sync/benchmarks/storageAdapter.bench.ts

@@ -1,9 +1,9 @@
 import { bench, describe } from "vitest";
 import { createRecsStorage, createSqliteStorage, createZustandStorage } from "../test/utils";
 import { logsToBlocks } from "../test/logsToBlocks";
-import worldRpcLogs10 from "../../../test-data/world-logs-10.json";
-import worldRpcLogs100 from "../../../test-data/world-logs-100.json";
-import worldRpcLogs1000 from "../../../test-data/world-logs-1000.json";
+import worldRpcLogs10 from "../../../test-data/world-logs-bulk-10.json";
+import worldRpcLogs100 from "../../../test-data/world-logs-bulk-100.json";
+import worldRpcLogs1000 from "../../../test-data/world-logs-bulk-1000.json";
 
 describe.each([
   { numRecords: 10, logs: worldRpcLogs10 },