feat(network): integrate initial sync from MODE

packages/network/jest.config.js

@@ -3,6 +3,7 @@ export default {
   preset: "ts-jest",
   roots: ["src"],
   testEnvironment: "jsdom",
+  setupFilesAfterEnv: ["./jest.setup.ts"],
   moduleNameMapper: {
     // fix TS build issues
     "^(..?/.*).js$": "$1",

packages/network/jest.setup.ts

@@ -0,0 +1,7 @@
+import { TextEncoder } from "util";
+
+// TextEncoder APIs are used by viem, but are not provided by
+// jsdom, all node versions supported provide these via the util module
+if (typeof globalThis.TextEncoder === "undefined") {
+  globalThis.TextEncoder = TextEncoder;
+}

packages/network/package.json

@@ -44,7 +44,8 @@
     "nice-grpc-web": "^2.0.0",
     "observable-webworker": "^4.0.1",
     "rxjs": "^7.5.5",
-    "threads": "^1.7.0"
+    "threads": "^1.7.0",
+    "viem": "^0.1.25"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.6.0",
@@ -77,7 +78,8 @@
     "tslib": "^2.3.1",
     "typedoc": "0.23.21",
     "typedoc-plugin-markdown": "^3.13.6",
-    "typescript": "^4.9.5"
+    "typescript": "^4.9.5",
+    "viem": "^0.1.25"
   },
   "gitHead": "914a1e0ae4a573d685841ca2ea921435057deb8f"
 }

packages/network/src/types.ts

@@ -135,6 +135,7 @@ export type SyncWorkerConfig = {
   worldContract: ContractConfig;
   disableCache?: boolean;
   chainId: number;
+  modeUrl?: string;
   snapshotServiceUrl?: string;
   streamServiceUrl?: string;
   fetchSystemCalls?: boolean;

packages/network/src/v2/common.ts

@@ -6,16 +6,16 @@ export const metadataTableId = new TableId("mudstore", "StoreMetadata");
 
 export const storeEvents = ["StoreSetRecord", "StoreSetField", "StoreDeleteRecord"] as const;
 
-export type TableSchema = {
+export type TableSchema = Readonly<{
   staticDataLength: number;
   staticFields: SchemaType[];
   dynamicFields: SchemaType[];
   rawSchema: string;
   abi: string;
   isEmpty: boolean;
-};
+}>;
 
-export type TableMetadata = {
+export type TableMetadata = Readonly<{
   tableName: string;
   fieldNames: string[];
-};
+}>;

packages/network/src/v2/fetchStoreEvents.ts

@@ -6,18 +6,18 @@ import { storeEvents } from "./common";
 import { ecsEventFromLog } from "./ecsEventFromLog";
 
 export async function fetchStoreEvents(
-  contract: Contract,
+  store: Contract,
   fromBlock: number,
   toBlock: number
 ): Promise<NetworkComponentUpdate[]> {
-  const topicSets = storeEvents.map((eventName) => contract.filters[eventName]().topics).filter(isDefined);
+  const topicSets = storeEvents.map((eventName) => store.filters[eventName]().topics).filter(isDefined);
 
   const logSets = await Promise.all(
-    topicSets.map((topics) => contract.provider.getLogs({ address: contract.address, topics, fromBlock, toBlock }))
+    topicSets.map((topics) => store.provider.getLogs({ address: store.address, topics, fromBlock, toBlock }))
   );
 
   const logs = orderBy(
-    logSets.flatMap((logs) => logs.map((log) => ({ log, parsedLog: contract.interface.parseLog(log) }))),
+    logSets.flatMap((logs) => logs.map((log) => ({ log, parsedLog: store.interface.parseLog(log) }))),
     ["log.blockNumber", "log.logIndex"]
   );
 
@@ -29,7 +29,7 @@ export async function fetchStoreEvents(
   const ecsEvents = await Promise.all(
     logs.map(({ log, parsedLog }) => {
       const { transactionHash, logIndex } = log;
-      return ecsEventFromLog(contract, log, parsedLog, lastLogForTx[transactionHash] === logIndex);
+      return ecsEventFromLog(store, log, parsedLog, lastLogForTx[transactionHash] === logIndex);
     })
   );
 

packages/network/src/v2/mode/createModeClient.ts

@@ -0,0 +1,11 @@
+import { QueryLayerClient, QueryLayerDefinition } from "@latticexyz/services/protobuf/ts/mode/mode";
+import { createChannel, createClient } from "nice-grpc-web";
+
+/**
+ * Create a MODE QueryLayerClient
+ * @param url MUDE URL
+ * @returns MODE QueryLayerClient
+ */
+export function createModeClient(url: string): QueryLayerClient {
+  return createClient(QueryLayerDefinition, createChannel(url));
+}

packages/network/src/v2/mode/getBlockNumberFromModeTable.ts

@@ -0,0 +1,8 @@
+import { GenericTable } from "@latticexyz/services/protobuf/ts/mode/mode";
+import { SchemaType } from "@latticexyz/schema-type";
+import { decodeValue } from "../schemas/decodeValue";
+
+export function getBlockNumberFromModeTable(table: GenericTable): number {
+  if (table.cols[0] !== "block_number") throw new Error("Table does not contain block_number column");
+  return Number(decodeValue(SchemaType.UINT256, table.rows[0].values[0]));
+}

packages/network/src/v2/mode/getModeBlockNumber.ts

@@ -0,0 +1,13 @@
+import { QueryLayerClient } from "@latticexyz/services/protobuf/ts/mode/mode";
+import { getBlockNumberFromModeTable } from "./getBlockNumberFromModeTable";
+
+export async function getModeBlockNumber(client: QueryLayerClient, chainId: number): Promise<number> {
+  const response = await client.single__GetState({
+    table: "block_number",
+    namespace: {
+      chainId: chainId.toString(),
+    },
+  });
+  const blockNumber = getBlockNumberFromModeTable(response.chainTables["block_number"]);
+  return blockNumber;
+}

packages/network/src/v2/mode/syncTablesFromMode.ts

@@ -0,0 +1,84 @@
+import { ComponentValue } from "@latticexyz/recs";
+import { AbiTypeToSchemaType, encodeSchema } from "@latticexyz/schema-type";
+import { QueryLayerClient } from "@latticexyz/services/protobuf/ts/mode/mode";
+import { arrayToHex, TableId } from "@latticexyz/utils";
+import { Contract } from "ethers";
+import { NetworkEvents } from "../../types";
+
+import { CacheStore, createCacheStore, storeEvent } from "../../workers";
+import { keyTupleToEntityID } from "../keyTupleToEntityID";
+import { registerMetadata } from "../schemas/tableMetadata";
+import { registerSchema } from "../schemas/tableSchemas";
+import { getBlockNumberFromModeTable } from "./getBlockNumberFromModeTable";
+import { decodeAbiParameters } from "viem";
+
+export async function syncTablesFromMode(
+  client: QueryLayerClient,
+  chainId: number,
+  world: Contract,
+  setPercentage?: (progress: number) => void
+): Promise<CacheStore> {
+  const cacheStore = createCacheStore();
+
+  const response = await client.getState({
+    chainTables: [],
+    worldTables: [],
+    namespace: {
+      chainId: chainId.toString(),
+      worldAddress: world.address,
+    },
+  });
+  console.log("syncTablesFromMode", response);
+
+  const numRowsTotal = Object.values(response.worldTables).reduce((sum, table) => sum + table.rows.length, 0);
+  let numRowsProcessed = 0;
+
+  const blockNumber = getBlockNumberFromModeTable(response.chainTables["block_number"]);
+  const registrationPromises: Promise<unknown>[] = [];
+
+  for (const [fullTableName, { rows, cols, types }] of Object.entries(response.worldTables)) {
+    const [tableNamespace, tableName] = fullTableName.split("__");
+    const tableId = new TableId(tableNamespace, tableName);
+
+    const component = tableId.toString();
+
+    // TODO: separate keys and values/fields in MODE, but we'll infer for now
+    const keyLength = cols.findIndex((col) => !col.startsWith("key_"));
+    const keyAbiTypes = types.slice(0, keyLength);
+    const fieldNames = cols.slice(keyLength);
+    // TODO: remove this hack once MODE is fixed (https://github.com/latticexyz/mud/issues/444)
+    const fieldAbiTypes = types.slice(keyLength).map((modeType) => modeType.match(/tuple\((.*)\[]\)/)?.[1] ?? modeType);
+    const fieldSchemaTypes = fieldAbiTypes.map((abiType) => AbiTypeToSchemaType[abiType]);
+
+    const rawSchema = arrayToHex(encodeSchema(fieldSchemaTypes));
+    // TODO: refactor registerSchema/registerMetadata to take chain+world address rather than Contract
+    registrationPromises.push(registerSchema(world, tableId, rawSchema));
+    registrationPromises.push(registerMetadata(world, tableId, { tableName, fieldNames }));
+
+    for (const row of rows) {
+      console.log(tableName, keyAbiTypes, fieldAbiTypes, row.values);
+      const keyTuple = row.values
+        .slice(0, keyLength)
+        .map((bytes, i) => decodeAbiParameters([{ type: keyAbiTypes[i] }], arrayToHex(bytes)));
+      const values = row.values
+        .slice(keyLength)
+        .map((bytes, i) => decodeAbiParameters([{ type: fieldAbiTypes[i] }], arrayToHex(bytes)));
+
+      const entity = keyTupleToEntityID(keyTuple);
+      const value = Object.fromEntries(values.map((value, i) => [fieldNames[i], value])) as ComponentValue;
+
+      storeEvent(cacheStore, { type: NetworkEvents.NetworkComponentUpdate, component, entity, value, blockNumber });
+
+      numRowsProcessed++;
+      // Update progress every 100 rows
+      if (numRowsProcessed % 100 === 0 && setPercentage) {
+        setPercentage(Math.floor(numRowsProcessed / numRowsTotal));
+      }
+    }
+  }
+
+  // make sure all schemas/metadata are registered before returning to avoid downstream lookup issues
+  await Promise.all(registrationPromises);
+
+  return cacheStore;
+}

packages/network/src/v2/schemas/decodeValue.ts

@@ -1,4 +1,4 @@
-import { SchemaType } from "@latticexyz/schema-type";
+import { SchemaType, SchemaTypeToPrimitive } from "@latticexyz/schema-type";
 import { decodeDynamicField } from "./decodeDynamicField";
 import { decodeStaticField } from "./decodeStaticField";
 
@@ -103,8 +103,8 @@ export function decodeValue<T extends SchemaType>(schemaType: T, bytes: Uint8Arr
     case SchemaType.BYTES31:
     case SchemaType.BYTES32:
     case SchemaType.ADDRESS:
-      return decodeStaticField(schemaType, bytes, 0);
+      return decodeStaticField(schemaType, bytes, 0) as SchemaTypeToPrimitive[T];
     default:
-      return decodeDynamicField(schemaType, bytes);
+      return decodeDynamicField(schemaType, bytes) as SchemaTypeToPrimitive[T];
   }
 }

packages/network/src/v2/schemas/tableMetadata.ts

@@ -1,4 +1,4 @@
-import { arrayToHex, TableId } from "@latticexyz/utils";
+import { TableId } from "@latticexyz/utils";
 import { Contract, utils } from "ethers";
 import { metadataTableId, schemaTableId, TableMetadata } from "../common";
 import { decodeData } from "./decodeData";
@@ -68,6 +68,13 @@ export function registerMetadata(
     }
     const decoded = decodeData(metadataSchema, metadataRecord);
     const tableName = decoded[0];
+    if (tableName !== table.name) {
+      console.warn("Metadata table name does not match table ID", {
+        tableName,
+        tableId: table.toString(),
+        world: world.address,
+      });
+    }
     const [fieldNames] = utils.defaultAbiCoder.decode(["string[]"], decoded[1]);
     return { tableName, fieldNames };
   });

packages/network/src/workers/SyncWorker.ts

@@ -51,6 +51,9 @@ import { debug as parentDebug } from "./debug";
 import { fetchStoreEvents } from "../v2/fetchStoreEvents";
 import { abi as IStoreAbi } from "@latticexyz/store/abi/IStore.json";
 import { Contract } from "ethers";
+import { createModeClient } from "../v2/mode/createModeClient";
+import { syncTablesFromMode } from "../v2/mode/syncTablesFromMode";
+import { getModeBlockNumber } from "../v2/mode/getModeBlockNumber";
 
 const debug = parentDebug.extend("SyncWorker");
 
@@ -127,6 +130,7 @@ export class SyncWorker<C extends Components> implements DoWork<Input, NetworkEv
     );
     const config = computedConfig.get();
     const {
+      modeUrl,
       snapshotServiceUrl,
       streamServiceUrl,
       chainId,
@@ -145,6 +149,7 @@ export class SyncWorker<C extends Components> implements DoWork<Input, NetworkEv
     const { providers } = await createReconnectingProvider(computed(() => computedConfig.get().provider));
     const provider = providers.get().json;
     const snapshotClient = snapshotServiceUrl ? createSnapshotClient(snapshotServiceUrl) : undefined;
+    const modeClient = modeUrl ? createModeClient(modeUrl) : undefined;
     const indexDbCache = await getIndexDbECSCache(chainId, worldContract.address);
     const decode = createDecode(worldContract, provider);
     const fetchWorldEvents = createFetchWorldEventsInBlockRange(
@@ -214,6 +219,8 @@ export class SyncWorker<C extends Components> implements DoWork<Input, NetworkEv
     const cacheBlockNumber = !disableCache ? await getIndexDBCacheStoreBlockNumber(indexDbCache) : -1;
     this.setLoadingState({ percentage: 50 });
     const snapshotBlockNumber = await getSnapshotBlockNumber(snapshotClient, worldContract.address);
+    const modeBlockNumber = modeClient ? await getModeBlockNumber(modeClient, chainId) : -1;
+
     this.setLoadingState({ percentage: 100 });
     debug(
       `cache block: ${cacheBlockNumber}, snapshot block: ${
@@ -222,13 +229,23 @@ export class SyncWorker<C extends Components> implements DoWork<Input, NetworkEv
     );
 
     let initialState = createCacheStore();
-    if (initialBlockNumber > Math.max(cacheBlockNumber, snapshotBlockNumber)) {
+    if (initialBlockNumber > Math.max(cacheBlockNumber, snapshotBlockNumber, modeBlockNumber)) {
+      // Skip initializing from cache/snapshot/mode if the initial block number is newer than all of them
       initialState.blockNumber = initialBlockNumber;
     } else {
-      // Load from cache if the snapshot is less than <cacheAgeThreshold> blocks newer than the cache
-      const syncFromSnapshot = snapshotClient && snapshotBlockNumber > cacheBlockNumber + cacheAgeThreshold;
-
-      if (syncFromSnapshot) {
+      // Load from cache if the mode/snapshot is less than <cacheAgeThreshold> blocks newer than the cache
+      const syncFromMode = modeClient && modeBlockNumber > cacheBlockNumber + cacheAgeThreshold;
+      const syncFromSnapshot =
+        !syncFromMode && snapshotClient && snapshotBlockNumber > cacheBlockNumber + cacheAgeThreshold;
+
+      if (syncFromMode) {
+        console.log("Initial sync from MODE");
+        this.setLoadingState({ state: SyncState.INITIAL, msg: "Fetching initial state from MODE", percentage: 0 });
+        initialState = await syncTablesFromMode(modeClient, chainId, storeContract, (percentage: number) =>
+          this.setLoadingState({ percentage })
+        );
+        this.setLoadingState({ percentage: 100 });
+      } else if (syncFromSnapshot) {
         this.setLoadingState({ state: SyncState.INITIAL, msg: "Fetching initial state from snapshot", percentage: 0 });
         initialState = await fetchSnapshotChunked(
           snapshotClient,

packages/recs/src/Component.ts

@@ -86,13 +86,15 @@ export function setComponent<S extends Schema, T = undefined>(
         // Otherwise, we should let the user know we found undefined data.
         console.warn(
           "Component definition for",
-          component.metadata?.contractId ?? component.id,
+          component.metadata?.tableId ?? component.metadata?.contractId ?? component.id,
           "is missing key",
           key,
           ", ignoring value",
           val,
           "for entity",
-          entity
+          entity,
+          ". Existing keys: ",
+          Object.keys(component.values)
         );
       }
     }

packages/std-client/src/mud-definitions/contractComponents.test-d.ts

@@ -12,7 +12,7 @@ describe("store/contractComponents", () => {
         y: RecsType.Number;
       },
       {
-        contractId: string;
+        contractId: `0x${string}`;
         tableId: string;
       },
       undefined

packages/std-client/src/setup/setupMUDV2Network.ts

@@ -64,7 +64,7 @@ export async function setupMUDV2Network<C extends ContractComponents>({
   const storeSchemaTableId = new TableId("mudstore", "schema");
   const storeSchemaComponent = defineComponent(
     world,
-    { schema: Type.T },
+    { valueSchema: Type.String, keySchema: Type.String },
     {
       metadata: {
         contractId: storeSchemaTableId.toHexString(),

packages/utils/src/v2/arrayToHex.ts

@@ -1,3 +1,3 @@
 // TODO: migrate to viem's toHex()
-export const arrayToHex = (array: Uint8Array | ArrayBuffer): string =>
+export const arrayToHex = (array: Uint8Array | ArrayBuffer): `0x${string}` =>
   `0x${[...new Uint8Array(array)].map((x) => x.toString(16).padStart(2, "0")).join("")}`;