feat(store-sync): add syncToStash util

packages/store-sync/package.json

@@ -73,6 +73,7 @@
     "@latticexyz/protocol-parser": "workspace:*",
     "@latticexyz/recs": "workspace:*",
     "@latticexyz/schema-type": "workspace:*",
+    "@latticexyz/stash": "workspace:*",
     "@latticexyz/store": "workspace:*",
     "@latticexyz/world": "workspace:*",
     "@trpc/client": "10.34.0",

packages/store-sync/src/common.ts

@@ -3,6 +3,7 @@ import { StoreEventsAbiItem, StoreEventsAbi } from "@latticexyz/store";
 import { Observable } from "rxjs";
 import { UnionPick } from "@latticexyz/common/type-utils";
 import {
+  ValueArgs,
   getKeySchema,
   getSchemaPrimitives,
   getSchemaTypes,
@@ -140,3 +141,9 @@ export const schemasTable = {
   keySchema: getSchemaTypes(getKeySchema(mudTables.Tables)),
   valueSchema: getSchemaTypes(getValueSchema(mudTables.Tables)),
 };
+
+export const emptyValueArgs = {
+  staticData: "0x",
+  encodedLengths: "0x",
+  dynamicData: "0x",
+} as const satisfies ValueArgs;

packages/store-sync/src/exports/internal.ts

@@ -1 +1,2 @@
 export * from "../sql";
+export * from "../stash";

packages/store-sync/src/stash/createStorageAdapter.test.ts

@@ -0,0 +1,92 @@
+import { beforeAll, describe, expect, it } from "vitest";
+import { storeEventsAbi } from "@latticexyz/store";
+import { createStorageAdapter } from "./createStorageAdapter";
+import { config, deployMockGame } from "../../test/mockGame";
+import { fetchAndStoreLogs } from "../fetchAndStoreLogs";
+import { testClient } from "../../test/common";
+import { getBlockNumber } from "viem/actions";
+import { createStash } from "@latticexyz/stash/internal";
+
+describe("createStorageAdapter", async () => {
+  beforeAll(async () => {
+    await deployMockGame();
+  });
+
+  it("sets component values from logs", async () => {
+    const stash = createStash(config);
+    const storageAdapter = createStorageAdapter({ stash });
+
+    console.log("fetching blocks");
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    for await (const block of fetchAndStoreLogs({
+      storageAdapter,
+      publicClient: testClient,
+      events: storeEventsAbi,
+      fromBlock: 0n,
+      toBlock: await getBlockNumber(testClient),
+    })) {
+      //
+    }
+
+    expect(stash.get().records).toMatchInlineSnapshot(`
+      {
+        "": {
+          "Health": {
+            "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb": {
+              "health": 0n,
+              "player": "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb",
+            },
+            "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e": {
+              "health": 5n,
+              "player": "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e",
+            },
+            "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6": {
+              "health": 5n,
+              "player": "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6",
+            },
+          },
+          "Inventory": {},
+          "Position": {
+            "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb": {
+              "player": "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb",
+              "x": 3,
+              "y": 5,
+            },
+            "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e": {
+              "player": "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e",
+              "x": 1,
+              "y": -1,
+            },
+            "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6": {
+              "player": "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6",
+              "x": 3,
+              "y": 5,
+            },
+            "0xdBa86119a787422C593ceF119E40887f396024E2": {
+              "player": "0xdBa86119a787422C593ceF119E40887f396024E2",
+              "x": 100,
+              "y": 100,
+            },
+          },
+          "Score": {},
+          "Terrain": {
+            "3|5": {
+              "terrainType": 2,
+              "x": 3,
+              "y": 5,
+            },
+          },
+          "Winner": {},
+        },
+      }
+    `);
+
+    expect(stash.getRecord({ table: config.tables.Terrain, key: { x: 3, y: 5 } })).toMatchInlineSnapshot(`
+      {
+        "terrainType": 2,
+        "x": 3,
+        "y": 5,
+      }
+    `);
+  });
+});

packages/store-sync/src/stash/createStorageAdapter.ts

@@ -0,0 +1,77 @@
+import { CreateStoreResult, getTable, StoreConfig } from "@latticexyz/stash/internal";
+import {
+  decodeKey,
+  decodeValueArgs,
+  encodeValueArgs,
+  getKeySchema,
+  getSchemaTypes,
+  getValueSchema,
+  KeySchema,
+} from "@latticexyz/protocol-parser/internal";
+import { spliceHex } from "@latticexyz/common";
+import { size } from "viem";
+import { Table } from "@latticexyz/config";
+import { StorageAdapter, StorageAdapterBlock, emptyValueArgs } from "../common";
+
+export type CreateStorageAdapter<config extends StoreConfig> = {
+  stash: CreateStoreResult<config>;
+};
+
+export function createStorageAdapter<const config extends StoreConfig>({
+  stash,
+}: CreateStorageAdapter<config>): StorageAdapter {
+  const tables = Object.values(stash.get().config).flatMap((namespace) => Object.values(namespace)) as readonly Table[];
+
+  return async function storageAdapter({ logs }: StorageAdapterBlock): Promise<void> {
+    for (const log of logs) {
+      const tableConfig = tables.find((t) => t.tableId === log.args.tableId);
+      if (!tableConfig) continue;
+
+      // TODO: this should probably return `| undefined`?
+      const table = getTable({ stash, table: tableConfig });
+
+      const valueSchema = getSchemaTypes(getValueSchema(tableConfig));
+      const keySchema = getSchemaTypes(getKeySchema(tableConfig)) as KeySchema;
+      const key = decodeKey(keySchema, log.args.keyTuple);
+
+      if (log.eventName === "Store_SetRecord") {
+        const value = decodeValueArgs(valueSchema, log.args);
+        table.setRecord({ key, value });
+      } else if (log.eventName === "Store_SpliceStaticData") {
+        const previousValue = table.getRecord({ key });
+
+        const {
+          staticData: previousStaticData,
+          encodedLengths,
+          dynamicData,
+        } = previousValue ? encodeValueArgs(valueSchema, previousValue) : emptyValueArgs;
+
+        const staticData = spliceHex(previousStaticData, log.args.start, size(log.args.data), log.args.data);
+        const value = decodeValueArgs(valueSchema, {
+          staticData,
+          encodedLengths,
+          dynamicData,
+        });
+
+        table.setRecord({ key, value });
+      } else if (log.eventName === "Store_SpliceDynamicData") {
+        const previousValue = table.getRecord({ key });
+
+        const { staticData, dynamicData: previousDynamicData } = previousValue
+          ? encodeValueArgs(valueSchema, previousValue)
+          : emptyValueArgs;
+
+        const dynamicData = spliceHex(previousDynamicData, log.args.start, log.args.deleteCount, log.args.data);
+        const value = decodeValueArgs(valueSchema, {
+          staticData,
+          encodedLengths: log.args.encodedLengths,
+          dynamicData,
+        });
+
+        table.setRecord({ key, value });
+      } else if (log.eventName === "Store_DeleteRecord") {
+        table.deleteRecord({ key });
+      }
+    }
+  };
+}

packages/store-sync/src/stash/index.ts

@@ -0,0 +1,2 @@
+export * from "./createStorageAdapter";
+export * from "./syncToStash";

packages/store-sync/src/stash/syncToStash.ts

@@ -0,0 +1,76 @@
+import { CreateStoreResult, StoreConfig } from "@latticexyz/stash/internal";
+import { Address, Client, publicActions } from "viem";
+import { createStorageAdapter } from "./createStorageAdapter";
+import { defineTable } from "@latticexyz/store/config/v2";
+import { SyncStep } from "../SyncStep";
+import { SyncResult } from "../common";
+import { createStoreSync } from "../createStoreSync";
+import { getSchemaPrimitives, getValueSchema } from "@latticexyz/protocol-parser/internal";
+
+export const SyncProgress = defineTable({
+  namespaceLabel: "syncToStash",
+  label: "SyncProgress",
+  schema: {
+    step: "string",
+    percentage: "uint32",
+    latestBlockNumber: "uint256",
+    lastBlockNumberProcessed: "uint256",
+    message: "string",
+  },
+  key: [],
+});
+
+export const initialProgress = {
+  step: SyncStep.INITIALIZE,
+  percentage: 0,
+  latestBlockNumber: 0n,
+  lastBlockNumberProcessed: 0n,
+  message: "Connecting",
+} satisfies getSchemaPrimitives<getValueSchema<typeof SyncProgress>>;
+
+export type SyncToStashOptions<config extends StoreConfig> = {
+  stash: CreateStoreResult<config>;
+  client: Client;
+  address: Address;
+  startSync?: boolean;
+};
+
+export type SyncToStashResult = Omit<SyncResult, "waitForTransaction"> & {
+  waitForStateChange: SyncResult["waitForTransaction"];
+  stopSync: () => void;
+};
+
+export async function syncToStash<const config extends StoreConfig>({
+  stash,
+  client,
+  address,
+  startSync = true,
+}: SyncToStashOptions<config>): Promise<SyncToStashResult> {
+  stash.registerTable({ table: SyncProgress });
+
+  const storageAdapter = createStorageAdapter({ stash });
+
+  const { waitForTransaction: waitForStateChange, ...sync } = await createStoreSync({
+    storageAdapter,
+    publicClient: client.extend(publicActions) as never,
+    address,
+    onProgress: (nextValue) => {
+      const currentValue = stash.getRecord({ table: SyncProgress, key: {} });
+      // update sync progress until we're caught up and live
+      if (currentValue?.step !== SyncStep.LIVE) {
+        stash.setRecord({ table: SyncProgress, key: {}, value: nextValue });
+      }
+    },
+  });
+
+  const sub = startSync ? sync.storedBlockLogs$.subscribe() : null;
+  function stopSync(): void {
+    sub?.unsubscribe();
+  }
+
+  return {
+    ...sync,
+    waitForStateChange,
+    stopSync,
+  };
+}