From 8dc588918c488f98603cbb7e183c88129942debe Mon Sep 17 00:00:00 2001 From: alvarius Date: Thu, 19 Sep 2024 13:39:25 +0100 Subject: [PATCH] feat(store-sync): add syncToStash util (#3192) Co-authored-by: Kevin Ingersoll --- .changeset/lucky-cows-fail.md | 23 +++++ packages/stash/package.json | 3 +- packages/stash/src/createStash.test.ts | 4 +- packages/stash/src/createStash.ts | 4 +- packages/store-sync/package.json | 1 + packages/store-sync/src/common.ts | 7 ++ packages/store-sync/src/exports/internal.ts | 1 + .../src/stash/createStorageAdapter.test.ts | 92 +++++++++++++++++++ .../src/stash/createStorageAdapter.ts | 75 +++++++++++++++ packages/store-sync/src/stash/index.ts | 2 + packages/store-sync/src/stash/syncToStash.ts | 76 +++++++++++++++ pnpm-lock.yaml | 3 + tsconfig.paths.json | 2 + 13 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 .changeset/lucky-cows-fail.md create mode 100644 packages/store-sync/src/stash/createStorageAdapter.test.ts create mode 100644 packages/store-sync/src/stash/createStorageAdapter.ts create mode 100644 packages/store-sync/src/stash/index.ts create mode 100644 packages/store-sync/src/stash/syncToStash.ts diff --git a/.changeset/lucky-cows-fail.md b/.changeset/lucky-cows-fail.md new file mode 100644 index 0000000000..98aa41d8b4 --- /dev/null +++ b/.changeset/lucky-cows-fail.md @@ -0,0 +1,23 @@ +--- +"@latticexyz/store-sync": patch +--- + +Added a `syncToStash` util to hydrate a `stash` client store from MUD contract state. This is currently exported from `@latticexyz/store-sync/internal` while Stash package is unstable/experimental. + +```ts +import { createClient, http } from "viem"; +import { anvil } from "viem/chains"; +import { createStash } from "@latticexyz/stash/internal"; +import { syncToStash } from "@latticexyz/store-sync/internal"; +import config from "../mud.config"; + +const client = createClient({ + chain: anvil, + transport: http(), +}); + +const address = "0x..."; + +const stash = createStash(config); +const sync = await syncToStash({ stash, client, address }); +``` diff --git a/packages/stash/package.json b/packages/stash/package.json index 056653dab2..20805bd286 100644 --- a/packages/stash/package.json +++ b/packages/stash/package.json @@ -11,8 +11,7 @@ "type": "module", "exports": { ".": "./dist/index.js", - "./internal": "./dist/internal.js", - "./recs": "./dist/recs.js" + "./internal": "./dist/internal.js" }, "typesVersions": { "*": { diff --git a/packages/stash/src/createStash.test.ts b/packages/stash/src/createStash.test.ts index b6c13544ae..be52ede999 100644 --- a/packages/stash/src/createStash.test.ts +++ b/packages/stash/src/createStash.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { attest } from "@arktype/attest"; -import { CreateStoreResult, createStash } from "./createStash"; +import { CreateStashResult, createStash } from "./createStash"; import { defineStore, defineTable } from "@latticexyz/store/config/v2"; import { Hex } from "viem"; @@ -60,7 +60,7 @@ describe("createStash", () => { value: { field1: "hello" }, }); - attest>(stash); + attest>(stash); attest<{ config: { namespace1: { diff --git a/packages/stash/src/createStash.ts b/packages/stash/src/createStash.ts index 722c85a2ed..3a6939ba7d 100644 --- a/packages/stash/src/createStash.ts +++ b/packages/stash/src/createStash.ts @@ -5,12 +5,12 @@ import { Table } from "@latticexyz/store/config/v2"; export type Config = StoreConfig; -export type CreateStoreResult = Stash & DefaultActions; +export type CreateStashResult = Stash & DefaultActions; /** * Initializes a Stash based on the provided store config. */ -export function createStash(storeConfig?: config): CreateStoreResult { +export function createStash(storeConfig?: config): CreateStashResult { const tableSubscribers: TableSubscribers = {}; const storeSubscribers: StoreSubscribers = new Set(); diff --git a/packages/store-sync/package.json b/packages/store-sync/package.json index a58497c0a4..bf589cce43 100644 --- a/packages/store-sync/package.json +++ b/packages/store-sync/package.json @@ -73,6 +73,7 @@ "@latticexyz/protocol-parser": "workspace:*", "@latticexyz/recs": "workspace:*", "@latticexyz/schema-type": "workspace:*", + "@latticexyz/stash": "workspace:*", "@latticexyz/store": "workspace:*", "@latticexyz/world": "workspace:*", "@trpc/client": "10.34.0", diff --git a/packages/store-sync/src/common.ts b/packages/store-sync/src/common.ts index 721a4491b1..44ad08ff9a 100644 --- a/packages/store-sync/src/common.ts +++ b/packages/store-sync/src/common.ts @@ -3,6 +3,7 @@ import { StoreEventsAbiItem, StoreEventsAbi } from "@latticexyz/store"; import { Observable } from "rxjs"; import { UnionPick } from "@latticexyz/common/type-utils"; import { + ValueArgs, getKeySchema, getSchemaPrimitives, getSchemaTypes, @@ -140,3 +141,9 @@ export const schemasTable = { keySchema: getSchemaTypes(getKeySchema(mudTables.Tables)), valueSchema: getSchemaTypes(getValueSchema(mudTables.Tables)), }; + +export const emptyValueArgs = { + staticData: "0x", + encodedLengths: "0x", + dynamicData: "0x", +} as const satisfies ValueArgs; diff --git a/packages/store-sync/src/exports/internal.ts b/packages/store-sync/src/exports/internal.ts index be3db7892b..65f0049838 100644 --- a/packages/store-sync/src/exports/internal.ts +++ b/packages/store-sync/src/exports/internal.ts @@ -1 +1,2 @@ export * from "../sql"; +export * from "../stash"; diff --git a/packages/store-sync/src/stash/createStorageAdapter.test.ts b/packages/store-sync/src/stash/createStorageAdapter.test.ts new file mode 100644 index 0000000000..08b15bac7c --- /dev/null +++ b/packages/store-sync/src/stash/createStorageAdapter.test.ts @@ -0,0 +1,92 @@ +import { beforeAll, describe, expect, it } from "vitest"; +import { storeEventsAbi } from "@latticexyz/store"; +import { createStorageAdapter } from "./createStorageAdapter"; +import { config, deployMockGame } from "../../test/mockGame"; +import { fetchAndStoreLogs } from "../fetchAndStoreLogs"; +import { testClient } from "../../test/common"; +import { getBlockNumber } from "viem/actions"; +import { createStash } from "@latticexyz/stash/internal"; + +describe("createStorageAdapter", async () => { + beforeAll(async () => { + await deployMockGame(); + }); + + it("sets component values from logs", async () => { + const stash = createStash(config); + const storageAdapter = createStorageAdapter({ stash }); + + console.log("fetching blocks"); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const block of fetchAndStoreLogs({ + storageAdapter, + publicClient: testClient, + events: storeEventsAbi, + fromBlock: 0n, + toBlock: await getBlockNumber(testClient), + })) { + // + } + + expect(stash.get().records).toMatchInlineSnapshot(` + { + "": { + "Health": { + "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb": { + "health": 0n, + "player": "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb", + }, + "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e": { + "health": 5n, + "player": "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e", + }, + "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6": { + "health": 5n, + "player": "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6", + }, + }, + "Inventory": {}, + "Position": { + "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb": { + "player": "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb", + "x": 3, + "y": 5, + }, + "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e": { + "player": "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e", + "x": 1, + "y": -1, + }, + "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6": { + "player": "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6", + "x": 3, + "y": 5, + }, + "0xdBa86119a787422C593ceF119E40887f396024E2": { + "player": "0xdBa86119a787422C593ceF119E40887f396024E2", + "x": 100, + "y": 100, + }, + }, + "Score": {}, + "Terrain": { + "3|5": { + "terrainType": 2, + "x": 3, + "y": 5, + }, + }, + "Winner": {}, + }, + } + `); + + expect(stash.getRecord({ table: config.tables.Terrain, key: { x: 3, y: 5 } })).toMatchInlineSnapshot(` + { + "terrainType": 2, + "x": 3, + "y": 5, + } + `); + }); +}); diff --git a/packages/store-sync/src/stash/createStorageAdapter.ts b/packages/store-sync/src/stash/createStorageAdapter.ts new file mode 100644 index 0000000000..a998ebf519 --- /dev/null +++ b/packages/store-sync/src/stash/createStorageAdapter.ts @@ -0,0 +1,75 @@ +import { Stash, deleteRecord, getRecord, setRecord } from "@latticexyz/stash/internal"; +import { + decodeKey, + decodeValueArgs, + encodeValueArgs, + getKeySchema, + getSchemaTypes, + getValueSchema, +} from "@latticexyz/protocol-parser/internal"; +import { spliceHex } from "@latticexyz/common"; +import { size } from "viem"; +import { Table } from "@latticexyz/config"; +import { StorageAdapter, StorageAdapterBlock, emptyValueArgs } from "../common"; + +export type CreateStorageAdapter = { + stash: Stash; +}; + +export function createStorageAdapter({ stash }: CreateStorageAdapter): StorageAdapter { + const tablesById = Object.fromEntries( + Object.values(stash.get().config) + .flatMap((namespace) => Object.values(namespace) as readonly Table[]) + .map((table) => [table.tableId, table]), + ); + + return async function storageAdapter({ logs }: StorageAdapterBlock): Promise { + for (const log of logs) { + const table = tablesById[log.args.tableId]; + if (!table) continue; + + const valueSchema = getSchemaTypes(getValueSchema(table)); + const keySchema = getSchemaTypes(getKeySchema(table)); + const key = decodeKey(keySchema, log.args.keyTuple); + + if (log.eventName === "Store_SetRecord") { + const value = decodeValueArgs(valueSchema, log.args); + setRecord({ stash, table, key, value }); + } else if (log.eventName === "Store_SpliceStaticData") { + const previousValue = getRecord({ stash, table, key }); + + const { + staticData: previousStaticData, + encodedLengths, + dynamicData, + } = previousValue ? encodeValueArgs(valueSchema, previousValue) : emptyValueArgs; + + const staticData = spliceHex(previousStaticData, log.args.start, size(log.args.data), log.args.data); + const value = decodeValueArgs(valueSchema, { + staticData, + encodedLengths, + dynamicData, + }); + + setRecord({ stash, table, key, value }); + } else if (log.eventName === "Store_SpliceDynamicData") { + const previousValue = getRecord({ stash, table, key }); + + const { staticData, dynamicData: previousDynamicData } = previousValue + ? encodeValueArgs(valueSchema, previousValue) + : emptyValueArgs; + + const dynamicData = spliceHex(previousDynamicData, log.args.start, log.args.deleteCount, log.args.data); + const value = decodeValueArgs(valueSchema, { + staticData, + encodedLengths: log.args.encodedLengths, + dynamicData, + }); + + setRecord({ stash, table, key, value }); + } else if (log.eventName === "Store_DeleteRecord") { + deleteRecord({ stash, table, key }); + } + } + }; +} diff --git a/packages/store-sync/src/stash/index.ts b/packages/store-sync/src/stash/index.ts new file mode 100644 index 0000000000..acfaf6b1ad --- /dev/null +++ b/packages/store-sync/src/stash/index.ts @@ -0,0 +1,2 @@ +export * from "./createStorageAdapter"; +export * from "./syncToStash"; diff --git a/packages/store-sync/src/stash/syncToStash.ts b/packages/store-sync/src/stash/syncToStash.ts new file mode 100644 index 0000000000..40ca66fed6 --- /dev/null +++ b/packages/store-sync/src/stash/syncToStash.ts @@ -0,0 +1,76 @@ +import { getRecord, setRecord, registerTable, Stash } from "@latticexyz/stash/internal"; +import { Address, Client, publicActions } from "viem"; +import { createStorageAdapter } from "./createStorageAdapter"; +import { defineTable } from "@latticexyz/store/config/v2"; +import { SyncStep } from "../SyncStep"; +import { SyncResult } from "../common"; +import { createStoreSync } from "../createStoreSync"; +import { getSchemaPrimitives, getValueSchema } from "@latticexyz/protocol-parser/internal"; + +export const SyncProgress = defineTable({ + namespaceLabel: "syncToStash", + label: "SyncProgress", + schema: { + step: "string", + percentage: "uint32", + latestBlockNumber: "uint256", + lastBlockNumberProcessed: "uint256", + message: "string", + }, + key: [], +}); + +export const initialProgress = { + step: SyncStep.INITIALIZE, + percentage: 0, + latestBlockNumber: 0n, + lastBlockNumberProcessed: 0n, + message: "Connecting", +} satisfies getSchemaPrimitives>; + +export type SyncToStashOptions = { + stash: Stash; + client: Client; + address: Address; + startSync?: boolean; +}; + +export type SyncToStashResult = Omit & { + waitForStateChange: SyncResult["waitForTransaction"]; + stopSync: () => void; +}; + +export async function syncToStash({ + stash, + client, + address, + startSync = true, +}: SyncToStashOptions): Promise { + registerTable({ stash, table: SyncProgress }); + + const storageAdapter = createStorageAdapter({ stash }); + + const { waitForTransaction: waitForStateChange, ...sync } = await createStoreSync({ + storageAdapter, + publicClient: client.extend(publicActions) as never, + address, + onProgress: (nextValue) => { + const currentValue = getRecord({ stash, table: SyncProgress, key: {} }); + // update sync progress until we're caught up and live + if (currentValue?.step !== SyncStep.LIVE) { + setRecord({ stash, table: SyncProgress, key: {}, value: nextValue }); + } + }, + }); + + const sub = startSync ? sync.storedBlockLogs$.subscribe() : null; + function stopSync(): void { + sub?.unsubscribe(); + } + + return { + ...sync, + waitForStateChange, + stopSync, + }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bd610b4419..bd3b14924a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1100,6 +1100,9 @@ importers: '@latticexyz/schema-type': specifier: workspace:* version: link:../schema-type + '@latticexyz/stash': + specifier: workspace:* + version: link:../stash '@latticexyz/store': specifier: workspace:* version: link:../store diff --git a/tsconfig.paths.json b/tsconfig.paths.json index f2d16c9e8d..a1c1be72d9 100644 --- a/tsconfig.paths.json +++ b/tsconfig.paths.json @@ -25,6 +25,8 @@ "@latticexyz/recs/deprecated": ["./packages/recs/src/deprecated/index.ts"], "@latticexyz/schema-type": ["./packages/schema-type/src/typescript/index.ts"], "@latticexyz/schema-type/*": ["./packages/schema-type/src/typescript/exports/*.ts"], + "@latticexyz/stash": ["./packages/stash/ts/exports/index.ts"], + "@latticexyz/stash/internal": ["./packages/stash/ts/exports/internal.ts"], "@latticexyz/store": ["./packages/store/ts/exports/index.ts"], "@latticexyz/store/mud.config": ["./packages/store/mud.config.ts"], "@latticexyz/store/codegen": ["./packages/store/ts/codegen/index.ts"],