From 9e5baf4fff0c60615b8f2b4645fb11cb78cb0bd8 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Mon, 31 Jul 2023 05:06:59 -0700 Subject: [PATCH] feat(store-sync): sync to RECS (#1197) --- .changeset/great-cooks-dream.md | 26 ++ packages/store-sync/package.json | 9 +- packages/store-sync/src/blockLogsToStorage.ts | 12 +- packages/store-sync/src/common.ts | 32 +-- packages/store-sync/src/recs/common.ts | 13 + packages/store-sync/src/recs/debug.ts | 3 + packages/store-sync/src/recs/decodeEntity.ts | 23 ++ .../src/recs/defineInternalComponents.ts | 25 ++ packages/store-sync/src/recs/encodeEntity.ts | 19 ++ .../src/recs/entityToHexKeyTuple.ts | 13 + packages/store-sync/src/recs/getTableKey.ts | 8 + .../src/recs/hexKeyTupleToEntity.ts | 6 + packages/store-sync/src/recs/index.ts | 7 + packages/store-sync/src/recs/recsStorage.ts | 96 ++++++++ packages/store-sync/src/recs/syncToRecs.ts | 228 ++++++++++++++++++ packages/store-sync/src/schemaToDefaults.ts | 2 +- packages/store-sync/tsup.config.ts | 2 +- packages/store/ts/common.ts | 48 ++++ packages/store/ts/index.ts | 1 + pnpm-lock.yaml | 9 +- 20 files changed, 550 insertions(+), 32 deletions(-) create mode 100644 .changeset/great-cooks-dream.md create mode 100644 packages/store-sync/src/recs/common.ts create mode 100644 packages/store-sync/src/recs/debug.ts create mode 100644 packages/store-sync/src/recs/decodeEntity.ts create mode 100644 packages/store-sync/src/recs/defineInternalComponents.ts create mode 100644 packages/store-sync/src/recs/encodeEntity.ts create mode 100644 packages/store-sync/src/recs/entityToHexKeyTuple.ts create mode 100644 packages/store-sync/src/recs/getTableKey.ts create mode 100644 packages/store-sync/src/recs/hexKeyTupleToEntity.ts create mode 100644 packages/store-sync/src/recs/index.ts create mode 100644 packages/store-sync/src/recs/recsStorage.ts create mode 100644 packages/store-sync/src/recs/syncToRecs.ts create mode 100644 packages/store/ts/common.ts diff --git a/.changeset/great-cooks-dream.md b/.changeset/great-cooks-dream.md new file mode 100644 index 0000000000..60ae359c8e --- /dev/null +++ b/.changeset/great-cooks-dream.md @@ -0,0 +1,26 @@ +--- +"@latticexyz/store-sync": patch +--- + +Add RECS sync strategy and corresponding utils + +```ts +import { createPublicClient, http } from 'viem'; +import { syncToRecs } from '@latticexyz/store-sync'; +import storeConfig from 'contracts/mud.config'; +import { defineContractComponents } from './defineContractComponents'; + +const publicClient = createPublicClient({ + chain, + transport: http(), + pollingInterval: 1000, +}); + +const { components, singletonEntity, latestBlock$, blockStorageOperations$, waitForTransaction } = await syncToRecs({ + world, + config: storeConfig, + address: '0x...', + publicClient, + components: defineContractComponents(...), +}); +``` diff --git a/packages/store-sync/package.json b/packages/store-sync/package.json index fb6b4b3633..402f1db30f 100644 --- a/packages/store-sync/package.json +++ b/packages/store-sync/package.json @@ -11,7 +11,8 @@ "type": "module", "exports": { ".": "./dist/index.js", - "./sqlite": "./dist/sqlite/index.js" + "./sqlite": "./dist/sqlite/index.js", + "./recs": "./dist/recs/index.js" }, "typesVersions": { "*": { @@ -20,6 +21,9 @@ ], "sqlite": [ "./src/sqlite/index.ts" + ], + "recs": [ + "./src/recs/index.ts" ] } }, @@ -36,13 +40,14 @@ "@latticexyz/block-logs-stream": "workspace:*", "@latticexyz/common": "workspace:*", "@latticexyz/protocol-parser": "workspace:*", + "@latticexyz/recs": "workspace:*", "@latticexyz/schema-type": "workspace:*", "@latticexyz/store": "workspace:*", - "@latticexyz/store-cache": "workspace:*", "better-sqlite3": "^8.4.0", "debug": "^4.3.4", "drizzle-orm": "^0.27.0", "kysely": "^0.26.1", + "rxjs": "7.5.5", "sql.js": "^1.8.0", "superjson": "^1.12.4", "viem": "1.3.1" diff --git a/packages/store-sync/src/blockLogsToStorage.ts b/packages/store-sync/src/blockLogsToStorage.ts index b675b520b8..480efb8b46 100644 --- a/packages/store-sync/src/blockLogsToStorage.ts +++ b/packages/store-sync/src/blockLogsToStorage.ts @@ -6,12 +6,10 @@ import { abiTypesToSchema, TableSchema, } from "@latticexyz/protocol-parser"; -import { StoreConfig } from "@latticexyz/store"; +import { StoreConfig, ConfigToKeyPrimitives as Key, ConfigToValuePrimitives as Value } from "@latticexyz/store"; import { TableId } from "@latticexyz/common"; import { Address, Hex, decodeAbiParameters, getAddress, parseAbiParameters } from "viem"; import { debug } from "./debug"; -// TODO: move these type helpers into store? -import { Key, Value } from "@latticexyz/store-cache"; import { isDefined } from "@latticexyz/common/utils"; import { BlockLogs, StorageOperation, Table, TableName, TableNamespace } from "./common"; @@ -33,10 +31,14 @@ export type BlockLogsToStorageOptions }) => Promise; }; -export type BlockLogsToStorageResult = (block: BlockLogs) => Promise<{ +export type BlockStorageOperations = { blockNumber: BlockLogs["blockNumber"]; operations: StorageOperation[]; -}>; +}; + +export type BlockLogsToStorageResult = ( + block: BlockLogs +) => Promise>; type TableKey = `${Address}:${TableNamespace}:${TableName}`; diff --git a/packages/store-sync/src/common.ts b/packages/store-sync/src/common.ts index 5cf174d1bb..41f00420d7 100644 --- a/packages/store-sync/src/common.ts +++ b/packages/store-sync/src/common.ts @@ -1,9 +1,13 @@ -import { SchemaAbiType, SchemaAbiTypeToPrimitiveType, StaticAbiType } from "@latticexyz/schema-type"; import { Address, Hex } from "viem"; -// TODO: move these type helpers into store? -import { Key, Value } from "@latticexyz/store-cache"; -import { GetLogsResult, GroupLogsByBlockNumberResult } from "@latticexyz/block-logs-stream"; -import { StoreEventsAbi, StoreConfig } from "@latticexyz/store"; +import { GetLogsResult, GroupLogsByBlockNumberResult, NonPendingLog } from "@latticexyz/block-logs-stream"; +import { + StoreEventsAbi, + StoreConfig, + KeySchema, + ValueSchema, + ConfigToKeyPrimitives as Key, + ConfigToValuePrimitives as Value, +} from "@latticexyz/store"; export type ChainId = number; export type WorldId = `${ChainId}:${Address}`; @@ -11,18 +15,6 @@ export type WorldId = `${ChainId}:${Address}`; export type TableNamespace = string; export type TableName = string; -export type KeySchema = Record; -export type ValueSchema = Record; - -export type SchemaToPrimitives = { - [key in keyof TSchema]: SchemaAbiTypeToPrimitiveType; -}; - -export type TableRecord = { - key: SchemaToPrimitives; - value: SchemaToPrimitives; -}; - export type Table = { address: Address; tableId: Hex; @@ -36,9 +28,9 @@ export type StoreEventsLog = GetLogsResult[number]; export type BlockLogs = GroupLogsByBlockNumberResult[number]; export type BaseStorageOperation = { - log: StoreEventsLog; - namespace: string; - name: string; + log: NonPendingLog; + namespace: TableNamespace; + name: TableName; }; export type SetRecordOperation = BaseStorageOperation & { diff --git a/packages/store-sync/src/recs/common.ts b/packages/store-sync/src/recs/common.ts new file mode 100644 index 0000000000..77327bc997 --- /dev/null +++ b/packages/store-sync/src/recs/common.ts @@ -0,0 +1,13 @@ +import { KeySchema, ValueSchema } from "@latticexyz/store"; + +export type StoreComponentMetadata = { + keySchema: KeySchema; + valueSchema: ValueSchema; +}; + +export enum SyncStep { + INITIALIZE = "initialize", + SNAPSHOT = "snapshot", + RPC = "rpc", + LIVE = "live", +} diff --git a/packages/store-sync/src/recs/debug.ts b/packages/store-sync/src/recs/debug.ts new file mode 100644 index 0000000000..0b392b2333 --- /dev/null +++ b/packages/store-sync/src/recs/debug.ts @@ -0,0 +1,3 @@ +import { debug as parentDebug } from "../debug"; + +export const debug = parentDebug.extend("recs"); diff --git a/packages/store-sync/src/recs/decodeEntity.ts b/packages/store-sync/src/recs/decodeEntity.ts new file mode 100644 index 0000000000..cadfad5df3 --- /dev/null +++ b/packages/store-sync/src/recs/decodeEntity.ts @@ -0,0 +1,23 @@ +import { Entity } from "@latticexyz/recs"; +import { StaticAbiType } from "@latticexyz/schema-type"; +import { Hex, decodeAbiParameters } from "viem"; +import { SchemaToPrimitives } from "@latticexyz/store"; +import { entityToHexKeyTuple } from "./entityToHexKeyTuple"; + +export function decodeEntity>( + keySchema: TKeySchema, + entity: Entity +): SchemaToPrimitives { + const hexKeyTuple = entityToHexKeyTuple(entity); + if (hexKeyTuple.length !== Object.keys(keySchema).length) { + throw new Error( + `entity key tuple length ${hexKeyTuple.length} does not match key schema length ${Object.keys(keySchema).length}` + ); + } + return Object.fromEntries( + Object.entries(keySchema).map(([key, type], index) => [ + key, + decodeAbiParameters([{ type }], hexKeyTuple[index] as Hex)[0], + ]) + ) as SchemaToPrimitives; +} diff --git a/packages/store-sync/src/recs/defineInternalComponents.ts b/packages/store-sync/src/recs/defineInternalComponents.ts new file mode 100644 index 0000000000..f7d3ed6b31 --- /dev/null +++ b/packages/store-sync/src/recs/defineInternalComponents.ts @@ -0,0 +1,25 @@ +import { World, defineComponent, Type } from "@latticexyz/recs"; +import { Table } from "../common"; +import { StoreComponentMetadata } from "./common"; + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function defineInternalComponents(world: World) { + return { + TableMetadata: defineComponent<{ table: Type.T }, StoreComponentMetadata, Table>( + world, + { table: Type.T }, + { metadata: { keySchema: {}, valueSchema: {} } } + ), + SyncProgress: defineComponent( + world, + { + step: Type.String, + message: Type.String, + percentage: Type.Number, + }, + { + metadata: { keySchema: {}, valueSchema: {} }, + } + ), + }; +} diff --git a/packages/store-sync/src/recs/encodeEntity.ts b/packages/store-sync/src/recs/encodeEntity.ts new file mode 100644 index 0000000000..92f4c839ef --- /dev/null +++ b/packages/store-sync/src/recs/encodeEntity.ts @@ -0,0 +1,19 @@ +import { Entity } from "@latticexyz/recs"; +import { StaticAbiType } from "@latticexyz/schema-type"; +import { encodeAbiParameters } from "viem"; +import { SchemaToPrimitives } from "@latticexyz/store"; +import { hexKeyTupleToEntity } from "./hexKeyTupleToEntity"; + +export function encodeEntity>( + keySchema: TKeySchema, + key: SchemaToPrimitives +): Entity { + if (Object.keys(keySchema).length !== Object.keys(key).length) { + throw new Error( + `key length ${Object.keys(key).length} does not match key schema length ${Object.keys(keySchema).length}` + ); + } + return hexKeyTupleToEntity( + Object.entries(keySchema).map(([keyName, type]) => encodeAbiParameters([{ type }], [key[keyName]])) + ); +} diff --git a/packages/store-sync/src/recs/entityToHexKeyTuple.ts b/packages/store-sync/src/recs/entityToHexKeyTuple.ts new file mode 100644 index 0000000000..40bc5cf0db --- /dev/null +++ b/packages/store-sync/src/recs/entityToHexKeyTuple.ts @@ -0,0 +1,13 @@ +import { Entity } from "@latticexyz/recs"; +import { Hex, sliceHex, size, isHex } from "viem"; + +export function entityToHexKeyTuple(entity: Entity): readonly Hex[] { + if (!isHex(entity)) { + throw new Error(`entity ${entity} is not a hex string`); + } + const length = size(entity); + if (length % 32 !== 0) { + throw new Error(`entity length ${length} is not a multiple of 32 bytes`); + } + return new Array(length / 32).fill(0).map((_, index) => sliceHex(entity, index * 32, (index + 1) * 32)); +} diff --git a/packages/store-sync/src/recs/getTableKey.ts b/packages/store-sync/src/recs/getTableKey.ts new file mode 100644 index 0000000000..5a8edb9f35 --- /dev/null +++ b/packages/store-sync/src/recs/getTableKey.ts @@ -0,0 +1,8 @@ +import { Address, getAddress } from "viem"; +import { Table, TableName, TableNamespace } from "../common"; + +export type TableKey = `${Address}:${TableNamespace}:${TableName}`; + +export function getTableKey(table: Pick): TableKey { + return `${getAddress(table.address)}:${table.namespace}:${table.name}`; +} diff --git a/packages/store-sync/src/recs/hexKeyTupleToEntity.ts b/packages/store-sync/src/recs/hexKeyTupleToEntity.ts new file mode 100644 index 0000000000..88cc408f2f --- /dev/null +++ b/packages/store-sync/src/recs/hexKeyTupleToEntity.ts @@ -0,0 +1,6 @@ +import { Entity } from "@latticexyz/recs"; +import { Hex, concatHex } from "viem"; + +export function hexKeyTupleToEntity(hexKeyTuple: readonly Hex[]): Entity { + return concatHex(hexKeyTuple as Hex[]) as Entity; +} diff --git a/packages/store-sync/src/recs/index.ts b/packages/store-sync/src/recs/index.ts new file mode 100644 index 0000000000..8809c234e0 --- /dev/null +++ b/packages/store-sync/src/recs/index.ts @@ -0,0 +1,7 @@ +export * from "./common"; +export * from "./decodeEntity"; +export * from "./encodeEntity"; +export * from "./entityToHexKeyTuple"; +export * from "./hexKeyTupleToEntity"; +export * from "./recsStorage"; +export * from "./syncToRecs"; diff --git a/packages/store-sync/src/recs/recsStorage.ts b/packages/store-sync/src/recs/recsStorage.ts new file mode 100644 index 0000000000..2924b4a6d0 --- /dev/null +++ b/packages/store-sync/src/recs/recsStorage.ts @@ -0,0 +1,96 @@ +import { BlockLogsToStorageOptions } from "../blockLogsToStorage"; +import { StoreConfig } from "@latticexyz/store"; +import { debug } from "./debug"; +import { + ComponentValue, + Entity, + Component as RecsComponent, + Schema as RecsSchema, + getComponentValue, + removeComponent, + setComponent, + updateComponent, +} from "@latticexyz/recs"; +import { isDefined } from "@latticexyz/common/utils"; +import { TableId } from "@latticexyz/common"; +import { schemaToDefaults } from "../schemaToDefaults"; +import { hexKeyTupleToEntity } from "./hexKeyTupleToEntity"; +import { defineInternalComponents } from "./defineInternalComponents"; +import { getTableKey } from "./getTableKey"; +import { StoreComponentMetadata } from "./common"; + +// TODO: should we create components here from config rather than passing them in? + +export function recsStorage({ + components, +}: { + components: ReturnType & + Record>; + config?: TConfig; +}): BlockLogsToStorageOptions { + // TODO: do we need to store block number? + + const componentsByTableId = Object.fromEntries( + Object.entries(components).map(([id, component]) => [component.id, component]) + ); + + return { + async registerTables({ tables }) { + for (const table of tables) { + // TODO: check if table exists already and skip/warn? + setComponent(components.TableMetadata, getTableKey(table) as Entity, { table }); + } + }, + async getTables({ tables }) { + // TODO: fetch schema from RPC if table not found? + return tables + .map((table) => getComponentValue(components.TableMetadata, getTableKey(table) as Entity)?.table) + .filter(isDefined); + }, + async storeOperations({ operations }) { + for (const operation of operations) { + const table = getComponentValue( + components.TableMetadata, + getTableKey({ + address: operation.log.address, + namespace: operation.namespace, + name: operation.name, + }) as Entity + )?.table; + if (!table) { + debug( + `skipping update for unknown table: ${operation.namespace}:${operation.name} at ${operation.log.address}` + ); + continue; + } + + const tableId = new TableId(operation.namespace, operation.name).toString(); + const component = componentsByTableId[operation.log.args.table]; + if (!component) { + debug(`skipping update for unknown component: ${tableId}. Available components: ${Object.keys(components)}`); + continue; + } + + const entity = hexKeyTupleToEntity(operation.log.args.key); + + if (operation.type === "SetRecord") { + debug("setting component", tableId, entity, operation.value); + setComponent(component, entity, operation.value as ComponentValue); + } else if (operation.type === "SetField") { + debug("updating component", tableId, entity, { + [operation.fieldName]: operation.fieldValue, + }); + updateComponent( + component, + entity, + { [operation.fieldName]: operation.fieldValue } as ComponentValue, + schemaToDefaults(table.valueSchema) as ComponentValue + ); + } else if (operation.type === "DeleteRecord") { + debug("deleting component", tableId, entity); + removeComponent(component, entity); + } + } + }, + } as BlockLogsToStorageOptions; +} diff --git a/packages/store-sync/src/recs/syncToRecs.ts b/packages/store-sync/src/recs/syncToRecs.ts new file mode 100644 index 0000000000..0588aabdbe --- /dev/null +++ b/packages/store-sync/src/recs/syncToRecs.ts @@ -0,0 +1,228 @@ +import { StoreConfig, storeEventsAbi } from "@latticexyz/store"; +import { Address, Block, Chain, Hex, PublicClient, TransactionReceipt, Transport } from "viem"; +import { + ComponentValue, + Entity, + Component as RecsComponent, + Schema as RecsSchema, + World as RecsWorld, + getComponentValue, + setComponent, +} from "@latticexyz/recs"; +import { BlockLogs, Table } from "../common"; +import { TableRecord } from "@latticexyz/store"; +import { + createBlockStream, + isNonPendingBlock, + blockRangeToLogs, + groupLogsByBlockNumber, +} from "@latticexyz/block-logs-stream"; +import { filter, map, tap, mergeMap, from, concatMap, Observable, share, firstValueFrom } from "rxjs"; +import { BlockStorageOperations, blockLogsToStorage } from "../blockLogsToStorage"; +import { recsStorage } from "./recsStorage"; +import { hexKeyTupleToEntity } from "./hexKeyTupleToEntity"; +import { debug } from "./debug"; +import { defineInternalComponents } from "./defineInternalComponents"; +import { getTableKey } from "./getTableKey"; +import { StoreComponentMetadata, SyncStep } from "./common"; +import { encodeEntity } from "./encodeEntity"; + +type SyncToRecsOptions< + TConfig extends StoreConfig = StoreConfig, + TComponents extends Record> = Record< + string, + RecsComponent + > +> = { + world: RecsWorld; + config: TConfig; + address: Address; + // TODO: make this optional and return one if none provided (but will need chain ID at least) + publicClient: PublicClient; + // TODO: generate these from config and return instead? + components: TComponents; + initialState?: { + blockNumber: bigint | null; + tables: (Table & { records: TableRecord[] })[]; + }; +}; + +type SyncToRecsResult< + TConfig extends StoreConfig = StoreConfig, + TComponents extends Record> = Record< + string, + RecsComponent + > +> = { + // TODO: return publicClient? + components: TComponents & ReturnType; + singletonEntity: Entity; + latestBlock$: Observable; + latestBlockNumber$: Observable; + blockLogs$: Observable; + blockStorageOperations$: Observable>; + waitForTransaction: (tx: Hex) => Promise<{ receipt: TransactionReceipt }>; + destroy: () => void; +}; + +export async function syncToRecs< + TConfig extends StoreConfig = StoreConfig, + TComponents extends Record> = Record< + string, + RecsComponent + > +>({ + world, + config, + address, + publicClient, + components: initialComponents, + initialState, +}: SyncToRecsOptions): Promise> { + const components = { + ...initialComponents, + ...defineInternalComponents(world), + }; + + const singletonEntity = world.registerEntity({ id: hexKeyTupleToEntity([]) }); + + let startBlock = 0n; + + if (initialState != null && initialState.blockNumber != null) { + debug("hydrating from initial state to block", initialState.blockNumber); + startBlock = initialState.blockNumber + 1n; + + setComponent(components.SyncProgress, singletonEntity, { + step: SyncStep.SNAPSHOT, + message: `Hydrating from snapshot to block ${initialState.blockNumber}`, + percentage: 0, + }); + + const componentList = Object.values(components); + + const numRecords = initialState.tables.reduce((sum, table) => sum + table.records.length, 0); + const recordsPerSyncProgressUpdate = Math.floor(numRecords / 100); + let recordsProcessed = 0; + + for (const table of initialState.tables) { + setComponent(components.TableMetadata, getTableKey(table) as Entity, { table }); + const component = componentList.find((component) => component.id === table.tableId); + if (component == null) { + debug(`no component found for table ${table.namespace}:${table.name}, skipping initial state`); + continue; + } + for (const record of table.records) { + const entity = encodeEntity(table.keySchema, record.key); + setComponent(component, entity, record.value as ComponentValue); + + recordsProcessed++; + if (recordsProcessed % recordsPerSyncProgressUpdate === 0) { + setComponent(components.SyncProgress, singletonEntity, { + step: SyncStep.SNAPSHOT, + message: `Hydrating from snapshot to block ${initialState.blockNumber}`, + percentage: (recordsProcessed / numRecords) * 100, + }); + } + } + debug(`hydrated ${table.records.length} records for table ${table.namespace}:${table.name}`); + } + + setComponent(components.SyncProgress, singletonEntity, { + step: SyncStep.SNAPSHOT, + message: `Hydrating from snapshot to block ${initialState.blockNumber}`, + percentage: (recordsProcessed / numRecords) * 100, + }); + } + + // TODO: if startBlock is still 0, find via deploy event + + debug("starting sync from block", startBlock); + + const latestBlock$ = createBlockStream({ publicClient, blockTag: "latest" }).pipe(share()); + + const latestBlockNumber$ = latestBlock$.pipe( + filter(isNonPendingBlock), + map((block) => block.number), + share() + ); + + let latestBlockNumber: bigint | null = null; + const blockLogs$ = latestBlockNumber$.pipe( + tap((blockNumber) => { + debug("latest block number", blockNumber); + latestBlockNumber = blockNumber; + }), + map((blockNumber) => ({ startBlock, endBlock: blockNumber })), + blockRangeToLogs({ + publicClient, + address, + events: storeEventsAbi, + }), + mergeMap(({ toBlock, logs }) => from(groupLogsByBlockNumber(logs, toBlock))), + share() + ); + + let latestBlockNumberProcessed: bigint | null = null; + const blockStorageOperations$ = blockLogs$.pipe( + concatMap(blockLogsToStorage(recsStorage({ components, config }))), + tap(({ blockNumber, operations }) => { + debug("stored", operations.length, "operations for block", blockNumber); + latestBlockNumberProcessed = blockNumber; + + if ( + latestBlockNumber != null && + getComponentValue(components.SyncProgress, singletonEntity)?.step !== SyncStep.LIVE + ) { + if (blockNumber < latestBlockNumber) { + setComponent(components.SyncProgress, singletonEntity, { + step: SyncStep.RPC, + message: `Hydrating from RPC to block ${latestBlockNumber}`, + percentage: (Number(blockNumber) / Number(latestBlockNumber)) * 100, + }); + } else { + setComponent(components.SyncProgress, singletonEntity, { + step: SyncStep.LIVE, + message: `All caught up!`, + percentage: 100, + }); + } + } + }), + share() + ); + + // Start the sync + const sub = blockStorageOperations$.subscribe(); + world.registerDisposer(() => sub.unsubscribe()); + + async function waitForTransaction(tx: Hex): Promise<{ + receipt: TransactionReceipt; + }> { + // Wait for tx to be mined + const receipt = await publicClient.waitForTransactionReceipt({ hash: tx }); + + // If we haven't processed a block yet or we haven't processed the block for the tx, wait for it + if (latestBlockNumberProcessed == null || latestBlockNumberProcessed < receipt.blockNumber) { + await firstValueFrom( + blockStorageOperations$.pipe( + filter(({ blockNumber }) => blockNumber != null && blockNumber >= receipt.blockNumber) + ) + ); + } + + return { receipt }; + } + + return { + components, + singletonEntity, + latestBlock$, + latestBlockNumber$, + blockLogs$, + blockStorageOperations$, + waitForTransaction, + destroy: (): void => { + world.dispose(); + }, + }; +} diff --git a/packages/store-sync/src/schemaToDefaults.ts b/packages/store-sync/src/schemaToDefaults.ts index 105308dc89..e964dc224a 100644 --- a/packages/store-sync/src/schemaToDefaults.ts +++ b/packages/store-sync/src/schemaToDefaults.ts @@ -1,5 +1,5 @@ import { schemaAbiTypeToDefaultValue } from "@latticexyz/schema-type"; -import { ValueSchema, SchemaToPrimitives } from "./common"; +import { ValueSchema, SchemaToPrimitives } from "@latticexyz/store"; export function schemaToDefaults(schema: TSchema): SchemaToPrimitives { return Object.fromEntries( diff --git a/packages/store-sync/tsup.config.ts b/packages/store-sync/tsup.config.ts index a6e720afa7..666f97a339 100644 --- a/packages/store-sync/tsup.config.ts +++ b/packages/store-sync/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts", "src/sqlite/index.ts"], + entry: ["src/index.ts", "src/sqlite/index.ts", "src/recs/index.ts"], target: "esnext", format: ["esm"], dts: false, diff --git a/packages/store/ts/common.ts b/packages/store/ts/common.ts new file mode 100644 index 0000000000..98df7a5240 --- /dev/null +++ b/packages/store/ts/common.ts @@ -0,0 +1,48 @@ +import { SchemaAbiType, SchemaAbiTypeToPrimitiveType, StaticAbiType } from "@latticexyz/schema-type"; +import { FieldData, FullSchemaConfig, StoreConfig } from "./config"; + +export type KeySchema = Record; +export type ValueSchema = Record; + +/** Map a table schema like `{ value: "uint256" }` to its primitive types like `{ value: bigint }` */ +export type SchemaToPrimitives = { + [key in keyof TSchema]: SchemaAbiTypeToPrimitiveType; +}; + +export type TableRecord = { + key: SchemaToPrimitives; + value: SchemaToPrimitives; +}; + +export type ConfigFieldTypeToPrimitiveType> = T extends SchemaAbiType + ? SchemaAbiTypeToPrimitiveType + : T extends `${string}[${string}]` // field type might include enums and enum arrays, which are mapped to uint8/uint8[] + ? number[] // map enum arrays to `number[]` + : number; // map enums to `number` + +/** Map a table schema config like `{ value: "uint256", type: "SomeEnum" }` to its primitive types like `{ value: bigint, type: number }` */ +export type SchemaConfigToPrimitives = { + [key in keyof T]: ConfigFieldTypeToPrimitiveType; +}; + +export type ConfigToTablesPrimitives = { + [key in keyof C["tables"]]: { + key: SchemaConfigToPrimitives; + value: SchemaConfigToPrimitives; + }; +}; + +export type ConfigToKeyPrimitives< + C extends StoreConfig, + Table extends keyof ConfigToTablesPrimitives +> = ConfigToTablesPrimitives[Table]["key"]; + +export type ConfigToValuePrimitives< + C extends StoreConfig, + Table extends keyof ConfigToTablesPrimitives +> = ConfigToTablesPrimitives[Table]["value"]; + +export type ConfigToRecordPrimitives> = { + key: ConfigToKeyPrimitives; + value: ConfigToValuePrimitives; +}; diff --git a/packages/store/ts/index.ts b/packages/store/ts/index.ts index d3a12ccc83..16df2f12f0 100644 --- a/packages/store/ts/index.ts +++ b/packages/store/ts/index.ts @@ -2,5 +2,6 @@ // (library neither creates nor extends MUDCoreContext when imported) export * from "./config"; +export * from "./common"; export * from "./storeEvents"; export * from "./storeEventsAbi"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 966bbda4b8..cf5ba9c639 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1037,15 +1037,15 @@ importers: '@latticexyz/protocol-parser': specifier: workspace:* version: link:../protocol-parser + '@latticexyz/recs': + specifier: workspace:* + version: link:../recs '@latticexyz/schema-type': specifier: workspace:* version: link:../schema-type '@latticexyz/store': specifier: workspace:* version: link:../store - '@latticexyz/store-cache': - specifier: workspace:* - version: link:../store-cache better-sqlite3: specifier: ^8.4.0 version: 8.4.0 @@ -1058,6 +1058,9 @@ importers: kysely: specifier: ^0.26.1 version: 0.26.1 + rxjs: + specifier: 7.5.5 + version: 7.5.5 sql.js: specifier: ^1.8.0 version: 1.8.0