From 4c2b6b4dc4a1df11be84567ec6bd4d6c53ad37ff Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 14 Jun 2023 21:36:35 -0700 Subject: [PATCH 01/20] first pass --- .../packages/client-react/package.json | 2 + .../client-react/src/mud/getNetworkConfig.ts | 3 + .../packages/client-react/src/mud/setup.ts | 3 + .../client-react/src/mud/setupViemNetwork.ts | 35 ++++++ examples/minimal/pnpm-lock.yaml | 6 ++ packages/block-events-stream/.eslintrc | 6 ++ packages/block-events-stream/.gitignore | 1 + packages/block-events-stream/.npmignore | 6 ++ packages/block-events-stream/package.json | 42 ++++++++ packages/block-events-stream/src/common.ts | 14 +++ .../src/createBlockEventsStream.ts | 102 ++++++++++++++++++ .../src/excludePendingLogs.ts | 20 ++++ packages/block-events-stream/src/index.ts | 1 + packages/block-events-stream/src/utils.ts | 7 ++ packages/block-events-stream/tsconfig.json | 14 +++ packages/block-events-stream/tsup.config.ts | 11 ++ packages/common/src/utils/index.ts | 3 + packages/common/src/utils/isDefined.ts | 3 + packages/common/src/utils/isNotNull.ts | 3 + packages/common/src/utils/isPresent.ts | 3 + packages/store/package.json | 1 + packages/store/ts/library/index.ts | 3 + packages/store/ts/library/storeEvents.ts | 6 ++ .../store/ts/library/storeEventsAbi.test.ts | 27 +++++ packages/store/ts/library/storeEventsAbi.ts | 4 + pnpm-lock.yaml | 41 ++++++- 26 files changed, 364 insertions(+), 3 deletions(-) create mode 100644 examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts create mode 100644 packages/block-events-stream/.eslintrc create mode 100644 packages/block-events-stream/.gitignore create mode 100644 packages/block-events-stream/.npmignore create mode 100644 packages/block-events-stream/package.json create mode 100644 packages/block-events-stream/src/common.ts create mode 100644 packages/block-events-stream/src/createBlockEventsStream.ts create mode 100644 packages/block-events-stream/src/excludePendingLogs.ts create mode 100644 packages/block-events-stream/src/index.ts create mode 100644 packages/block-events-stream/src/utils.ts create mode 100644 packages/block-events-stream/tsconfig.json create mode 100644 packages/block-events-stream/tsup.config.ts create mode 100644 packages/common/src/utils/isDefined.ts create mode 100644 packages/common/src/utils/isNotNull.ts create mode 100644 packages/common/src/utils/isPresent.ts create mode 100644 packages/store/ts/library/storeEvents.ts create mode 100644 packages/store/ts/library/storeEventsAbi.test.ts create mode 100644 packages/store/ts/library/storeEventsAbi.ts diff --git a/examples/minimal/packages/client-react/package.json b/examples/minimal/packages/client-react/package.json index 36951fd8cc..1e6420c293 100644 --- a/examples/minimal/packages/client-react/package.json +++ b/examples/minimal/packages/client-react/package.json @@ -12,6 +12,7 @@ "dependencies": { "@ethersproject/providers": "^5.7.2", "@improbable-eng/grpc-web": "^0.15.0", + "@latticexyz/block-events-stream": "link:../../../../packages/block-events-stream", "@latticexyz/common": "link:../../../../packages/common", "@latticexyz/dev-tools": "link:../../../../packages/dev-tools", "@latticexyz/network": "link:../../../../packages/network", @@ -20,6 +21,7 @@ "@latticexyz/schema-type": "link:../../../../packages/schema-type", "@latticexyz/services": "link:../../../../packages/services", "@latticexyz/std-client": "link:../../../../packages/std-client", + "@latticexyz/store": "link:../../../../packages/store", "@latticexyz/utils": "link:../../../../packages/utils", "@latticexyz/world": "link:../../../../packages/world", "@wagmi/chains": "^0.2.22", diff --git a/examples/minimal/packages/client-react/src/mud/getNetworkConfig.ts b/examples/minimal/packages/client-react/src/mud/getNetworkConfig.ts index 0867bb5c66..45e49c8ee5 100644 --- a/examples/minimal/packages/client-react/src/mud/getNetworkConfig.ts +++ b/examples/minimal/packages/client-react/src/mud/getNetworkConfig.ts @@ -1,6 +1,7 @@ import { SetupContractConfig, getBurnerWallet } from "@latticexyz/std-client"; import worldsJson from "contracts/worlds.json"; import { supportedChains } from "./supportedChains"; +import { Chain } from "@wagmi/chains"; const worlds = worldsJson as Partial>; @@ -8,6 +9,7 @@ type NetworkConfig = SetupContractConfig & { privateKey: string; faucetServiceUrl?: string; snapSync?: boolean; + chain?: Chain; }; export async function getNetworkConfig(): Promise { @@ -49,5 +51,6 @@ export async function getNetworkConfig(): Promise { initialBlockNumber, snapSync: params.get("snapSync") === "true", disableCache: params.get("cache") === "false", + chain, }; } diff --git a/examples/minimal/packages/client-react/src/mud/setup.ts b/examples/minimal/packages/client-react/src/mud/setup.ts index 4f79edd8f3..3783ac9fa9 100644 --- a/examples/minimal/packages/client-react/src/mud/setup.ts +++ b/examples/minimal/packages/client-react/src/mud/setup.ts @@ -1,10 +1,13 @@ import { createClientComponents } from "./createClientComponents"; import { createSystemCalls } from "./createSystemCalls"; import { setupNetwork } from "./setupNetwork"; +import { setupViemNetwork } from "./setupViemNetwork"; export type SetupResult = Awaited>; export async function setup() { + await setupViemNetwork(); + const network = await setupNetwork(); const components = createClientComponents(network); const systemCalls = createSystemCalls(network, components); diff --git a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts new file mode 100644 index 0000000000..7b9d247e22 --- /dev/null +++ b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts @@ -0,0 +1,35 @@ +import { getNetworkConfig } from "./getNetworkConfig"; +import { createBlockEventsStream } from "@latticexyz/block-events-stream"; +// import { storeEventsAbi } from "@latticexyz/store"; +import { createPublicClient, http, fallback, webSocket } from "viem"; +import { parseAbi } from "viem/abi"; + +// TODO: import from store once we move aside the solidity codegen stuff that breaks browser bundles +export const storeEvents = [ + "event StoreDeleteRecord(bytes32 table, bytes32[] key)", + "event StoreSetField(bytes32 table, bytes32[] key, uint8 schemaIndex, bytes data)", + "event StoreSetRecord(bytes32 table, bytes32[] key, bytes data)", + "event StoreEphemeralRecord(bytes32 table, bytes32[] key, bytes data)", +] as const; +export const storeEventsAbi = parseAbi(storeEvents); + +export async function setupViemNetwork() { + const { chain } = await getNetworkConfig(); + console.log("viem chain", chain); + + const publicClient = createPublicClient({ + chain, + transport: fallback([webSocket(), http()]), + }); + + const stream = await createBlockEventsStream({ + publicClient, + events: storeEventsAbi, + }); + + console.log("stream established"); + + stream.subscribe((block) => { + console.log("stream block", block); + }); +} diff --git a/examples/minimal/pnpm-lock.yaml b/examples/minimal/pnpm-lock.yaml index 7c3f69dd4e..cb0a482ce2 100644 --- a/examples/minimal/pnpm-lock.yaml +++ b/examples/minimal/pnpm-lock.yaml @@ -156,6 +156,9 @@ importers: '@improbable-eng/grpc-web': specifier: ^0.15.0 version: 0.15.0(google-protobuf@3.21.2) + '@latticexyz/block-events-stream': + specifier: link:../../../../packages/block-events-stream + version: link:../../../../packages/block-events-stream '@latticexyz/common': specifier: link:../../../../packages/common version: link:../../../../packages/common @@ -180,6 +183,9 @@ importers: '@latticexyz/std-client': specifier: link:../../../../packages/std-client version: link:../../../../packages/std-client + '@latticexyz/store': + specifier: link:../../../../packages/store + version: link:../../../../packages/store '@latticexyz/utils': specifier: link:../../../../packages/utils version: link:../../../../packages/utils diff --git a/packages/block-events-stream/.eslintrc b/packages/block-events-stream/.eslintrc new file mode 100644 index 0000000000..6db0063ad7 --- /dev/null +++ b/packages/block-events-stream/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": ["../../.eslintrc"], + "rules": { + "@typescript-eslint/explicit-function-return-type": "error" + } +} diff --git a/packages/block-events-stream/.gitignore b/packages/block-events-stream/.gitignore new file mode 100644 index 0000000000..1521c8b765 --- /dev/null +++ b/packages/block-events-stream/.gitignore @@ -0,0 +1 @@ +dist diff --git a/packages/block-events-stream/.npmignore b/packages/block-events-stream/.npmignore new file mode 100644 index 0000000000..84815f1eba --- /dev/null +++ b/packages/block-events-stream/.npmignore @@ -0,0 +1,6 @@ +* + +!dist/** +!src/** +!package.json +!README.md diff --git a/packages/block-events-stream/package.json b/packages/block-events-stream/package.json new file mode 100644 index 0000000000..113701164e --- /dev/null +++ b/packages/block-events-stream/package.json @@ -0,0 +1,42 @@ +{ + "name": "@latticexyz/block-events-stream", + "version": "1.42.0", + "description": "Create a stream of EVM block events", + "repository": { + "type": "git", + "url": "https://github.com/latticexyz/mud.git", + "directory": "packages/block-events-stream" + }, + "license": "MIT", + "type": "module", + "exports": { + ".": "./dist/index.js" + }, + "types": "src/index.ts", + "scripts": { + "build": "pnpm run build:js", + "build:js": "tsup", + "clean": "pnpm run clean:js", + "clean:js": "rimraf dist", + "dev": "tsup --watch", + "lint": "eslint .", + "test": "vitest typecheck --run && vitest --run" + }, + "dependencies": { + "@latticexyz/common": "workspace:*", + "@latticexyz/config": "workspace:*", + "@latticexyz/schema-type": "workspace:*", + "@latticexyz/store": "workspace:*", + "abitype": "0.8.7", + "rxjs": "^7.5.5", + "viem": "1.0.6" + }, + "devDependencies": { + "tsup": "^6.7.0", + "vitest": "0.31.4" + }, + "publishConfig": { + "access": "public" + }, + "gitHead": "914a1e0ae4a573d685841ca2ea921435057deb8f" +} diff --git a/packages/block-events-stream/src/common.ts b/packages/block-events-stream/src/common.ts new file mode 100644 index 0000000000..7ad42d267a --- /dev/null +++ b/packages/block-events-stream/src/common.ts @@ -0,0 +1,14 @@ +import { Observable } from "rxjs"; +import type { BlockNumber, BlockTag, GetLogsReturnType, Hex } from "viem"; +import type { AbiEvent } from "abitype"; + +// TODO: support pending blocks +export type BlockNumberOrTag = BlockNumber | Exclude; + +export type BlockEvents = { + blockNumber: BlockNumber; + blockHash: Hex; + events: GetLogsReturnType; // TODO: refine to be a store event log +}; + +export type BlockEventsStream = Observable>; diff --git a/packages/block-events-stream/src/createBlockEventsStream.ts b/packages/block-events-stream/src/createBlockEventsStream.ts new file mode 100644 index 0000000000..00128a0a5e --- /dev/null +++ b/packages/block-events-stream/src/createBlockEventsStream.ts @@ -0,0 +1,102 @@ +import { Subject } from "rxjs"; +import type { Hex, PublicClient } from "viem"; +import type { AbiEvent } from "abitype"; +import { BlockEvents, BlockEventsStream, BlockNumberOrTag } from "./common"; +import { bigIntMin } from "./utils"; +import { excludePendingLogs } from "./excludePendingLogs"; + +// TODO: add nice logging with debub lib or similar +// TODO: make `toBlock` accept a `BehaviorSubject` or add `latestBlockStream` so we only need one listener/watcher/poller +// TODO: consider excluding `pending` block tags so we can just assume all block numbers are present + +export type CreateBlockEventsStreamOptions = { + publicClient: PublicClient; + fromBlock?: BlockNumberOrTag; // defaults to "earliest" + toBlock?: BlockNumberOrTag; // defaults to "latest" + address?: Hex; + events: readonly TAbiEvent[]; + maxBlockRange?: number; // defaults to 1000 +}; + +export async function createBlockEventsStream({ + publicClient, + fromBlock: initialFromBlock = "earliest", + toBlock: initialToBlock = "latest", + address, + events, + maxBlockRange = 1000, +}: CreateBlockEventsStreamOptions): Promise> { + const [firstBlock, lastBlock] = await Promise.all([ + publicClient.getBlock( + typeof initialFromBlock === "bigint" ? { blockNumber: initialFromBlock } : { blockTag: initialFromBlock } + ), + publicClient.getBlock( + typeof initialToBlock === "bigint" ? { blockNumber: initialToBlock } : { blockTag: initialToBlock } + ), + ]); + + if (firstBlock.number == null) { + // TODO: better error + throw new Error(`pending or missing fromBlock "${initialFromBlock}"`); + } + if (lastBlock.number == null) { + // TODO: better error + throw new Error(`pending or missing toBlock "${initialToBlock}"`); + } + + const stream = new Subject>(); + fetchBlockRange(firstBlock.number, maxBlockRange, lastBlock.number); + + async function fetchBlockRange(fromBlock: bigint, maxBlockRange: number, lastBlockNumber: bigint): Promise { + try { + const toBlock = bigIntMin(fromBlock + BigInt(maxBlockRange), lastBlockNumber); + + // TODO: convert to one `getLogs` call when viem supports multiple events or topics + // TODO: do something other than just throwing out pending logs + const logs = excludePendingLogs( + ( + await Promise.all( + events.map((event) => + publicClient.getLogs({ + address, + event, + fromBlock, + toBlock, + }) + ) + ) + ).flat() + ); + + // TODO: handle RPC block range errors + // TODO: handle RPC rate limit errors (hopefully via client retry policy) + + const blockNumbers = Array.from(new Set(logs.map((log) => log.blockNumber))); + blockNumbers.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + + for (const blockNumber of blockNumbers) { + const blockLogs = logs.filter((log) => log.blockNumber === blockNumber); + blockLogs.sort((a, b) => (a.logIndex < b.logIndex ? -1 : a.logIndex > b.logIndex ? 1 : 0)); + + stream.next({ + blockNumber, + blockHash: blockLogs[0].blockHash, + events: blockLogs, + }); + } + + if (toBlock > lastBlockNumber) { + fetchBlockRange(toBlock + 1n, maxBlockRange, lastBlockNumber); + return; + } + + // TODO: determine if we can/should start adjusting `lastBlock` (based on if we got a block tag for `initialToBlock`) + stream.complete(); + } catch (error: unknown) { + // TODO: do more specific error handling? + stream.error(error); + } + } + + return stream.asObservable(); +} diff --git a/packages/block-events-stream/src/excludePendingLogs.ts b/packages/block-events-stream/src/excludePendingLogs.ts new file mode 100644 index 0000000000..93700ef638 --- /dev/null +++ b/packages/block-events-stream/src/excludePendingLogs.ts @@ -0,0 +1,20 @@ +import type { Log } from "viem"; + +type NonPendingLog = TLog & { + blockHash: NonNullable; + blockNumber: NonNullable; + logIndex: NonNullable; + transactionHash: NonNullable; + transactionIndex: NonNullable; +}; + +export function excludePendingLogs(logs: TLog[]): NonPendingLog[] { + return logs.filter( + (log) => + log.blockHash != null && + log.blockNumber != null && + log.logIndex != null && + log.transactionHash != null && + log.transactionIndex != null + ) as NonPendingLog[]; +} diff --git a/packages/block-events-stream/src/index.ts b/packages/block-events-stream/src/index.ts new file mode 100644 index 0000000000..62502925c9 --- /dev/null +++ b/packages/block-events-stream/src/index.ts @@ -0,0 +1 @@ +export * from "./createBlockEventsStream"; diff --git a/packages/block-events-stream/src/utils.ts b/packages/block-events-stream/src/utils.ts new file mode 100644 index 0000000000..33b0f506bc --- /dev/null +++ b/packages/block-events-stream/src/utils.ts @@ -0,0 +1,7 @@ +export function bigIntMin(...args: bigint[]): bigint { + return args.reduce((m, e) => (e < m ? e : m)); +} + +export function bigIntMax(...args: bigint[]): bigint { + return args.reduce((m, e) => (e > m ? e : m)); +} diff --git a/packages/block-events-stream/tsconfig.json b/packages/block-events-stream/tsconfig.json new file mode 100644 index 0000000000..e590f0c026 --- /dev/null +++ b/packages/block-events-stream/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "es2021", + "module": "esnext", + "moduleResolution": "node", + "declaration": true, + "sourceMap": true, + "outDir": "dist", + "isolatedModules": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true + } +} diff --git a/packages/block-events-stream/tsup.config.ts b/packages/block-events-stream/tsup.config.ts new file mode 100644 index 0000000000..b755469f90 --- /dev/null +++ b/packages/block-events-stream/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + target: "esnext", + format: ["esm"], + dts: false, + sourcemap: true, + clean: true, + minify: true, +}); diff --git a/packages/common/src/utils/index.ts b/packages/common/src/utils/index.ts index f3798fe37b..5ff39b2b9e 100644 --- a/packages/common/src/utils/index.ts +++ b/packages/common/src/utils/index.ts @@ -1 +1,4 @@ export * from "./curry"; +export * from "./isDefined"; +export * from "./isNotNull"; +export * from "./isPresent"; diff --git a/packages/common/src/utils/isDefined.ts b/packages/common/src/utils/isDefined.ts new file mode 100644 index 0000000000..ba1a0e8b23 --- /dev/null +++ b/packages/common/src/utils/isDefined.ts @@ -0,0 +1,3 @@ +export function isDefined(argument: T | undefined): argument is T { + return argument !== undefined; +} diff --git a/packages/common/src/utils/isNotNull.ts b/packages/common/src/utils/isNotNull.ts new file mode 100644 index 0000000000..220471a8a6 --- /dev/null +++ b/packages/common/src/utils/isNotNull.ts @@ -0,0 +1,3 @@ +export function isNotNull(argument: T | null): argument is T { + return argument !== null; +} diff --git a/packages/common/src/utils/isPresent.ts b/packages/common/src/utils/isPresent.ts new file mode 100644 index 0000000000..0a901269d1 --- /dev/null +++ b/packages/common/src/utils/isPresent.ts @@ -0,0 +1,3 @@ +export function isPresent(argument: T | null | undefined): argument is T { + return argument !== null && argument !== undefined; +} diff --git a/packages/store/package.json b/packages/store/package.json index 9abca9b640..6db0505c5c 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -52,6 +52,7 @@ "@latticexyz/common": "workspace:*", "@latticexyz/config": "workspace:*", "@latticexyz/schema-type": "workspace:*", + "abitype": "0.8.7", "ethers": "^5.7.2", "zod": "^3.21.4" }, diff --git a/packages/store/ts/library/index.ts b/packages/store/ts/library/index.ts index aff09fcf79..4387851ef6 100644 --- a/packages/store/ts/library/index.ts +++ b/packages/store/ts/library/index.ts @@ -2,3 +2,6 @@ // (library neither creates nor extends MUDCoreContext when imported) export * from "./config"; export * from "./render-solidity"; + +export * from "./storeEvents"; +export * from "./storeEventsAbi"; diff --git a/packages/store/ts/library/storeEvents.ts b/packages/store/ts/library/storeEvents.ts new file mode 100644 index 0000000000..f65d884db5 --- /dev/null +++ b/packages/store/ts/library/storeEvents.ts @@ -0,0 +1,6 @@ +export const storeEvents = [ + "event StoreDeleteRecord(bytes32 table, bytes32[] key)", + "event StoreSetField(bytes32 table, bytes32[] key, uint8 schemaIndex, bytes data)", + "event StoreSetRecord(bytes32 table, bytes32[] key, bytes data)", + "event StoreEphemeralRecord(bytes32 table, bytes32[] key, bytes data)", +] as const; diff --git a/packages/store/ts/library/storeEventsAbi.test.ts b/packages/store/ts/library/storeEventsAbi.test.ts new file mode 100644 index 0000000000..f08e15ed72 --- /dev/null +++ b/packages/store/ts/library/storeEventsAbi.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { storeEventsAbi } from "./storeEventsAbi"; +import { IStore__factory } from "../../types/ethers-contracts"; + +// Make sure `storeEvents` stays in sync with Solidity definition/events + +describe("storeEventsAbi", () => { + it("should match the store ABI", () => { + const expectedAbi = IStore__factory.abi + .filter((item) => item.type === "event") + .map((item) => ({ + // transform because typechain adds a bunch of data that abitype doesn't care about + type: item.type, + name: item.name, + inputs: [ + ...item.inputs.map((input) => ({ + name: input.name, + type: input.type, + })), + ], + })) + .sort((a, b) => a.name.localeCompare(b.name)); + + const sortedStoreEventsAbi = storeEventsAbi.slice().sort((a, b) => a.name.localeCompare(b.name)); + expect(sortedStoreEventsAbi).toStrictEqual(expectedAbi); + }); +}); diff --git a/packages/store/ts/library/storeEventsAbi.ts b/packages/store/ts/library/storeEventsAbi.ts new file mode 100644 index 0000000000..3d383d6e1c --- /dev/null +++ b/packages/store/ts/library/storeEventsAbi.ts @@ -0,0 +1,4 @@ +import { parseAbi, AbiEvent } from "abitype"; +import { storeEvents } from "./storeEvents"; + +export const storeEventsAbi = parseAbi(storeEvents) satisfies readonly AbiEvent[]; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be8b39f32b..16cc2965dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -57,6 +57,37 @@ importers: specifier: ^4.9.5 version: 4.9.5 + packages/block-events-stream: + dependencies: + '@latticexyz/common': + specifier: workspace:* + version: link:../common + '@latticexyz/config': + specifier: workspace:* + version: link:../config + '@latticexyz/schema-type': + specifier: workspace:* + version: link:../schema-type + '@latticexyz/store': + specifier: workspace:* + version: link:../store + abitype: + specifier: 0.8.7 + version: 0.8.7(typescript@5.0.4) + rxjs: + specifier: ^7.5.5 + version: 7.5.6 + viem: + specifier: 1.0.6 + version: 1.0.6(typescript@5.0.4) + devDependencies: + tsup: + specifier: ^6.7.0 + version: 6.7.0(postcss@8.4.23)(typescript@5.0.4) + vitest: + specifier: 0.31.4 + version: 0.31.4 + packages/cli: dependencies: '@ethersproject/abi': @@ -818,7 +849,7 @@ importers: version: link:../world abitype: specifier: 0.8.7 - version: 0.8.7(typescript@4.9.5) + version: 0.8.7(typescript@4.9.5)(zod@3.21.4) ethers: specifier: ^5.7.2 version: 5.7.2 @@ -925,6 +956,9 @@ importers: '@latticexyz/schema-type': specifier: workspace:* version: link:../schema-type + abitype: + specifier: 0.8.7 + version: 0.8.7(typescript@4.9.5)(zod@3.21.4) ethers: specifier: ^5.7.2 version: 5.7.2 @@ -4111,7 +4145,7 @@ packages: engines: {node: '>=6', npm: '>=3'} dev: false - /abitype@0.8.7(typescript@4.9.5): + /abitype@0.8.7(typescript@4.9.5)(zod@3.21.4): resolution: {integrity: sha512-wQ7hV8Yg/yKmGyFpqrNZufCxbszDe5es4AZGYPBitocfSqXtjrTG9JMWFcc4N30ukl2ve48aBTwt7NJxVQdU3w==} peerDependencies: typescript: '>=5.0.4' @@ -4121,6 +4155,7 @@ packages: optional: true dependencies: typescript: 4.9.5 + zod: 3.21.4 dev: false /abitype@0.8.7(typescript@5.0.4): @@ -14546,7 +14581,7 @@ packages: '@scure/bip32': 1.3.0 '@scure/bip39': 1.2.0 '@wagmi/chains': 1.1.0(typescript@4.9.5) - abitype: 0.8.7(typescript@4.9.5) + abitype: 0.8.7(typescript@4.9.5)(zod@3.21.4) isomorphic-ws: 5.0.0(ws@8.12.0) ws: 8.12.0 transitivePeerDependencies: From 9e40c41e9e3a6fb9bb80fd291a03ff0e2a1e899c Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 14 Jun 2023 21:48:31 -0700 Subject: [PATCH 02/20] move around store exports --- packages/cli/scripts/generate-test-tables.ts | 3 ++- packages/cli/src/commands/dev-contracts.ts | 3 ++- packages/cli/src/commands/tablegen.ts | 3 ++- packages/cli/src/render-ts/recsV1TableOptions.ts | 3 ++- packages/cli/src/utils/deploy.ts | 3 ++- packages/store/package.json | 16 ++++++++++------ .../render-solidity => codegen}/ephemeral.ts | 0 .../render-solidity => codegen}/field.ts | 0 .../render-solidity => codegen}/index.ts | 0 .../render-solidity => codegen}/record.ts | 0 .../render-solidity => codegen}/renderTable.ts | 0 .../renderTableIndex.ts | 0 .../renderTypesFromConfig.ts | 0 .../render-solidity => codegen}/tableOptions.ts | 0 .../render-solidity => codegen}/tablegen.ts | 0 .../render-solidity => codegen}/types.ts | 0 .../render-solidity => codegen}/userType.ts | 0 .../store/ts/{library => }/config/defaults.ts | 0 packages/store/ts/{library => }/config/index.ts | 0 .../{library => }/config/storeConfig.test-d.ts | 0 .../store/ts/{library => }/config/storeConfig.ts | 0 packages/store/ts/{library => }/index.ts | 1 - packages/store/ts/register/configExtensions.ts | 2 +- packages/store/ts/register/mudConfig.ts | 2 +- packages/store/ts/register/typeExtensions.ts | 4 ++-- packages/store/ts/scripts/tablegen.ts | 3 ++- packages/store/ts/{library => }/storeEvents.ts | 0 .../ts/{library => }/storeEventsAbi.test.ts | 0 .../store/ts/{library => }/storeEventsAbi.ts | 0 packages/store/tsup.config.ts | 2 +- .../ts/plugins/snapsync/configExtensions.ts | 2 +- packages/world/ts/scripts/tablegen.ts | 3 ++- 32 files changed, 30 insertions(+), 20 deletions(-) rename packages/store/ts/{library/render-solidity => codegen}/ephemeral.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/field.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/index.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/record.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/renderTable.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/renderTableIndex.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/renderTypesFromConfig.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/tableOptions.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/tablegen.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/types.ts (100%) rename packages/store/ts/{library/render-solidity => codegen}/userType.ts (100%) rename packages/store/ts/{library => }/config/defaults.ts (100%) rename packages/store/ts/{library => }/config/index.ts (100%) rename packages/store/ts/{library => }/config/storeConfig.test-d.ts (100%) rename packages/store/ts/{library => }/config/storeConfig.ts (100%) rename packages/store/ts/{library => }/index.ts (86%) rename packages/store/ts/{library => }/storeEvents.ts (100%) rename packages/store/ts/{library => }/storeEventsAbi.test.ts (100%) rename packages/store/ts/{library => }/storeEventsAbi.ts (100%) diff --git a/packages/cli/scripts/generate-test-tables.ts b/packages/cli/scripts/generate-test-tables.ts index 313860fc8d..ec8174f3ee 100644 --- a/packages/cli/scripts/generate-test-tables.ts +++ b/packages/cli/scripts/generate-test-tables.ts @@ -1,5 +1,6 @@ import path from "path"; -import { tablegen } from "@latticexyz/store"; +import { StoreConfig } from "@latticexyz/store"; +import { tablegen } from "@latticexyz/store/codegen"; import { mudConfig } from "@latticexyz/world/register"; import { getSrcDirectory } from "@latticexyz/common/foundry"; import { logError } from "../src/utils/errors"; diff --git a/packages/cli/src/commands/dev-contracts.ts b/packages/cli/src/commands/dev-contracts.ts index f770920872..43902e4b82 100644 --- a/packages/cli/src/commands/dev-contracts.ts +++ b/packages/cli/src/commands/dev-contracts.ts @@ -3,7 +3,8 @@ import { anvil, forge, getRpcUrl, getScriptDirectory, getSrcDirectory } from "@l import chalk from "chalk"; import chokidar from "chokidar"; import { loadConfig, resolveConfigPath } from "@latticexyz/config/node"; -import { StoreConfig, tablegen } from "@latticexyz/store"; +import { StoreConfig } from "@latticexyz/store"; +import { tablegen } from "@latticexyz/store/codegen"; import path from "path"; import { tsgen } from "../render-ts"; import { debounce } from "throttle-debounce"; diff --git a/packages/cli/src/commands/tablegen.ts b/packages/cli/src/commands/tablegen.ts index d230494f5d..5bca2b0f3b 100644 --- a/packages/cli/src/commands/tablegen.ts +++ b/packages/cli/src/commands/tablegen.ts @@ -1,7 +1,8 @@ import path from "path"; import type { CommandModule } from "yargs"; import { loadConfig } from "@latticexyz/config/node"; -import { StoreConfig, tablegen } from "@latticexyz/store"; +import { StoreConfig } from "@latticexyz/store"; +import { tablegen } from "@latticexyz/store/codegen"; import { getSrcDirectory } from "@latticexyz/common/foundry"; type Options = { diff --git a/packages/cli/src/render-ts/recsV1TableOptions.ts b/packages/cli/src/render-ts/recsV1TableOptions.ts index e19739d806..c7bcbd184b 100644 --- a/packages/cli/src/render-ts/recsV1TableOptions.ts +++ b/packages/cli/src/render-ts/recsV1TableOptions.ts @@ -1,4 +1,5 @@ -import { StoreConfig, resolveAbiOrUserType } from "@latticexyz/store"; +import { StoreConfig } from "@latticexyz/store"; +import { resolveAbiOrUserType } from "@latticexyz/store/codegen"; import { schemaTypesToRecsTypeStrings } from "./schemaTypesToRecsTypeStrings"; import { RecsV1TableOptions } from "./types"; diff --git a/packages/cli/src/utils/deploy.ts b/packages/cli/src/utils/deploy.ts index e1ea413eef..2ba43ec3fe 100644 --- a/packages/cli/src/utils/deploy.ts +++ b/packages/cli/src/utils/deploy.ts @@ -8,7 +8,8 @@ import { getOutDirectory, getScriptDirectory, cast, forge } from "@latticexyz/co import { resolveWithContext } from "@latticexyz/config"; import { MUDError } from "@latticexyz/common/errors"; import { encodeSchema } from "@latticexyz/schema-type"; -import { StoreConfig, resolveAbiOrUserType } from "@latticexyz/store"; +import { StoreConfig } from "@latticexyz/store"; +import { resolveAbiOrUserType } from "@latticexyz/store/codegen"; import { WorldConfig, resolveWorldConfig } from "@latticexyz/world"; import { IBaseWorld } from "@latticexyz/world/types/ethers-contracts/IBaseWorld"; diff --git a/packages/store/package.json b/packages/store/package.json index 6db0505c5c..bbf4b22da0 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -10,22 +10,26 @@ "license": "MIT", "type": "module", "exports": { - ".": "./dist/ts/library/index.js", + ".": "./dist/ts/index.js", + "./codegen": "./dist/ts/codegen/index.js", + "./config": "./dist/ts/config/index.js", "./register": "./dist/ts/register/index.js", "./abi/*": "./abi/*", - "./config": "./dist/ts/library/config/index.js", "./*": "./dist/*" }, "typesVersions": { "*": { "index": [ - "./ts/library/index.ts" + "./ts/index.ts" ], - "register": [ - "./ts/register/index.ts" + "codegen": [ + "./ts/codegen/index.ts" ], "config": [ - "./ts/library/config/index.ts" + "./ts/config/index.ts" + ], + "register": [ + "./ts/register/index.ts" ] } }, diff --git a/packages/store/ts/library/render-solidity/ephemeral.ts b/packages/store/ts/codegen/ephemeral.ts similarity index 100% rename from packages/store/ts/library/render-solidity/ephemeral.ts rename to packages/store/ts/codegen/ephemeral.ts diff --git a/packages/store/ts/library/render-solidity/field.ts b/packages/store/ts/codegen/field.ts similarity index 100% rename from packages/store/ts/library/render-solidity/field.ts rename to packages/store/ts/codegen/field.ts diff --git a/packages/store/ts/library/render-solidity/index.ts b/packages/store/ts/codegen/index.ts similarity index 100% rename from packages/store/ts/library/render-solidity/index.ts rename to packages/store/ts/codegen/index.ts diff --git a/packages/store/ts/library/render-solidity/record.ts b/packages/store/ts/codegen/record.ts similarity index 100% rename from packages/store/ts/library/render-solidity/record.ts rename to packages/store/ts/codegen/record.ts diff --git a/packages/store/ts/library/render-solidity/renderTable.ts b/packages/store/ts/codegen/renderTable.ts similarity index 100% rename from packages/store/ts/library/render-solidity/renderTable.ts rename to packages/store/ts/codegen/renderTable.ts diff --git a/packages/store/ts/library/render-solidity/renderTableIndex.ts b/packages/store/ts/codegen/renderTableIndex.ts similarity index 100% rename from packages/store/ts/library/render-solidity/renderTableIndex.ts rename to packages/store/ts/codegen/renderTableIndex.ts diff --git a/packages/store/ts/library/render-solidity/renderTypesFromConfig.ts b/packages/store/ts/codegen/renderTypesFromConfig.ts similarity index 100% rename from packages/store/ts/library/render-solidity/renderTypesFromConfig.ts rename to packages/store/ts/codegen/renderTypesFromConfig.ts diff --git a/packages/store/ts/library/render-solidity/tableOptions.ts b/packages/store/ts/codegen/tableOptions.ts similarity index 100% rename from packages/store/ts/library/render-solidity/tableOptions.ts rename to packages/store/ts/codegen/tableOptions.ts diff --git a/packages/store/ts/library/render-solidity/tablegen.ts b/packages/store/ts/codegen/tablegen.ts similarity index 100% rename from packages/store/ts/library/render-solidity/tablegen.ts rename to packages/store/ts/codegen/tablegen.ts diff --git a/packages/store/ts/library/render-solidity/types.ts b/packages/store/ts/codegen/types.ts similarity index 100% rename from packages/store/ts/library/render-solidity/types.ts rename to packages/store/ts/codegen/types.ts diff --git a/packages/store/ts/library/render-solidity/userType.ts b/packages/store/ts/codegen/userType.ts similarity index 100% rename from packages/store/ts/library/render-solidity/userType.ts rename to packages/store/ts/codegen/userType.ts diff --git a/packages/store/ts/library/config/defaults.ts b/packages/store/ts/config/defaults.ts similarity index 100% rename from packages/store/ts/library/config/defaults.ts rename to packages/store/ts/config/defaults.ts diff --git a/packages/store/ts/library/config/index.ts b/packages/store/ts/config/index.ts similarity index 100% rename from packages/store/ts/library/config/index.ts rename to packages/store/ts/config/index.ts diff --git a/packages/store/ts/library/config/storeConfig.test-d.ts b/packages/store/ts/config/storeConfig.test-d.ts similarity index 100% rename from packages/store/ts/library/config/storeConfig.test-d.ts rename to packages/store/ts/config/storeConfig.test-d.ts diff --git a/packages/store/ts/library/config/storeConfig.ts b/packages/store/ts/config/storeConfig.ts similarity index 100% rename from packages/store/ts/library/config/storeConfig.ts rename to packages/store/ts/config/storeConfig.ts diff --git a/packages/store/ts/library/index.ts b/packages/store/ts/index.ts similarity index 86% rename from packages/store/ts/library/index.ts rename to packages/store/ts/index.ts index 4387851ef6..d3a12ccc83 100644 --- a/packages/store/ts/library/index.ts +++ b/packages/store/ts/index.ts @@ -1,7 +1,6 @@ // Importing library has no side-effects, unlike register // (library neither creates nor extends MUDCoreContext when imported) export * from "./config"; -export * from "./render-solidity"; export * from "./storeEvents"; export * from "./storeEventsAbi"; diff --git a/packages/store/ts/register/configExtensions.ts b/packages/store/ts/register/configExtensions.ts index 64fd11bac1..3daf8b35ab 100644 --- a/packages/store/ts/register/configExtensions.ts +++ b/packages/store/ts/register/configExtensions.ts @@ -1,6 +1,6 @@ import { extendMUDCoreConfig, fromZodErrorCustom } from "@latticexyz/config"; import { ZodError } from "zod"; -import { zPluginStoreConfig } from "../library/config"; +import { zPluginStoreConfig } from "../config"; extendMUDCoreConfig((config) => { // This function gets called within mudConfig. diff --git a/packages/store/ts/register/mudConfig.ts b/packages/store/ts/register/mudConfig.ts index 0077a7f942..afc08f9dd5 100644 --- a/packages/store/ts/register/mudConfig.ts +++ b/packages/store/ts/register/mudConfig.ts @@ -1,6 +1,6 @@ import { mudCoreConfig, MUDCoreUserConfig } from "@latticexyz/config"; import { ExtractUserTypes, StringForUnion } from "@latticexyz/common/type-utils"; -import { MUDUserConfig } from "../library"; +import { MUDUserConfig } from ".."; import { ExpandMUDUserConfig } from "./typeExtensions"; /** mudCoreConfig wrapper to use generics in some options for better type inference */ diff --git a/packages/store/ts/register/typeExtensions.ts b/packages/store/ts/register/typeExtensions.ts index 815103d2aa..88dd09dee4 100644 --- a/packages/store/ts/register/typeExtensions.ts +++ b/packages/store/ts/register/typeExtensions.ts @@ -1,7 +1,7 @@ import { OrDefaults } from "@latticexyz/common/type-utils"; import { MUDCoreUserConfig } from "@latticexyz/config"; -import { ExpandTablesConfig, StoreConfig, StoreUserConfig } from "../library/config"; -import { DEFAULTS, PATH_DEFAULTS } from "../library/config/defaults"; +import { ExpandTablesConfig, StoreConfig, StoreUserConfig } from "../config"; +import { DEFAULTS, PATH_DEFAULTS } from "../config/defaults"; // Inject non-generic options into the core config. // Re-exporting an interface of an existing module merges them, adding new options to the interface. diff --git a/packages/store/ts/scripts/tablegen.ts b/packages/store/ts/scripts/tablegen.ts index d5dd1a4302..57e6efc9de 100644 --- a/packages/store/ts/scripts/tablegen.ts +++ b/packages/store/ts/scripts/tablegen.ts @@ -1,7 +1,8 @@ import path from "path"; import { loadConfig } from "@latticexyz/config/node"; import { getSrcDirectory } from "@latticexyz/common/foundry"; -import { tablegen, StoreConfig } from "../library"; +import { tablegen } from "../codegen"; +import { StoreConfig } from ".."; const config = (await loadConfig()) as StoreConfig; const srcDir = await getSrcDirectory(); diff --git a/packages/store/ts/library/storeEvents.ts b/packages/store/ts/storeEvents.ts similarity index 100% rename from packages/store/ts/library/storeEvents.ts rename to packages/store/ts/storeEvents.ts diff --git a/packages/store/ts/library/storeEventsAbi.test.ts b/packages/store/ts/storeEventsAbi.test.ts similarity index 100% rename from packages/store/ts/library/storeEventsAbi.test.ts rename to packages/store/ts/storeEventsAbi.test.ts diff --git a/packages/store/ts/library/storeEventsAbi.ts b/packages/store/ts/storeEventsAbi.ts similarity index 100% rename from packages/store/ts/library/storeEventsAbi.ts rename to packages/store/ts/storeEventsAbi.ts diff --git a/packages/store/tsup.config.ts b/packages/store/tsup.config.ts index 4cc5802af0..e46c0dc59d 100644 --- a/packages/store/tsup.config.ts +++ b/packages/store/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["mud.config.ts", "ts/library/index.ts", "ts/register/index.ts", "ts/library/config/index.ts"], + entry: ["mud.config.ts", "ts/index.ts", "ts/codegen/index.ts", "ts/config/index.ts", "ts/register/index.ts"], target: "esnext", format: ["esm"], dts: false, diff --git a/packages/world/ts/plugins/snapsync/configExtensions.ts b/packages/world/ts/plugins/snapsync/configExtensions.ts index 63f8e0094f..311e9d54e2 100644 --- a/packages/world/ts/plugins/snapsync/configExtensions.ts +++ b/packages/world/ts/plugins/snapsync/configExtensions.ts @@ -1,5 +1,5 @@ import { extendMUDCoreConfig, resolveTableId } from "@latticexyz/config"; -import { zPluginStoreConfig } from "@latticexyz/store/config"; +import { zPluginStoreConfig } from "@latticexyz/store/ts/config"; import { zSnapSyncPluginConfig } from "./plugin"; import { zPluginWorldConfig } from "../../library"; diff --git a/packages/world/ts/scripts/tablegen.ts b/packages/world/ts/scripts/tablegen.ts index f236eb7831..9c77691154 100644 --- a/packages/world/ts/scripts/tablegen.ts +++ b/packages/world/ts/scripts/tablegen.ts @@ -1,7 +1,8 @@ import path from "path"; import { loadConfig } from "@latticexyz/config/node"; import { getSrcDirectory } from "@latticexyz/common/foundry"; -import { StoreConfig, tablegen } from "@latticexyz/store"; +import { StoreConfig } from "@latticexyz/store"; +import { tablegen } from "@latticexyz/store/codegen"; const config = (await loadConfig()) as StoreConfig; const srcDir = await getSrcDirectory(); From 926bf0a77ff82cce8d1e493004f4aab2c5c67c37 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 14 Jun 2023 21:48:51 -0700 Subject: [PATCH 03/20] use events from store package --- .../client-react/src/mud/setupViemNetwork.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts index 7b9d247e22..b7e8852114 100644 --- a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts +++ b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts @@ -1,17 +1,7 @@ import { getNetworkConfig } from "./getNetworkConfig"; import { createBlockEventsStream } from "@latticexyz/block-events-stream"; -// import { storeEventsAbi } from "@latticexyz/store"; +import { storeEventsAbi } from "@latticexyz/store"; import { createPublicClient, http, fallback, webSocket } from "viem"; -import { parseAbi } from "viem/abi"; - -// TODO: import from store once we move aside the solidity codegen stuff that breaks browser bundles -export const storeEvents = [ - "event StoreDeleteRecord(bytes32 table, bytes32[] key)", - "event StoreSetField(bytes32 table, bytes32[] key, uint8 schemaIndex, bytes data)", - "event StoreSetRecord(bytes32 table, bytes32[] key, bytes data)", - "event StoreEphemeralRecord(bytes32 table, bytes32[] key, bytes data)", -] as const; -export const storeEventsAbi = parseAbi(storeEvents); export async function setupViemNetwork() { const { chain } = await getNetworkConfig(); From e976b03770fec9884ca01a9213c1fe80ecafd393 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 14 Jun 2023 22:12:15 -0700 Subject: [PATCH 04/20] fix import --- packages/world/ts/plugins/snapsync/configExtensions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/world/ts/plugins/snapsync/configExtensions.ts b/packages/world/ts/plugins/snapsync/configExtensions.ts index 311e9d54e2..58234eac3b 100644 --- a/packages/world/ts/plugins/snapsync/configExtensions.ts +++ b/packages/world/ts/plugins/snapsync/configExtensions.ts @@ -1,5 +1,5 @@ import { extendMUDCoreConfig, resolveTableId } from "@latticexyz/config"; -import { zPluginStoreConfig } from "@latticexyz/store/ts/config"; +import { zPluginStoreConfig } from "@latticexyz/store"; import { zSnapSyncPluginConfig } from "./plugin"; import { zPluginWorldConfig } from "../../library"; From 3683898b335786201ed3befe88f80d0398884987 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Fri, 16 Jun 2023 16:23:11 -0700 Subject: [PATCH 05/20] wip --- .../client-react/src/mud/setupViemNetwork.ts | 7 +- packages/block-events-stream/package.json | 2 + .../src/createBlockEventsStream.ts | 95 ++++++++++++++----- .../src/createBlockStream.ts | 33 +++++++ packages/block-events-stream/src/debug.ts | 3 + ...cludePendingLogs.ts => isNonPendingLog.ts} | 17 ++-- pnpm-lock.yaml | 6 ++ 7 files changed, 129 insertions(+), 34 deletions(-) create mode 100644 packages/block-events-stream/src/createBlockStream.ts create mode 100644 packages/block-events-stream/src/debug.ts rename packages/block-events-stream/src/{excludePendingLogs.ts => isNonPendingLog.ts} (51%) diff --git a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts index b7e8852114..d20bd4541f 100644 --- a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts +++ b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts @@ -9,7 +9,12 @@ export async function setupViemNetwork() { const publicClient = createPublicClient({ chain, - transport: fallback([webSocket(), http()]), + // TODO: use fallback with websocket first once encoding issues are fixed + // https://github.com/wagmi-dev/viem/issues/725 + // transport: fallback([webSocket(), http()]), + transport: http(), + // TODO: do this per chain? maybe in the MUDChain config? + pollingInterval: 1000, }); const stream = await createBlockEventsStream({ diff --git a/packages/block-events-stream/package.json b/packages/block-events-stream/package.json index 113701164e..f21de69bf0 100644 --- a/packages/block-events-stream/package.json +++ b/packages/block-events-stream/package.json @@ -28,10 +28,12 @@ "@latticexyz/schema-type": "workspace:*", "@latticexyz/store": "workspace:*", "abitype": "0.8.7", + "debug": "^4.3.4", "rxjs": "^7.5.5", "viem": "1.0.6" }, "devDependencies": { + "@types/debug": "^4.1.7", "tsup": "^6.7.0", "vitest": "0.31.4" }, diff --git a/packages/block-events-stream/src/createBlockEventsStream.ts b/packages/block-events-stream/src/createBlockEventsStream.ts index 00128a0a5e..b8b09e4c20 100644 --- a/packages/block-events-stream/src/createBlockEventsStream.ts +++ b/packages/block-events-stream/src/createBlockEventsStream.ts @@ -3,7 +3,8 @@ import type { Hex, PublicClient } from "viem"; import type { AbiEvent } from "abitype"; import { BlockEvents, BlockEventsStream, BlockNumberOrTag } from "./common"; import { bigIntMin } from "./utils"; -import { excludePendingLogs } from "./excludePendingLogs"; +import { isNonPendingLog } from "./isNonPendingLog"; +import { debug } from "./debug"; // TODO: add nice logging with debub lib or similar // TODO: make `toBlock` accept a `BehaviorSubject` or add `latestBlockStream` so we only need one listener/watcher/poller @@ -12,7 +13,7 @@ import { excludePendingLogs } from "./excludePendingLogs"; export type CreateBlockEventsStreamOptions = { publicClient: PublicClient; fromBlock?: BlockNumberOrTag; // defaults to "earliest" - toBlock?: BlockNumberOrTag; // defaults to "latest" + toBlock?: Exclude; // defaults to "latest" address?: Hex; events: readonly TAbiEvent[]; maxBlockRange?: number; // defaults to 1000 @@ -26,6 +27,9 @@ export async function createBlockEventsStream({ events, maxBlockRange = 1000, }: CreateBlockEventsStreamOptions): Promise> { + debug("createBlockEventsStream", { initialFromBlock, initialToBlock, address, events, maxBlockRange }); + + debug("Getting first/last blocks"); const [firstBlock, lastBlock] = await Promise.all([ publicClient.getBlock( typeof initialFromBlock === "bigint" ? { blockNumber: initialFromBlock } : { blockTag: initialFromBlock } @@ -44,52 +48,95 @@ export async function createBlockEventsStream({ throw new Error(`pending or missing toBlock "${initialToBlock}"`); } + debug("Got first/last blocks", { firstBlock, lastBlock }); + const stream = new Subject>(); fetchBlockRange(firstBlock.number, maxBlockRange, lastBlock.number); async function fetchBlockRange(fromBlock: bigint, maxBlockRange: number, lastBlockNumber: bigint): Promise { try { const toBlock = bigIntMin(fromBlock + BigInt(maxBlockRange), lastBlockNumber); + debug("fetching block range", { fromBlock, toBlock }); // TODO: convert to one `getLogs` call when viem supports multiple events or topics - // TODO: do something other than just throwing out pending logs - const logs = excludePendingLogs( - ( - await Promise.all( - events.map((event) => - publicClient.getLogs({ - address, - event, - fromBlock, - toBlock, - }) - ) + const logs = ( + await Promise.all( + events.map((event) => + publicClient.getLogs({ + address, + event, + fromBlock, + toBlock, + }) ) - ).flat() - ); + ) + ).flat(); + + // TODO: do something other than just throwing out pending logs + const nonPendingLogs = logs.filter(isNonPendingLog); + + if (logs.length !== nonPendingLogs.length) { + // TODO: better error + console.warn("pending logs discarded"); + } // TODO: handle RPC block range errors // TODO: handle RPC rate limit errors (hopefully via client retry policy) - const blockNumbers = Array.from(new Set(logs.map((log) => log.blockNumber))); + const blockNumbers = Array.from(new Set(nonPendingLogs.map((log) => log.blockNumber))); blockNumbers.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); for (const blockNumber of blockNumbers) { - const blockLogs = logs.filter((log) => log.blockNumber === blockNumber); + const blockLogs = nonPendingLogs.filter((log) => log.blockNumber === blockNumber); blockLogs.sort((a, b) => (a.logIndex < b.logIndex ? -1 : a.logIndex > b.logIndex ? 1 : 0)); - stream.next({ - blockNumber, - blockHash: blockLogs[0].blockHash, - events: blockLogs, - }); + if (blockLogs.length) { + debug("emitting events for block", { blockNumber, blockHash: blockLogs[0].blockHash, events: blockLogs }); + stream.next({ + blockNumber, + blockHash: blockLogs[0].blockHash, + events: blockLogs, + }); + } } - if (toBlock > lastBlockNumber) { + if (toBlock < lastBlockNumber) { fetchBlockRange(toBlock + 1n, maxBlockRange, lastBlockNumber); return; } + if (typeof initialToBlock !== "bigint") { + debug("updating last block", { initialToBlock }); + const lastBlock = await publicClient.getBlock({ blockTag: initialToBlock }); + if (lastBlock.number == null) { + // TODO: better error + throw new Error(`pending or missing toBlock "${initialToBlock}"`); + } + + debug("got last block", { lastBlock }); + if (lastBlock.number > toBlock) { + fetchBlockRange(toBlock + 1n, maxBlockRange, lastBlock.number); + return; + } + + debug("waiting for next block"); + const unwatch = publicClient.watchBlocks({ + blockTag: initialToBlock, + onBlock: (block) => { + debug("got next block", block); + + if (block.number == null) { + // TODO: better error + throw new Error(`pending or missing toBlock "${initialToBlock}"`); + } + + unwatch(); + fetchBlockRange(toBlock + 1n, maxBlockRange, block.number); + }, + }); + return; + } + // TODO: determine if we can/should start adjusting `lastBlock` (based on if we got a block tag for `initialToBlock`) stream.complete(); } catch (error: unknown) { diff --git a/packages/block-events-stream/src/createBlockStream.ts b/packages/block-events-stream/src/createBlockStream.ts new file mode 100644 index 0000000000..8f6a0aed87 --- /dev/null +++ b/packages/block-events-stream/src/createBlockStream.ts @@ -0,0 +1,33 @@ +import { BehaviorSubject } from "rxjs"; +import type { Block, BlockTag, PublicClient } from "viem"; + +// TODO: pass through viem's types, e.g. WatchBlocksParameters -> GetBlockReturnType + +export type CreateBlockStreamOptions = { + publicClient: PublicClient; + blockTag: BlockTag; +}; + +export function createBlockStream({ + publicClient, + blockTag, +}: CreateBlockStreamOptions): Promise> { + return new Promise((resolve, reject) => { + let stream: BehaviorSubject | undefined; + const unwatch = publicClient.watchBlocks({ + blockTag, + emitOnBegin: true, + onBlock: (block) => { + if (!stream) { + stream = new BehaviorSubject(block); + resolve(stream); + } else { + stream.next(block); + } + }, + onError: reject, + }); + // TODO: do something with `unwatch`? + // TODO: return readonly BehaviorSubject, something like an observable but with a current value + }); +} diff --git a/packages/block-events-stream/src/debug.ts b/packages/block-events-stream/src/debug.ts new file mode 100644 index 0000000000..b6536cc8aa --- /dev/null +++ b/packages/block-events-stream/src/debug.ts @@ -0,0 +1,3 @@ +import createDebug from "debug"; + +export const debug = createDebug("mud:block-events-stream"); diff --git a/packages/block-events-stream/src/excludePendingLogs.ts b/packages/block-events-stream/src/isNonPendingLog.ts similarity index 51% rename from packages/block-events-stream/src/excludePendingLogs.ts rename to packages/block-events-stream/src/isNonPendingLog.ts index 93700ef638..a936767161 100644 --- a/packages/block-events-stream/src/excludePendingLogs.ts +++ b/packages/block-events-stream/src/isNonPendingLog.ts @@ -8,13 +8,12 @@ type NonPendingLog = TLog & { transactionIndex: NonNullable; }; -export function excludePendingLogs(logs: TLog[]): NonPendingLog[] { - return logs.filter( - (log) => - log.blockHash != null && - log.blockNumber != null && - log.logIndex != null && - log.transactionHash != null && - log.transactionIndex != null - ) as NonPendingLog[]; +export function isNonPendingLog(log: TLog): log is NonPendingLog { + return ( + log.blockHash != null && + log.blockNumber != null && + log.logIndex != null && + log.transactionHash != null && + log.transactionIndex != null + ); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 16cc2965dc..0065cefbea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -74,6 +74,9 @@ importers: abitype: specifier: 0.8.7 version: 0.8.7(typescript@5.0.4) + debug: + specifier: ^4.3.4 + version: 4.3.4(supports-color@8.1.1) rxjs: specifier: ^7.5.5 version: 7.5.6 @@ -81,6 +84,9 @@ importers: specifier: 1.0.6 version: 1.0.6(typescript@5.0.4) devDependencies: + '@types/debug': + specifier: ^4.1.7 + version: 4.1.7 tsup: specifier: ^6.7.0 version: 6.7.0(postcss@8.4.23)(typescript@5.0.4) From 32c0398342e108b86901cc0ff7ad95c3382e2e33 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Fri, 16 Jun 2023 17:54:57 -0700 Subject: [PATCH 06/20] use streams for last block --- .../packages/client-phaser/package.json | 2 +- .../packages/client-react/package.json | 3 +- .../client-react/src/mud/setupViemNetwork.ts | 33 +++++-- .../packages/client-vanilla/package.json | 2 +- examples/minimal/pnpm-lock.yaml | 9 +- .../packages/client-vanilla/package.json | 2 +- packages/block-events-stream/package.json | 2 +- packages/block-events-stream/src/common.ts | 9 +- .../src/createBlockEventsStream.ts | 86 ++++++++----------- .../src/createBlockNumberStream.ts | 47 ++++++++++ .../src/createBlockStream.ts | 15 ++-- packages/block-events-stream/src/index.ts | 3 + .../src/isNonPendingBlock.ts | 12 +++ .../src/isNonPendingLog.ts | 2 +- packages/dev-tools/package.json | 2 +- packages/network/package.json | 2 +- packages/phaserx/package.json | 2 +- packages/react/package.json | 2 +- packages/recs/package.json | 2 +- packages/std-client/package.json | 2 +- packages/utils/package.json | 2 +- pnpm-lock.yaml | 47 +++++----- 22 files changed, 177 insertions(+), 111 deletions(-) create mode 100644 packages/block-events-stream/src/createBlockNumberStream.ts create mode 100644 packages/block-events-stream/src/isNonPendingBlock.ts diff --git a/examples/minimal/packages/client-phaser/package.json b/examples/minimal/packages/client-phaser/package.json index 1c9523fc7c..11df104c65 100644 --- a/examples/minimal/packages/client-phaser/package.json +++ b/examples/minimal/packages/client-phaser/package.json @@ -34,7 +34,7 @@ "proxy-deep": "^3.1.1", "react": "^18.2.0", "react-dom": "^18.2.0", - "rxjs": "^7.5.5", + "rxjs": "7.5.5", "simplex-noise": "^4.0.1", "styled-components": "^5.3.10", "threads": "^1.7.0", diff --git a/examples/minimal/packages/client-react/package.json b/examples/minimal/packages/client-react/package.json index 1e6420c293..91d327c3b6 100644 --- a/examples/minimal/packages/client-react/package.json +++ b/examples/minimal/packages/client-react/package.json @@ -22,6 +22,7 @@ "@latticexyz/services": "link:../../../../packages/services", "@latticexyz/std-client": "link:../../../../packages/std-client", "@latticexyz/store": "link:../../../../packages/store", + "@latticexyz/store-cache": "link:../../../../packages/store-cache", "@latticexyz/utils": "link:../../../../packages/utils", "@latticexyz/world": "link:../../../../packages/world", "@wagmi/chains": "^0.2.22", @@ -35,7 +36,7 @@ "proxy-deep": "^3.1.1", "react": "^18.2.0", "react-dom": "^18.2.0", - "rxjs": "^7.5.5", + "rxjs": "7.5.5", "threads": "^1.7.0", "viem": "1.0.6" }, diff --git a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts index d20bd4541f..6fc384994e 100644 --- a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts +++ b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts @@ -1,7 +1,9 @@ -import { getNetworkConfig } from "./getNetworkConfig"; -import { createBlockEventsStream } from "@latticexyz/block-events-stream"; -import { storeEventsAbi } from "@latticexyz/store"; import { createPublicClient, http, fallback, webSocket } from "viem"; +import { createBlockEventsStream, createBlockNumberStream, createBlockStream } from "@latticexyz/block-events-stream"; +import { storeEventsAbi } from "@latticexyz/store"; +import { createDatabase, createDatabaseClient } from "@latticexyz/store-cache"; +import { getNetworkConfig } from "./getNetworkConfig"; +import mudConfig from "contracts/mud.config"; export async function setupViemNetwork() { const { chain } = await getNetworkConfig(); @@ -17,14 +19,31 @@ export async function setupViemNetwork() { pollingInterval: 1000, }); - const stream = await createBlockEventsStream({ + // Optional but recommended to avoid multiple instances of polling for blocks + const latestBlock$ = await createBlockStream({ publicClient, blockTag: "latest" }); + const latestBlockNumber$ = await createBlockNumberStream({ block$: latestBlock$ }); + + const blockEvents$ = await createBlockEventsStream({ publicClient, events: storeEventsAbi, + toBlock: latestBlockNumber$, }); - console.log("stream established"); + const db = createDatabase(); + const storeCache = createDatabaseClient(db, mudConfig); - stream.subscribe((block) => { - console.log("stream block", block); + blockEvents$.subscribe((block) => { + // TODO: iterate through events + // TODO: parse event data + // TODO: assemble and store schemas in store-cache + // TODO: store records in store-cache }); + + return { + publicClient, + storeCache, + latestBlock$, + latestBlockNumber$, + blockEvents$, + }; } diff --git a/examples/minimal/packages/client-vanilla/package.json b/examples/minimal/packages/client-vanilla/package.json index 09cb8ab295..05aebaef8b 100644 --- a/examples/minimal/packages/client-vanilla/package.json +++ b/examples/minimal/packages/client-vanilla/package.json @@ -31,7 +31,7 @@ "nice-grpc-web": "^2.0.1", "proxy-deep": "^3.1.1", "react": "^18.2.0", - "rxjs": "^7.5.5", + "rxjs": "7.5.5", "threads": "^1.7.0", "viem": "1.0.6" }, diff --git a/examples/minimal/pnpm-lock.yaml b/examples/minimal/pnpm-lock.yaml index cb0a482ce2..9000356d82 100644 --- a/examples/minimal/pnpm-lock.yaml +++ b/examples/minimal/pnpm-lock.yaml @@ -99,7 +99,7 @@ importers: specifier: ^18.2.0 version: 18.2.0(react@18.2.0) rxjs: - specifier: ^7.5.5 + specifier: 7.5.5 version: 7.5.5 simplex-noise: specifier: ^4.0.1 @@ -186,6 +186,9 @@ importers: '@latticexyz/store': specifier: link:../../../../packages/store version: link:../../../../packages/store + '@latticexyz/store-cache': + specifier: link:../../../../packages/store-cache + version: link:../../../../packages/store-cache '@latticexyz/utils': specifier: link:../../../../packages/utils version: link:../../../../packages/utils @@ -226,7 +229,7 @@ importers: specifier: ^18.2.0 version: 18.2.0(react@18.2.0) rxjs: - specifier: ^7.5.5 + specifier: 7.5.5 version: 7.5.5 threads: specifier: ^1.7.0 @@ -323,7 +326,7 @@ importers: specifier: ^18.2.0 version: 18.2.0 rxjs: - specifier: ^7.5.5 + specifier: 7.5.5 version: 7.5.5 threads: specifier: ^1.7.0 diff --git a/integration-sandbox/packages/client-vanilla/package.json b/integration-sandbox/packages/client-vanilla/package.json index 241bf8bbb5..6cb195fdd6 100644 --- a/integration-sandbox/packages/client-vanilla/package.json +++ b/integration-sandbox/packages/client-vanilla/package.json @@ -31,7 +31,7 @@ "nice-grpc-web": "^2.0.1", "proxy-deep": "^3.1.1", "react": "^18.2.0", - "rxjs": "^7.5.5", + "rxjs": "7.5.5", "threads": "^1.7.0", "viem": "1.0.6" }, diff --git a/packages/block-events-stream/package.json b/packages/block-events-stream/package.json index f21de69bf0..42c9293253 100644 --- a/packages/block-events-stream/package.json +++ b/packages/block-events-stream/package.json @@ -29,7 +29,7 @@ "@latticexyz/store": "workspace:*", "abitype": "0.8.7", "debug": "^4.3.4", - "rxjs": "^7.5.5", + "rxjs": "7.5.5", "viem": "1.0.6" }, "devDependencies": { diff --git a/packages/block-events-stream/src/common.ts b/packages/block-events-stream/src/common.ts index 7ad42d267a..cef3b7abad 100644 --- a/packages/block-events-stream/src/common.ts +++ b/packages/block-events-stream/src/common.ts @@ -1,14 +1,13 @@ import { Observable } from "rxjs"; -import type { BlockNumber, BlockTag, GetLogsReturnType, Hex } from "viem"; +import type { BlockNumber, GetLogsReturnType, Hex } from "viem"; import type { AbiEvent } from "abitype"; -// TODO: support pending blocks -export type BlockNumberOrTag = BlockNumber | Exclude; - export type BlockEvents = { - blockNumber: BlockNumber; + blockNumber: BlockNumber; blockHash: Hex; events: GetLogsReturnType; // TODO: refine to be a store event log }; export type BlockEventsStream = Observable>; + +export type ReadonlySubject = Omit; diff --git a/packages/block-events-stream/src/createBlockEventsStream.ts b/packages/block-events-stream/src/createBlockEventsStream.ts index b8b09e4c20..f36800f32a 100644 --- a/packages/block-events-stream/src/createBlockEventsStream.ts +++ b/packages/block-events-stream/src/createBlockEventsStream.ts @@ -1,19 +1,20 @@ -import { Subject } from "rxjs"; -import type { Hex, PublicClient } from "viem"; +import { BehaviorSubject, Subject, lastValueFrom } from "rxjs"; +import type { BlockNumber, Hex, PublicClient } from "viem"; import type { AbiEvent } from "abitype"; -import { BlockEvents, BlockEventsStream, BlockNumberOrTag } from "./common"; +import { BlockEvents, BlockEventsStream, ReadonlySubject } from "./common"; import { bigIntMin } from "./utils"; import { isNonPendingLog } from "./isNonPendingLog"; import { debug } from "./debug"; +import { createBlockNumberStream } from "./createBlockNumberStream"; // TODO: add nice logging with debub lib or similar -// TODO: make `toBlock` accept a `BehaviorSubject` or add `latestBlockStream` so we only need one listener/watcher/poller +// TODO: make `toBlock` accept a `BehaviorSubject` or add `latestBlockStream` so we only need one listener/watcher/poller // TODO: consider excluding `pending` block tags so we can just assume all block numbers are present export type CreateBlockEventsStreamOptions = { publicClient: PublicClient; - fromBlock?: BlockNumberOrTag; // defaults to "earliest" - toBlock?: Exclude; // defaults to "latest" + fromBlock?: BlockNumber; + toBlock?: BlockNumber | ReadonlySubject>; address?: Hex; events: readonly TAbiEvent[]; maxBlockRange?: number; // defaults to 1000 @@ -21,37 +22,36 @@ export type CreateBlockEventsStreamOptions = { export async function createBlockEventsStream({ publicClient, - fromBlock: initialFromBlock = "earliest", - toBlock: initialToBlock = "latest", + fromBlock: initialFromBlock, + toBlock: initialToBlock, address, events, maxBlockRange = 1000, }: CreateBlockEventsStreamOptions): Promise> { debug("createBlockEventsStream", { initialFromBlock, initialToBlock, address, events, maxBlockRange }); - debug("Getting first/last blocks"); - const [firstBlock, lastBlock] = await Promise.all([ - publicClient.getBlock( - typeof initialFromBlock === "bigint" ? { blockNumber: initialFromBlock } : { blockTag: initialFromBlock } - ), - publicClient.getBlock( - typeof initialToBlock === "bigint" ? { blockNumber: initialToBlock } : { blockTag: initialToBlock } - ), - ]); - - if (firstBlock.number == null) { - // TODO: better error - throw new Error(`pending or missing fromBlock "${initialFromBlock}"`); - } - if (lastBlock.number == null) { - // TODO: better error - throw new Error(`pending or missing toBlock "${initialToBlock}"`); + if (initialFromBlock == null) { + debug("getting earliest block"); + const earliestBlock = await publicClient.getBlock({ blockTag: "earliest" }); + debug("earliest block", earliestBlock); + if (earliestBlock.number == null) { + // TODO: better error + throw new Error(`pending or missing earliest block`); + } + initialFromBlock = earliestBlock.number; } - debug("Got first/last blocks", { firstBlock, lastBlock }); + if (initialToBlock == null) { + debug("creating latest block number stream"); + initialToBlock = await createBlockNumberStream({ publicClient, blockTag: "latest" }); + } const stream = new Subject>(); - fetchBlockRange(firstBlock.number, maxBlockRange, lastBlock.number); + fetchBlockRange( + initialFromBlock, + maxBlockRange, + initialToBlock instanceof BehaviorSubject ? initialToBlock.value : initialToBlock + ); async function fetchBlockRange(fromBlock: bigint, maxBlockRange: number, lastBlockNumber: bigint): Promise { try { @@ -105,34 +105,18 @@ export async function createBlockEventsStream({ return; } - if (typeof initialToBlock !== "bigint") { - debug("updating last block", { initialToBlock }); - const lastBlock = await publicClient.getBlock({ blockTag: initialToBlock }); - if (lastBlock.number == null) { - // TODO: better error - throw new Error(`pending or missing toBlock "${initialToBlock}"`); - } - - debug("got last block", { lastBlock }); - if (lastBlock.number > toBlock) { - fetchBlockRange(toBlock + 1n, maxBlockRange, lastBlock.number); + if (initialToBlock instanceof BehaviorSubject) { + if (initialToBlock.value > toBlock) { + fetchBlockRange(toBlock + 1n, maxBlockRange, initialToBlock.value); return; } debug("waiting for next block"); - const unwatch = publicClient.watchBlocks({ - blockTag: initialToBlock, - onBlock: (block) => { - debug("got next block", block); - - if (block.number == null) { - // TODO: better error - throw new Error(`pending or missing toBlock "${initialToBlock}"`); - } - - unwatch(); - fetchBlockRange(toBlock + 1n, maxBlockRange, block.number); - }, + const sub = initialToBlock.subscribe((blockNumber) => { + if (blockNumber > toBlock) { + sub.unsubscribe(); + fetchBlockRange(toBlock + 1n, maxBlockRange, blockNumber); + } }); return; } diff --git a/packages/block-events-stream/src/createBlockNumberStream.ts b/packages/block-events-stream/src/createBlockNumberStream.ts new file mode 100644 index 0000000000..f18f6a526c --- /dev/null +++ b/packages/block-events-stream/src/createBlockNumberStream.ts @@ -0,0 +1,47 @@ +import { BehaviorSubject } from "rxjs"; +import type { Block, BlockNumber, BlockTag, PublicClient } from "viem"; +import { ReadonlySubject } from "./common"; +import { createBlockStream } from "./createBlockStream"; + +// TODO: pass through viem's types, e.g. WatchBlocksParameters -> GetBlockReturnType +// TODO: make stream closeable? + +export type CreateBlockNumberStreamOptions = + | { + publicClient: PublicClient; + blockTag: Omit; + block$?: never; + } + | { + publicClient?: never; + blockTag?: never; + block$: ReadonlySubject>; + }; + +export async function createBlockNumberStream({ + publicClient, + blockTag, + block$: initialBlock$, +}: CreateBlockNumberStreamOptions): Promise>> { + const block$ = initialBlock$ ?? (await createBlockStream({ publicClient, blockTag: blockTag as BlockTag })); + const block = block$.value; + if (!block.number) { + // TODO: better error + throw new Error(`${blockTag} block missing or pending`); + } + + const blockNumber$ = new BehaviorSubject(block.number); + // TODO: do something with unwatch? + const unwatch = block$.subscribe({ + next: (block) => { + if (block.number) { + blockNumber$.next(block.number); + } + // TODO: warn/error on blocks with missing block number? + }, + error: blockNumber$.error, + complete: blockNumber$.complete, + }); + + return blockNumber$; +} diff --git a/packages/block-events-stream/src/createBlockStream.ts b/packages/block-events-stream/src/createBlockStream.ts index 8f6a0aed87..bfcf0e3e79 100644 --- a/packages/block-events-stream/src/createBlockStream.ts +++ b/packages/block-events-stream/src/createBlockStream.ts @@ -1,7 +1,9 @@ import { BehaviorSubject } from "rxjs"; import type { Block, BlockTag, PublicClient } from "viem"; +import { ReadonlySubject } from "./common"; // TODO: pass through viem's types, e.g. WatchBlocksParameters -> GetBlockReturnType +// TODO: make stream closeable? export type CreateBlockStreamOptions = { publicClient: PublicClient; @@ -11,23 +13,26 @@ export type CreateBlockStreamOptions = { export function createBlockStream({ publicClient, blockTag, -}: CreateBlockStreamOptions): Promise> { +}: CreateBlockStreamOptions): Promise>> { return new Promise((resolve, reject) => { let stream: BehaviorSubject | undefined; + // TODO: do something with unwatch? const unwatch = publicClient.watchBlocks({ blockTag, emitOnBegin: true, onBlock: (block) => { if (!stream) { stream = new BehaviorSubject(block); - resolve(stream); + // TODO: return observable with a current value? + resolve(stream as ReadonlySubject>); } else { stream.next(block); } }, - onError: reject, + onError: (error) => { + reject(error); + stream?.error(error); + }, }); - // TODO: do something with `unwatch`? - // TODO: return readonly BehaviorSubject, something like an observable but with a current value }); } diff --git a/packages/block-events-stream/src/index.ts b/packages/block-events-stream/src/index.ts index 62502925c9..c767c01088 100644 --- a/packages/block-events-stream/src/index.ts +++ b/packages/block-events-stream/src/index.ts @@ -1 +1,4 @@ +export * from "./common"; export * from "./createBlockEventsStream"; +export * from "./createBlockNumberStream"; +export * from "./createBlockStream"; diff --git a/packages/block-events-stream/src/isNonPendingBlock.ts b/packages/block-events-stream/src/isNonPendingBlock.ts new file mode 100644 index 0000000000..5545fc23a5 --- /dev/null +++ b/packages/block-events-stream/src/isNonPendingBlock.ts @@ -0,0 +1,12 @@ +import type { Block } from "viem"; + +export type NonPendingBlock = TBlock & { + hash: NonNullable; + logsBloom: NonNullable; + nonce: NonNullable; + number: NonNullable; +}; + +export function isNonPendingBlock(block: TBlock): block is NonPendingBlock { + return block.hash != null && block.logsBloom != null && block.nonce != null && block.number != null; +} diff --git a/packages/block-events-stream/src/isNonPendingLog.ts b/packages/block-events-stream/src/isNonPendingLog.ts index a936767161..6110d18171 100644 --- a/packages/block-events-stream/src/isNonPendingLog.ts +++ b/packages/block-events-stream/src/isNonPendingLog.ts @@ -1,6 +1,6 @@ import type { Log } from "viem"; -type NonPendingLog = TLog & { +export type NonPendingLog = TLog & { blockHash: NonNullable; blockNumber: NonNullable; logIndex: NonNullable; diff --git a/packages/dev-tools/package.json b/packages/dev-tools/package.json index e7c9a7699a..66340e2101 100644 --- a/packages/dev-tools/package.json +++ b/packages/dev-tools/package.json @@ -31,7 +31,7 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.11.0", - "rxjs": "^7.5.5", + "rxjs": "7.5.5", "tailwind-merge": "^1.12.0", "use-local-storage-state": "^18.3.2", "viem": "1.0.6", diff --git a/packages/network/package.json b/packages/network/package.json index 44d8e1c971..9bd6926a0d 100644 --- a/packages/network/package.json +++ b/packages/network/package.json @@ -51,7 +51,7 @@ "lodash": "^4.17.21", "mobx": "^6.7.0", "nice-grpc-web": "^2.0.1", - "rxjs": "^7.5.5", + "rxjs": "7.5.5", "threads": "^1.7.0", "viem": "1.0.6" }, diff --git a/packages/phaserx/package.json b/packages/phaserx/package.json index 24a89e3935..824203516e 100644 --- a/packages/phaserx/package.json +++ b/packages/phaserx/package.json @@ -27,7 +27,7 @@ "@use-gesture/vanilla": "10.2.9", "mobx": "^6.7.0", "phaser": "3.60.0-beta.14", - "rxjs": "^7.5.5" + "rxjs": "7.5.5" }, "devDependencies": { "tsup": "^6.7.0", diff --git a/packages/react/package.json b/packages/react/package.json index 63f83be8ec..366232d1b5 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -29,7 +29,7 @@ "fast-deep-equal": "^3.1.3", "mobx": "^6.7.0", "react": "^18.2.0", - "rxjs": "^7.5.5" + "rxjs": "7.5.5" }, "devDependencies": { "@testing-library/react-hooks": "^8.0.1", diff --git a/packages/recs/package.json b/packages/recs/package.json index b20db6aa1a..a43ae02dd1 100644 --- a/packages/recs/package.json +++ b/packages/recs/package.json @@ -26,7 +26,7 @@ "@latticexyz/schema-type": "workspace:*", "@latticexyz/utils": "workspace:*", "mobx": "^6.7.0", - "rxjs": "^7.5.5" + "rxjs": "7.5.5" }, "devDependencies": { "@types/jest": "^27.4.1", diff --git a/packages/std-client/package.json b/packages/std-client/package.json index 1115978d19..55f508b148 100644 --- a/packages/std-client/package.json +++ b/packages/std-client/package.json @@ -49,7 +49,7 @@ "ethers": "^5.7.2", "mobx": "^6.7.0", "react": "^18.2.0", - "rxjs": "^7.5.5", + "rxjs": "7.5.5", "viem": "1.0.6" }, "devDependencies": { diff --git a/packages/utils/package.json b/packages/utils/package.json index d6b96a43ab..9654f7b060 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -26,7 +26,7 @@ "ethers": "^5.7.2", "mobx": "^6.7.0", "proxy-deep": "^3.1.1", - "rxjs": "^7.5.5" + "rxjs": "7.5.5" }, "devDependencies": { "@types/jest": "^27.4.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0065cefbea..94cbe6a8b3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,8 +78,8 @@ importers: specifier: ^4.3.4 version: 4.3.4(supports-color@8.1.1) rxjs: - specifier: ^7.5.5 - version: 7.5.6 + specifier: 7.5.5 + version: 7.5.5 viem: specifier: 1.0.6 version: 1.0.6(typescript@5.0.4) @@ -332,8 +332,8 @@ importers: specifier: ^6.11.0 version: 6.11.0(react-dom@18.2.0)(react@18.2.0) rxjs: - specifier: ^7.5.5 - version: 7.5.6 + specifier: 7.5.5 + version: 7.5.5 tailwind-merge: specifier: ^1.12.0 version: 1.12.0 @@ -469,8 +469,8 @@ importers: specifier: ^2.0.1 version: 2.0.1(google-protobuf@3.21.2) rxjs: - specifier: ^7.5.5 - version: 7.5.6 + specifier: 7.5.5 + version: 7.5.5 threads: specifier: ^1.7.0 version: 1.7.0 @@ -573,8 +573,8 @@ importers: specifier: 3.60.0-beta.14 version: 3.60.0-beta.14 rxjs: - specifier: ^7.5.5 - version: 7.5.6 + specifier: 7.5.5 + version: 7.5.5 devDependencies: tsup: specifier: ^6.7.0 @@ -635,8 +635,8 @@ importers: specifier: ^18.2.0 version: 18.2.0 rxjs: - specifier: ^7.5.5 - version: 7.5.6 + specifier: 7.5.5 + version: 7.5.5 devDependencies: '@testing-library/react-hooks': specifier: ^8.0.1 @@ -684,8 +684,8 @@ importers: specifier: ^6.7.0 version: 6.9.0 rxjs: - specifier: ^7.5.5 - version: 7.5.6 + specifier: 7.5.5 + version: 7.5.5 devDependencies: '@types/jest': specifier: ^27.4.1 @@ -866,8 +866,8 @@ importers: specifier: ^18.2.0 version: 18.2.0 rxjs: - specifier: ^7.5.5 - version: 7.5.6 + specifier: 7.5.5 + version: 7.5.5 viem: specifier: 1.0.6 version: 1.0.6(typescript@4.9.5) @@ -1061,8 +1061,8 @@ importers: specifier: ^3.1.1 version: 3.1.1 rxjs: - specifier: ^7.5.5 - version: 7.5.6 + specifier: 7.5.5 + version: 7.5.5 devDependencies: '@types/jest': specifier: ^27.4.1 @@ -8337,7 +8337,7 @@ packages: mute-stream: 0.0.8 ora: 5.4.1 run-async: 2.4.1 - rxjs: 7.8.0 + rxjs: 7.5.5 string-width: 4.2.3 strip-ansi: 6.0.1 through: 2.3.8 @@ -8358,7 +8358,7 @@ packages: mute-stream: 0.0.8 ora: 5.4.1 run-async: 2.4.1 - rxjs: 7.8.0 + rxjs: 7.5.5 string-width: 4.2.3 strip-ansi: 6.0.1 through: 2.3.8 @@ -12741,17 +12741,10 @@ packages: dependencies: tslib: 1.14.1 - /rxjs@7.5.6: - resolution: {integrity: sha512-dnyv2/YsXhnm461G+R/Pe5bWP41Nm6LBXEYWI6eiFP4fiwx6WRI/CD0zbdVAudd9xwLEF2IDcKXLHit0FYjUzw==} + /rxjs@7.5.5: + resolution: {integrity: sha512-sy+H0pQofO95VDmFLzyaw9xNJU4KTRSwQIGM6+iG3SypAtCiLDzpeG8sJrNCWn2Up9km+KhkvTdbkrdy+yzZdw==} dependencies: tslib: 2.5.0 - dev: false - - /rxjs@7.8.0: - resolution: {integrity: sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg==} - dependencies: - tslib: 2.5.0 - dev: true /safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} From 75e5ec3307fbfd01cb00e0230addfb4977badcb7 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 20 Jun 2023 10:15:18 +0100 Subject: [PATCH 07/20] wip --- .../packages/client-react/package.json | 1 + .../packages/client-react/src/mud/setup.ts | 5 +- .../client-react/src/mud/setupViemNetwork.ts | 104 +++++++++++++++++- examples/minimal/pnpm-lock.yaml | 3 + packages/block-events-stream/src/common.ts | 4 +- .../src/createBlockEventsStream.ts | 1 + packages/protocol-parser/src/common.ts | 5 +- packages/protocol-parser/src/hexToSchema.ts | 63 ++++++++++- packages/store-cache/package.json | 1 + pnpm-lock.yaml | 3 + 10 files changed, 178 insertions(+), 12 deletions(-) diff --git a/examples/minimal/packages/client-react/package.json b/examples/minimal/packages/client-react/package.json index 91d327c3b6..81df8b09be 100644 --- a/examples/minimal/packages/client-react/package.json +++ b/examples/minimal/packages/client-react/package.json @@ -16,6 +16,7 @@ "@latticexyz/common": "link:../../../../packages/common", "@latticexyz/dev-tools": "link:../../../../packages/dev-tools", "@latticexyz/network": "link:../../../../packages/network", + "@latticexyz/protocol-parser": "link:../../../../packages/protocol-parser", "@latticexyz/react": "link:../../../../packages/react", "@latticexyz/recs": "link:../../../../packages/recs", "@latticexyz/schema-type": "link:../../../../packages/schema-type", diff --git a/examples/minimal/packages/client-react/src/mud/setup.ts b/examples/minimal/packages/client-react/src/mud/setup.ts index 3783ac9fa9..ff4668bb4f 100644 --- a/examples/minimal/packages/client-react/src/mud/setup.ts +++ b/examples/minimal/packages/client-react/src/mud/setup.ts @@ -6,7 +6,10 @@ import { setupViemNetwork } from "./setupViemNetwork"; export type SetupResult = Awaited>; export async function setup() { - await setupViemNetwork(); + const { storeCache } = await setupViemNetwork(); + + setInterval(() => console.log("inventory", storeCache.tables.Inventory.scan()), 2000); + // return {}; const network = await setupNetwork(); const components = createClientComponents(network); diff --git a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts index 6fc384994e..f794ea9e68 100644 --- a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts +++ b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts @@ -1,10 +1,20 @@ -import { createPublicClient, http, fallback, webSocket } from "viem"; -import { createBlockEventsStream, createBlockNumberStream, createBlockStream } from "@latticexyz/block-events-stream"; +import { createPublicClient, http, fallback, webSocket, Hex, decodeAbiParameters, parseAbiParameters } from "viem"; +import { Schema, TableSchema, hexToTableSchema } from "@latticexyz/protocol-parser"; +import { + BlockEvents, + createBlockEventsStream, + createBlockNumberStream, + createBlockStream, +} from "@latticexyz/block-events-stream"; import { storeEventsAbi } from "@latticexyz/store"; import { createDatabase, createDatabaseClient } from "@latticexyz/store-cache"; +import { TableId } from "@latticexyz/utils"; import { getNetworkConfig } from "./getNetworkConfig"; import mudConfig from "contracts/mud.config"; +export const schemaTableId = new TableId("mudstore", "schema"); +export const metadataTableId = new TableId("mudstore", "StoreMetadata"); + export async function setupViemNetwork() { const { chain } = await getNetworkConfig(); console.log("viem chain", chain); @@ -29,14 +39,96 @@ export async function setupViemNetwork() { toBlock: latestBlockNumber$, }); + // TODO: create a cache per chain/world address + // TODO: check for world deploy block hash, invalidate cache if it changes const db = createDatabase(); const storeCache = createDatabaseClient(db, mudConfig); + // TODO: store these in store cache, load into memory + const tableSchemas: Record = {}; + const tableValueNames: Record = {}; + + // TODO: emit both schema and metadata within the same event, to avoid this complexity + // TODO: move this to protocol-parser or a separate "store events stream" lib/package + function registerSchemas(block: BlockEvents<(typeof storeEventsAbi)[number]>) { + // parse/store all schemas + block.events.forEach((event) => { + if (event.eventName !== "StoreSetRecord") return; + + const { table: tableId, key: keyTuple } = event.args; + if (tableId !== schemaTableId.toHexString()) return; + + const [tableForSchema, ...otherKeys] = keyTuple; + if (otherKeys.length) { + console.warn("registerSchema event is expected to have only one key in key tuple, but got multiple", { + tableId, + keyTuple, + }); + } + + const tableSchema = hexToTableSchema(event.args.data); + tableSchemas[tableForSchema] = tableSchema; + console.log("registered schema", TableId.fromHexString(tableForSchema).toString(), tableSchema); + }); + + const metadataTableSchema = tableSchemas[metadataTableId.toHexString()]; + if (!metadataTableSchema) { + throw new Error("metadata table schema was not registered"); + } + + // parse/store all metadata + block.events.forEach((event) => { + if (event.eventName !== "StoreSetRecord") return; + + const { table: tableId, key: keyTuple } = event.args; + if (tableId !== metadataTableId.toHexString()) return; + + const [tableForSchema, ...otherKeys] = keyTuple; + if (otherKeys.length) { + console.warn("setMetadata event is expected to have only one key in key tuple, but got multiple", { + tableId, + keyTuple, + }); + } + + const [tableName, abiEncodedFieldNames] = metadataTableSchema.valueSchema.decodeData(event.args.data); + const fieldNames = decodeAbiParameters(parseAbiParameters("string[]"), abiEncodedFieldNames as Hex)[0]; + + tableValueNames[tableForSchema] = fieldNames; + console.log("registered metadata", TableId.fromHexString(tableForSchema).toString(), fieldNames); + }); + } + blockEvents$.subscribe((block) => { - // TODO: iterate through events - // TODO: parse event data - // TODO: assemble and store schemas in store-cache - // TODO: store records in store-cache + registerSchemas(block); + + block.events.forEach((event) => { + const { table: tableId, key: keyTuple } = event.args; + const tableSchema = tableSchemas[tableId]; + if (!tableSchema) { + console.warn("no table schema found for event", event); + return; + } + const valueNames = tableValueNames[tableId]; + if (!valueNames) { + console.warn("no table metadata found for event", event); + return; + } + + if (event.eventName === "StoreSetRecord") { + const values = tableSchema.valueSchema.decodeData(event.args.data); + const record = Object.fromEntries(valueNames.map((name, i) => [name, values[i]])); + + const table = TableId.fromHexString(tableId); + storeCache.set(table.namespace, table.name, keyTuple, record); + console.log("stored record", table.toString(), keyTuple, record); + } + + if (event.eventName === "StoreSetField") { + // const values = tableSchema.valueSchema.decodeData(event.args.data); + // const record = Object.fromEntries(valueNames.map((name, i) => [name, values[i]])); + } + }); }); return { diff --git a/examples/minimal/pnpm-lock.yaml b/examples/minimal/pnpm-lock.yaml index 9000356d82..42e7a04c02 100644 --- a/examples/minimal/pnpm-lock.yaml +++ b/examples/minimal/pnpm-lock.yaml @@ -168,6 +168,9 @@ importers: '@latticexyz/network': specifier: link:../../../../packages/network version: link:../../../../packages/network + '@latticexyz/protocol-parser': + specifier: link:../../../../packages/protocol-parser + version: link:../../../../packages/protocol-parser '@latticexyz/react': specifier: link:../../../../packages/react version: link:../../../../packages/react diff --git a/packages/block-events-stream/src/common.ts b/packages/block-events-stream/src/common.ts index cef3b7abad..9a0f3ccff5 100644 --- a/packages/block-events-stream/src/common.ts +++ b/packages/block-events-stream/src/common.ts @@ -2,12 +2,14 @@ import { Observable } from "rxjs"; import type { BlockNumber, GetLogsReturnType, Hex } from "viem"; import type { AbiEvent } from "abitype"; +// TODO: default to store abi events? export type BlockEvents = { blockNumber: BlockNumber; blockHash: Hex; - events: GetLogsReturnType; // TODO: refine to be a store event log + events: GetLogsReturnType; // TODO: refine to be a store event log }; +// TODO: default to store abi events? export type BlockEventsStream = Observable>; export type ReadonlySubject = Omit; diff --git a/packages/block-events-stream/src/createBlockEventsStream.ts b/packages/block-events-stream/src/createBlockEventsStream.ts index f36800f32a..6677e7da49 100644 --- a/packages/block-events-stream/src/createBlockEventsStream.ts +++ b/packages/block-events-stream/src/createBlockEventsStream.ts @@ -67,6 +67,7 @@ export async function createBlockEventsStream({ event, fromBlock, toBlock, + strict: true, }) ) ) diff --git a/packages/protocol-parser/src/common.ts b/packages/protocol-parser/src/common.ts index a28788aced..5e4521e646 100644 --- a/packages/protocol-parser/src/common.ts +++ b/packages/protocol-parser/src/common.ts @@ -1,6 +1,6 @@ import { Hex } from "viem"; -import { DynamicAbiType } from "./dynamicAbiTypes"; -import { StaticAbiType } from "./staticAbiTypes"; +import { DynamicAbiType, DynamicPrimitiveType } from "./dynamicAbiTypes"; +import { StaticAbiType, StaticPrimitiveType } from "./staticAbiTypes"; export type Schema = Readonly<{ staticDataLength: number; @@ -8,6 +8,7 @@ export type Schema = Readonly<{ dynamicFields: DynamicAbiType[]; isEmpty: boolean; schemaData: Hex; + decodeData: (data: Hex) => (StaticPrimitiveType | DynamicPrimitiveType)[]; }>; export type TableSchema = { keySchema: Schema; valueSchema: Schema; isEmpty: boolean; schemaData: Hex }; diff --git a/packages/protocol-parser/src/hexToSchema.ts b/packages/protocol-parser/src/hexToSchema.ts index b58cbdfff6..9b0099d742 100644 --- a/packages/protocol-parser/src/hexToSchema.ts +++ b/packages/protocol-parser/src/hexToSchema.ts @@ -1,9 +1,14 @@ import { Hex, hexToNumber, sliceHex } from "viem"; import { Schema } from "./common"; -import { StaticAbiType, staticAbiTypeToByteLength } from "./staticAbiTypes"; -import { DynamicAbiType } from "./dynamicAbiTypes"; +import { StaticAbiType, StaticPrimitiveType, staticAbiTypeToByteLength } from "./staticAbiTypes"; +import { DynamicAbiType, DynamicPrimitiveType } from "./dynamicAbiTypes"; import { schemaAbiTypes } from "./schemaAbiTypes"; import { InvalidHexLengthForSchemaError, SchemaStaticLengthMismatchError } from "./errors"; +import { decodeStaticField } from "./decodeStaticField"; +import { hexToPackedCounter } from "./hexToPackedCounter"; +import { decodeDynamicField } from "./decodeDynamicField"; + +// TODO: convert schema to class so that we can have static methods for decoding, etc. export function hexToSchema(data: Hex): Schema { if (data.length !== 66) { @@ -31,11 +36,65 @@ export function hexToSchema(data: Hex): Schema { throw new SchemaStaticLengthMismatchError(data, staticDataLength, actualStaticDataLength); } + function decodeData(data: Hex): (StaticPrimitiveType | DynamicPrimitiveType)[] { + const values: (StaticPrimitiveType | DynamicPrimitiveType)[] = []; + + let bytesOffset = 0; + staticFields.forEach((fieldType) => { + const fieldByteLength = staticAbiTypeToByteLength[fieldType]; + const value = decodeStaticField(fieldType, sliceHex(data, bytesOffset, bytesOffset + fieldByteLength)); + bytesOffset += fieldByteLength; + values.push(value); + }); + + // Warn user if static data length doesn't match the schema, because data corruption might be possible. + const actualStaticDataLength = bytesOffset; + if (actualStaticDataLength !== staticDataLength) { + console.warn( + "Decoded static data length does not match schema's expected static data length. Data may get corrupted. Is `getStaticByteLength` outdated?", + { + expectedLength: staticDataLength, + actualLength: actualStaticDataLength, + bytesOffset, + } + ); + } + + if (dynamicFields.length > 0) { + const dataLayout = hexToPackedCounter(sliceHex(data, bytesOffset, bytesOffset + 32)); + bytesOffset += 32; + + dynamicFields.forEach((fieldType, i) => { + const dataLength = dataLayout.fieldByteLengths[i]; + const value = decodeDynamicField(fieldType, sliceHex(data, bytesOffset, bytesOffset + dataLength)); + bytesOffset += dataLength; + values.push(value); + }); + + // Warn user if dynamic data length doesn't match the schema, because data corruption might be possible. + const actualDynamicDataLength = bytesOffset - 32 - actualStaticDataLength; + // TODO: refactor this so we don't break for bytes offsets >UINT40 + if (BigInt(actualDynamicDataLength) !== dataLayout.totalByteLength) { + console.warn( + "Decoded dynamic data length does not match data layout's expected data length. Data may get corrupted. Did the data layout change?", + { + expectedLength: dataLayout.totalByteLength, + actualLength: actualDynamicDataLength, + bytesOffset, + } + ); + } + } + + return values; + } + return { staticDataLength, staticFields, dynamicFields, isEmpty: data === "0x", schemaData: data, + decodeData, }; } diff --git a/packages/store-cache/package.json b/packages/store-cache/package.json index e9e5f3c54e..35a0034287 100644 --- a/packages/store-cache/package.json +++ b/packages/store-cache/package.json @@ -24,6 +24,7 @@ "dependencies": { "@latticexyz/common": "workspace:*", "@latticexyz/config": "workspace:*", + "@latticexyz/protocol-parser": "workspace:*", "@latticexyz/schema-type": "workspace:*", "@latticexyz/store": "workspace:*", "abitype": "0.8.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 94cbe6a8b3..9f22021cb8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1029,6 +1029,9 @@ importers: '@latticexyz/config': specifier: workspace:* version: link:../config + '@latticexyz/protocol-parser': + specifier: workspace:* + version: link:../protocol-parser '@latticexyz/schema-type': specifier: workspace:* version: link:../schema-type From 5a36b5db3e11f2fcace0532d9af608cfd250743c Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 20 Jun 2023 12:33:58 +0100 Subject: [PATCH 08/20] it workssss --- .../packages/client-react/src/mud/setup.ts | 5 +-- .../client-react/src/mud/setupViemNetwork.ts | 36 ++++++++++++------- packages/network/src/v2/ecsEventFromLog.ts | 1 - packages/protocol-parser/src/common.ts | 1 + .../protocol-parser/src/decodeKeyTuple.ts | 2 +- packages/protocol-parser/src/hexToSchema.ts | 7 ++++ 6 files changed, 36 insertions(+), 16 deletions(-) diff --git a/examples/minimal/packages/client-react/src/mud/setup.ts b/examples/minimal/packages/client-react/src/mud/setup.ts index ff4668bb4f..a9ad9862be 100644 --- a/examples/minimal/packages/client-react/src/mud/setup.ts +++ b/examples/minimal/packages/client-react/src/mud/setup.ts @@ -8,8 +8,9 @@ export type SetupResult = Awaited>; export async function setup() { const { storeCache } = await setupViemNetwork(); - setInterval(() => console.log("inventory", storeCache.tables.Inventory.scan()), 2000); - // return {}; + storeCache.tables.Inventory.subscribe((updates) => { + console.log("inventory updates", updates); + }); const network = await setupNetwork(); const components = createClientComponents(network); diff --git a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts index f794ea9e68..f29d544697 100644 --- a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts +++ b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts @@ -1,5 +1,5 @@ import { createPublicClient, http, fallback, webSocket, Hex, decodeAbiParameters, parseAbiParameters } from "viem"; -import { Schema, TableSchema, hexToTableSchema } from "@latticexyz/protocol-parser"; +import { Schema, TableSchema, decodeKeyTuple, hexToTableSchema } from "@latticexyz/protocol-parser"; import { BlockEvents, createBlockEventsStream, @@ -103,30 +103,42 @@ export async function setupViemNetwork() { registerSchemas(block); block.events.forEach((event) => { - const { table: tableId, key: keyTuple } = event.args; - const tableSchema = tableSchemas[tableId]; + const { table: tableIdHex, key: keyTupleHex } = event.args; + const tableSchema = tableSchemas[tableIdHex]; if (!tableSchema) { console.warn("no table schema found for event", event); return; } - const valueNames = tableValueNames[tableId]; + const valueNames = tableValueNames[tableIdHex]; if (!valueNames) { console.warn("no table metadata found for event", event); return; } - - if (event.eventName === "StoreSetRecord") { + const tableId = TableId.fromHexString(tableIdHex); + const keyTupleValues = decodeKeyTuple(tableSchema.keySchema, keyTupleHex); + // TODO: add key names/metadata to registerSchema or setMetadata + const keyTupleNames = Object.getOwnPropertyNames( + mudConfig.tables[tableId.name as keyof typeof mudConfig.tables]?.keySchema ?? {} + ); + const keyTuple = Object.fromEntries(keyTupleValues.map((value, i) => [keyTupleNames[i] ?? i, value])); + + if (event.eventName === "StoreSetRecord" || event.eventName === "StoreEphemeralRecord") { const values = tableSchema.valueSchema.decodeData(event.args.data); const record = Object.fromEntries(valueNames.map((name, i) => [name, values[i]])); - - const table = TableId.fromHexString(tableId); - storeCache.set(table.namespace, table.name, keyTuple, record); - console.log("stored record", table.toString(), keyTuple, record); + storeCache.set(tableId.namespace, tableId.name, keyTuple, record); + console.log("stored record", tableId.toString(), keyTuple, record); } if (event.eventName === "StoreSetField") { - // const values = tableSchema.valueSchema.decodeData(event.args.data); - // const record = Object.fromEntries(valueNames.map((name, i) => [name, values[i]])); + const valueName = valueNames[event.args.schemaIndex]; + const value = tableSchema.valueSchema.decodeField(event.args.schemaIndex, event.args.data); + storeCache.set(tableId.namespace, tableId.name, keyTuple, { [valueName]: value }); + console.log("stored field", tableId.toString(), keyTuple, valueName, "=>", value); + } + + if (event.eventName === "StoreDeleteRecord") { + storeCache.remove(tableId.namespace, tableId.name, keyTuple); + console.log("removed record", tableId.toString(), keyTuple); } }); }); diff --git a/packages/network/src/v2/ecsEventFromLog.ts b/packages/network/src/v2/ecsEventFromLog.ts index 9d7e1d88dc..96d42952f1 100644 --- a/packages/network/src/v2/ecsEventFromLog.ts +++ b/packages/network/src/v2/ecsEventFromLog.ts @@ -108,7 +108,6 @@ export const ecsEventFromLog = async ( } if (name === "StoreSetField") { - console.log("set field"); const { indexedValues, indexedInitialValues, namedValues, namedInitialValues, indexedKey, namedKey } = await decodeStoreSetField(contract, tableId, args.key, args.schemaIndex, args.data); return { diff --git a/packages/protocol-parser/src/common.ts b/packages/protocol-parser/src/common.ts index 5e4521e646..2342982496 100644 --- a/packages/protocol-parser/src/common.ts +++ b/packages/protocol-parser/src/common.ts @@ -9,6 +9,7 @@ export type Schema = Readonly<{ isEmpty: boolean; schemaData: Hex; decodeData: (data: Hex) => (StaticPrimitiveType | DynamicPrimitiveType)[]; + decodeField: (fieldIndex: number, data: Hex) => StaticPrimitiveType | DynamicPrimitiveType; }>; export type TableSchema = { keySchema: Schema; valueSchema: Schema; isEmpty: boolean; schemaData: Hex }; diff --git a/packages/protocol-parser/src/decodeKeyTuple.ts b/packages/protocol-parser/src/decodeKeyTuple.ts index dd9b4a43c5..dd7e20bfb7 100644 --- a/packages/protocol-parser/src/decodeKeyTuple.ts +++ b/packages/protocol-parser/src/decodeKeyTuple.ts @@ -4,7 +4,7 @@ import { StaticPrimitiveType } from "./staticAbiTypes"; // key tuples are encoded in the same way as abi.encode, so we can decode them with viem -export function decodeKeyTuple(keySchema: Schema, keyTuple: Hex[]): StaticPrimitiveType[] { +export function decodeKeyTuple(keySchema: Schema, keyTuple: readonly Hex[]): StaticPrimitiveType[] { return keyTuple.map( (key, index) => decodeAbiParameters([{ type: keySchema.staticFields[index] }], key)[0] as StaticPrimitiveType ); diff --git a/packages/protocol-parser/src/hexToSchema.ts b/packages/protocol-parser/src/hexToSchema.ts index 9b0099d742..f3651ec14d 100644 --- a/packages/protocol-parser/src/hexToSchema.ts +++ b/packages/protocol-parser/src/hexToSchema.ts @@ -89,6 +89,12 @@ export function hexToSchema(data: Hex): Schema { return values; } + function decodeField(fieldIndex: number, data: Hex): StaticPrimitiveType | DynamicPrimitiveType { + return fieldIndex < staticFields.length + ? decodeStaticField(staticFields[fieldIndex], data) + : decodeDynamicField(dynamicFields[fieldIndex - staticFields.length], data); + } + return { staticDataLength, staticFields, @@ -96,5 +102,6 @@ export function hexToSchema(data: Hex): Schema { isEmpty: data === "0x", schemaData: data, decodeData, + decodeField, }; } From ae8f71aeac4873fbdbfbc8115a7896783690722d Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 20 Jun 2023 13:19:48 +0100 Subject: [PATCH 09/20] look ma, no ethers --- .../minimal/packages/client-react/src/App.tsx | 22 +++++-- .../client-react/src/mud/createSystemCalls.ts | 4 +- .../packages/client-react/src/mud/setup.ts | 29 ++++++--- .../client-react/src/mud/setupViemNetwork.ts | 63 ++++++++++++++----- 4 files changed, 85 insertions(+), 33 deletions(-) diff --git a/examples/minimal/packages/client-react/src/App.tsx b/examples/minimal/packages/client-react/src/App.tsx index fa09241b2b..5bf25f7d1b 100644 --- a/examples/minimal/packages/client-react/src/App.tsx +++ b/examples/minimal/packages/client-react/src/App.tsx @@ -1,4 +1,4 @@ -import { useComponentValue, useRows } from "@latticexyz/react"; +import { useRow, useRows } from "@latticexyz/react"; import { useMUD } from "./MUDContext"; import { useEffect, useState } from "react"; @@ -7,15 +7,27 @@ const VARIANTS = ["yellow", "green", "red"]; export const App = () => { const { - components: { CounterTable, MessageTable }, - network: { singletonEntity, worldSend, storeCache }, + storeCache, + // network: { worldSend }, } = useMUD(); - const counter = useComponentValue(CounterTable, singletonEntity); + async function worldSend(...args: any[]) { + // TODO + return { + wait: async () => { + // TODO + }, + }; + } + + const counter = useRow(storeCache, { + table: "CounterTable", + key: { key: "0x000000000000000000000000000000000000000000000000000000000000060d" }, + })?.value; const [myMessage, setMyMessage] = useState(""); const [messages, setMessages] = useState([]); - const message = useComponentValue(MessageTable, singletonEntity); + const message = useRow(storeCache, { table: "MessageTable", key: {} })?.value; const inventory = useRows(storeCache, { table: "Inventory" }); diff --git a/examples/minimal/packages/client-react/src/mud/createSystemCalls.ts b/examples/minimal/packages/client-react/src/mud/createSystemCalls.ts index ef995d575d..9b40c95c50 100644 --- a/examples/minimal/packages/client-react/src/mud/createSystemCalls.ts +++ b/examples/minimal/packages/client-react/src/mud/createSystemCalls.ts @@ -7,12 +7,12 @@ export type SystemCalls = ReturnType; export function createSystemCalls( { worldSend, txReduced$, singletonEntity }: SetupNetworkResult, - { Counter }: ClientComponents + { CounterTable }: ClientComponents ) { const increment = async () => { const tx = await worldSend("increment", []); await awaitStreamValue(txReduced$, (txHash) => txHash === tx.hash); - return getComponentValue(Counter, singletonEntity); + return getComponentValue(CounterTable, singletonEntity); }; return { diff --git a/examples/minimal/packages/client-react/src/mud/setup.ts b/examples/minimal/packages/client-react/src/mud/setup.ts index a9ad9862be..a261bf54f2 100644 --- a/examples/minimal/packages/client-react/src/mud/setup.ts +++ b/examples/minimal/packages/client-react/src/mud/setup.ts @@ -1,23 +1,32 @@ -import { createClientComponents } from "./createClientComponents"; -import { createSystemCalls } from "./createSystemCalls"; -import { setupNetwork } from "./setupNetwork"; +import { Hex, createPublicClient, http } from "viem"; import { setupViemNetwork } from "./setupViemNetwork"; +import { getNetworkConfig } from "./getNetworkConfig"; export type SetupResult = Awaited>; export async function setup() { - const { storeCache } = await setupViemNetwork(); + const { chain, worldAddress } = await getNetworkConfig(); + console.log("viem chain", chain); + + const publicClient = createPublicClient({ + chain, + // TODO: use fallback with websocket first once encoding issues are fixed + // https://github.com/wagmi-dev/viem/issues/725 + // transport: fallback([webSocket(), http()]), + transport: http(), + // TODO: do this per chain? maybe in the MUDChain config? + pollingInterval: 1000, + }); + + const { storeCache } = await setupViemNetwork(publicClient, worldAddress as Hex); storeCache.tables.Inventory.subscribe((updates) => { console.log("inventory updates", updates); }); - const network = await setupNetwork(); - const components = createClientComponents(network); - const systemCalls = createSystemCalls(network, components); return { - network, - components, - systemCalls, + worldAddress, + publicClient, + storeCache, }; } diff --git a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts index f29d544697..7ee8d7c05f 100644 --- a/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts +++ b/examples/minimal/packages/client-react/src/mud/setupViemNetwork.ts @@ -1,5 +1,5 @@ -import { createPublicClient, http, fallback, webSocket, Hex, decodeAbiParameters, parseAbiParameters } from "viem"; -import { Schema, TableSchema, decodeKeyTuple, hexToTableSchema } from "@latticexyz/protocol-parser"; +import { PublicClient, Transport, Chain, Hex, decodeAbiParameters, parseAbiParameters } from "viem"; +import { TableSchema, decodeKeyTuple, hexToTableSchema } from "@latticexyz/protocol-parser"; import { BlockEvents, createBlockEventsStream, @@ -9,25 +9,18 @@ import { import { storeEventsAbi } from "@latticexyz/store"; import { createDatabase, createDatabaseClient } from "@latticexyz/store-cache"; import { TableId } from "@latticexyz/utils"; -import { getNetworkConfig } from "./getNetworkConfig"; import mudConfig from "contracts/mud.config"; +import * as devObservables from "@latticexyz/network/dev"; export const schemaTableId = new TableId("mudstore", "schema"); export const metadataTableId = new TableId("mudstore", "StoreMetadata"); -export async function setupViemNetwork() { - const { chain } = await getNetworkConfig(); - console.log("viem chain", chain); - - const publicClient = createPublicClient({ - chain, - // TODO: use fallback with websocket first once encoding issues are fixed - // https://github.com/wagmi-dev/viem/issues/725 - // transport: fallback([webSocket(), http()]), - transport: http(), - // TODO: do this per chain? maybe in the MUDChain config? - pollingInterval: 1000, - }); +export async function setupViemNetwork>( + publicClient: TPublicClient, + worldAddress: Hex +) { + devObservables.publicClient$.next(publicClient); + devObservables.worldAddress$.next(worldAddress); // Optional but recommended to avoid multiple instances of polling for blocks const latestBlock$ = await createBlockStream({ publicClient, blockTag: "latest" }); @@ -35,6 +28,7 @@ export async function setupViemNetwork() { const blockEvents$ = await createBlockEventsStream({ publicClient, + address: worldAddress, events: storeEventsAbi, toBlock: latestBlockNumber$, }); @@ -127,6 +121,19 @@ export async function setupViemNetwork() { const record = Object.fromEntries(valueNames.map((name, i) => [name, values[i]])); storeCache.set(tableId.namespace, tableId.name, keyTuple, record); console.log("stored record", tableId.toString(), keyTuple, record); + + devObservables.storeEvent$.next({ + event: event.eventName, + chainId: publicClient.chain.id, + worldAddress, + blockNumber: Number(block.blockNumber), + logIndex: event.logIndex!, + transactionHash: event.transactionHash!, + table: tableId, + keyTuple: event.args.key, + indexedValues: Object.fromEntries(values.map((value, i) => [i, value])), + namedValues: record, + }); } if (event.eventName === "StoreSetField") { @@ -134,11 +141,35 @@ export async function setupViemNetwork() { const value = tableSchema.valueSchema.decodeField(event.args.schemaIndex, event.args.data); storeCache.set(tableId.namespace, tableId.name, keyTuple, { [valueName]: value }); console.log("stored field", tableId.toString(), keyTuple, valueName, "=>", value); + + devObservables.storeEvent$.next({ + event: event.eventName, + chainId: publicClient.chain.id, + worldAddress, + blockNumber: Number(block.blockNumber), + logIndex: event.logIndex!, + transactionHash: event.transactionHash!, + table: tableId, + keyTuple: event.args.key, + indexedValues: { [event.args.schemaIndex]: value }, + namedValues: { [valueName]: value }, + }); } if (event.eventName === "StoreDeleteRecord") { storeCache.remove(tableId.namespace, tableId.name, keyTuple); console.log("removed record", tableId.toString(), keyTuple); + + devObservables.storeEvent$.next({ + event: event.eventName, + chainId: publicClient.chain.id, + worldAddress, + blockNumber: Number(block.blockNumber), + logIndex: event.logIndex!, + transactionHash: event.transactionHash!, + table: tableId, + keyTuple: event.args.key, + }); } }); }); From 5122c98336ef6c09b6d56440dd11ba9e9760c423 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 20 Jun 2023 19:31:18 +0100 Subject: [PATCH 10/20] ditch v1 network stuff --- .../minimal/packages/client-react/src/App.tsx | 30 ++---- .../src/mud/contractComponents.ts | 54 ---------- .../src/mud/createClientComponents.ts | 10 -- .../client-react/src/mud/createSystemCalls.ts | 21 ---- .../client-react/src/mud/getNetworkConfig.ts | 56 ---------- .../packages/client-react/src/mud/setup.ts | 52 +++++++-- .../client-react/src/mud/setupNetwork.ts | 102 ------------------ .../packages/client-react/src/mud/world.ts | 3 - .../packages/client-react/src/mud/worlds.ts | 5 + 9 files changed, 57 insertions(+), 276 deletions(-) delete mode 100644 examples/minimal/packages/client-react/src/mud/contractComponents.ts delete mode 100644 examples/minimal/packages/client-react/src/mud/createClientComponents.ts delete mode 100644 examples/minimal/packages/client-react/src/mud/createSystemCalls.ts delete mode 100644 examples/minimal/packages/client-react/src/mud/getNetworkConfig.ts delete mode 100644 examples/minimal/packages/client-react/src/mud/setupNetwork.ts delete mode 100644 examples/minimal/packages/client-react/src/mud/world.ts create mode 100644 examples/minimal/packages/client-react/src/mud/worlds.ts diff --git a/examples/minimal/packages/client-react/src/App.tsx b/examples/minimal/packages/client-react/src/App.tsx index 5bf25f7d1b..ece5409769 100644 --- a/examples/minimal/packages/client-react/src/App.tsx +++ b/examples/minimal/packages/client-react/src/App.tsx @@ -6,19 +6,7 @@ const ITEMS = ["cup", "spoon", "fork"]; const VARIANTS = ["yellow", "green", "red"]; export const App = () => { - const { - storeCache, - // network: { worldSend }, - } = useMUD(); - - async function worldSend(...args: any[]) { - // TODO - return { - wait: async () => { - // TODO - }, - }; - } + const { storeCache, world, publicClient } = useMUD(); const counter = useRow(storeCache, { table: "CounterTable", @@ -45,10 +33,10 @@ export const App = () => {