Merge remote-tracking branch 'origin/main' into holic/mud-pull

.changeset/blue-starfishes-fry.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/explorer": patch
+---
+
+Observe tab is now populated by transactions flowing through the world, in addition to local transactions when using the `observer` transport wrapper.

.changeset/config.json

@@ -11,6 +11,10 @@
     "useCalculatedVersion": true,
     "prereleaseTemplate": "{tag}-{commit}"
   },
+  "privatePackages": {
+    "tag": false,
+    "version": false
+  },
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true
   }

.changeset/early-colts-smile.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/block-logs-stream": patch
+---
+
+`fetchLogs` and `blockRangeToLogs` now accept a `getLogs` option to override the default behavior.

.changeset/gold-mangos-sneeze.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/store": patch
+---
+
+Added `getStoreLogs` and `flattenStoreLogs` to aid in fetching data from store contracts. For now, these are internal exports and considered unstable/experimental.

.changeset/great-dragons-sit.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/store-sync": patch
+---
+
+Added support for streaming logs from the indexer.

.changeset/neat-impalas-arrive.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/explorer": patch
+---
+
+Observe tab is now populated by rejected transactions coming from the `observer` transport wrapper.

.changeset/nervous-clouds-collect.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/cli": patch
+---
+
+Deployer now has a better method for fetching store logs from the world that should be more efficient and resilient to block range errors and rate limiting.

.changeset/sour-bats-compare.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/store": patch
+---
+
+Added an `unwrap` function to the `ResourceIdInstance` library to make it easier to unwrap a `ResourceId` with `resourceId.unwrap()`.

.changeset/ten-llamas-protect.md

@@ -0,0 +1,12 @@
+---
+"@latticexyz/common": patch
+---
+
+Added `logSort` method to help when sorting logs fetched from RPC, where they come back ordered relative to the topics used.
+
+```ts
+import { logSort } from "@latticexyz/common";
+
+const logs = getLogs(...);
+logs.sort(logSort);
+```

.gitignore

@@ -5,11 +5,11 @@ node_modules
 package-lock.json
 yarn.lock
 
+*.log
+
 .eslintcache
 .parcel-cache
 .docs
-lerna-debug.log
-yarn-error.log
 .turbo
 .attest
 .tstrace

CODEOWNERS

@@ -3,7 +3,7 @@
 
 * @alvrs @holic
 
-/docs @ludns @qbzzt @alvrs @holic
+/docs @ludns @alvrs @holic
 
 /packages/explorer @alvrs @holic @karooolis
 /examples/local-explorer @alvrs @holic @karooolis

docs/pages/store/reference/misc.mdx

@@ -1870,6 +1870,26 @@ function getResourceName(ResourceId resourceId) internal pure returns (bytes30);
 | -------- | --------- | --------------- |
 | `<none>` | `bytes30` | A 30-byte name. |
 
+#### unwrap
+
+Unwrap a resource ID into a bytes32.
+
+```solidity
+function unwrap(ResourceId resourceId) internal pure returns (bytes32);
+```
+
+**Parameters**
+
+| Name         | Type         | Description      |
+| ------------ | ------------ | ---------------- |
+| `resourceId` | `ResourceId` | The resource ID. |
+
+**Returns**
+
+| Name     | Type      | Description             |
+| -------- | --------- | ----------------------- |
+| `<none>` | `bytes32` | The underlying bytes32. |
+
 ## ResourceIdLib
 
 [Git Source](https://github.com/latticexyz/mud/blob/main/packages/store/src/ResourceId.sol)

package.json

@@ -12,7 +12,7 @@
     "build": "turbo run build",
     "changelog:generate": "tsx scripts/changelog.ts",
     "clean": "turbo run clean",
-    "dev": "TSUP_SKIP_DTS=true turbo run dev --concurrency 100",
+    "dev": "TSUP_SKIP_DTS=true turbo run dev --concurrency 100 --filter !@latticexyz/explorer",
     "dist-tag-rm": "pnpm recursive exec -- sh -c 'npm dist-tag rm $(cat package.json | jq -r \".name\") $TAG || true'",
     "docs:generate:api": "tsx scripts/render-api-docs.ts",
     "fix:package-json": "sort-package-json package.json 'packages/*/package.json' 'templates/*/package.json' 'templates/*/packages/*/package.json' 'examples/*/package.json' 'examples/*/packages/*/package.json' 'e2e/*/package.json' 'e2e/*/packages/*/package.json' 'docs/package.json' 'test/*/package.json'",
@@ -25,10 +25,12 @@
     "release:check": "changeset status --verbose --since=origin/main",
     "release:publish": "pnpm install && pnpm build && changeset publish",
     "release:version": "changeset version && pnpm install --lockfile-only && pnpm run changelog:generate",
-    "test": "pnpm run --recursive test",
-    "test:ci": "pnpm run --recursive --parallel test:ci",
+    "test": "pnpm run test:setup && pnpm run --recursive test",
+    "test:ci": "pnpm run test:setup && pnpm run --recursive test:ci",
+    "test:setup": "tsx test-setup/startAnvil.ts & pnpm run --filter mock-game-contracts build",
     "type-bench": "pnpm --filter ./test/ts-benchmarks bench",
-    "type-stats-repo": "attest stats packages/*"
+    "type-stats-repo": "attest stats packages/*",
+    "vercel:prepare": "(forge --version || pnpm foundryup) && ln -sf /vercel/.foundry/bin/* node_modules/.bin/ && forge --version"
   },
   "lint-staged": {
     "*.{ts,tsx}": "eslint --cache --fix",
@@ -41,9 +43,10 @@
     "@types/node": "^18.15.11",
     "@typescript-eslint/eslint-plugin": "7.1.1",
     "@typescript-eslint/parser": "7.1.1",
+    "@viem/anvil": "^0.0.7",
     "chalk": "^5.2.0",
     "eslint": "8.57.0",
-    "execa": "^7.0.0",
+    "execa": "^9.4.0",
     "glob": "^10.4.2",
     "husky": ">=6",
     "lint-staged": "^15.2.10",
@@ -53,11 +56,13 @@
     "sort-package-json": "^2.10.1",
     "tsx": "4.16.2",
     "turbo": "^1.9.3",
-    "typescript": "5.4.2"
+    "typescript": "5.4.2",
+    "viem": "catalog:",
+    "vitest": "2.1.2"
   },
   "packageManager": "[email protected]",
   "engines": {
-    "node": "^18.20.1",
+    "node": "18.x || 20.x",
     "pnpm": "^9.6.0"
   }
 }

packages/block-logs-stream/src/blockRangeToLogs.ts

@@ -1,29 +1,13 @@
 import { EMPTY, OperatorFunction, concatMap, from, pipe, tap } from "rxjs";
-import { FetchLogsResult, fetchLogs } from "./fetchLogs";
+import { FetchLogsOptions, FetchLogsResult, fetchLogs } from "./fetchLogs";
 import { AbiEvent } from "abitype";
-import { Address, BlockNumber, Client } from "viem";
+import { BlockNumber, UnionOmit } from "viem";
 import { debug } from "./debug";
 
-export type BlockRangeToLogsOptions<abiEvents extends readonly AbiEvent[]> = {
-  /**
-   * [viem `Client`][0] used for fetching logs from the RPC.
-   *
-   * [0]: https://viem.sh/docs/clients/public.html
-   */
-  publicClient: Client;
-  /**
-   * Optional contract address(es) to fetch logs for.
-   */
-  address?: Address | Address[];
-  /**
-   * Events to fetch logs for.
-   */
-  events: abiEvents;
-  /**
-   * Optional maximum block range, if your RPC limits the amount of blocks fetched at a time.
-   */
-  maxBlockRange?: bigint;
-};
+export type BlockRangeToLogsOptions<abiEvents extends readonly AbiEvent[]> = UnionOmit<
+  FetchLogsOptions<abiEvents>,
+  "fromBlock" | "toBlock"
+>;
 
 export type BlockRangeToLogsResult<abiEvents extends readonly AbiEvent[]> = OperatorFunction<
   { startBlock: BlockNumber; endBlock: BlockNumber },
@@ -38,12 +22,9 @@ export type BlockRangeToLogsResult<abiEvents extends readonly AbiEvent[]> = Oper
  * @param {BlockRangeToLogsOptions<AbiEvent[]>} options See `BlockRangeToLogsOptions`.
  * @returns {BlockRangeToLogsResult<AbiEvent[]>} An operator function that transforms a stream of block ranges into a stream of fetched logs.
  */
-export function blockRangeToLogs<abiEvents extends readonly AbiEvent[]>({
-  publicClient,
-  address,
-  events,
-  maxBlockRange,
-}: BlockRangeToLogsOptions<abiEvents>): BlockRangeToLogsResult<abiEvents> {
+export function blockRangeToLogs<abiEvents extends readonly AbiEvent[]>(
+  opts: BlockRangeToLogsOptions<abiEvents>,
+): BlockRangeToLogsResult<abiEvents> {
   let fromBlock: bigint;
   let toBlock: bigint;
 
@@ -57,16 +38,7 @@ export function blockRangeToLogs<abiEvents extends readonly AbiEvent[]>({
     concatMap(() => {
       if (fromBlock > toBlock) return EMPTY;
       debug(`fetching logs for block range ${fromBlock}-${toBlock}`);
-      return from(
-        fetchLogs({
-          publicClient,
-          address,
-          events,
-          fromBlock,
-          toBlock,
-          maxBlockRange,
-        }),
-      ).pipe(
+      return from(fetchLogs<abiEvents>({ ...opts, fromBlock, toBlock })).pipe(
         tap(({ toBlock }) => {
           fromBlock = toBlock + 1n;
         }),

packages/block-logs-stream/src/fetchLogs.ts

@@ -1,25 +1,11 @@
 import { AbiEvent } from "abitype";
-import { Address, Client, BlockNumber, GetLogsReturnType } from "viem";
+import { Address, Client, BlockNumber, GetLogsReturnType, OneOf } from "viem";
 import { bigIntMin, wait } from "@latticexyz/common/utils";
 import { debug } from "./debug";
 import { getAction } from "viem/utils";
-import { getLogs } from "viem/actions";
+import { getLogs as viem_getLogs } from "viem/actions";
 
 export type FetchLogsOptions<abiEvents extends readonly AbiEvent[]> = {
-  /**
-   * [viem `Client`][0] used for fetching logs from the RPC.
-   *
-   * [0]: https://viem.sh/docs/clients/public.html
-   */
-  publicClient: Client;
-  /**
-   * Optional contract address(es) to fetch logs for.
-   */
-  address?: Address | Address[];
-  /**
-   * Events to fetch logs for.
-   */
-  events: abiEvents;
   /**
    * The block number to start fetching logs from (inclusive).
    */
@@ -36,7 +22,33 @@ export type FetchLogsOptions<abiEvents extends readonly AbiEvent[]> = {
    * Optional maximum amount of retries if the RPC returns a rate limit error. Defaults to 3.
    */
   maxRetryCount?: number;
-};
+} & OneOf<
+  | {
+      /**
+       * Async function to return logs for the given block range.
+       */
+      getLogs: (args: {
+        fromBlock: bigint;
+        toBlock: bigint;
+      }) => Promise<GetLogsReturnType<undefined, abiEvents, true, BlockNumber, BlockNumber>>;
+    }
+  | {
+      /**
+       * [viem `Client`][0] used for fetching logs from the RPC.
+       *
+       * [0]: https://viem.sh/docs/clients/public.html
+       */
+      publicClient: Client;
+      /**
+       * Optional contract address(es) to fetch logs for.
+       */
+      address?: Address | Address[];
+      /**
+       * Events to fetch logs for.
+       */
+      events: abiEvents;
+    }
+>;
 
 export type FetchLogsResult<abiEvents extends readonly AbiEvent[]> = {
   fromBlock: BlockNumber;
@@ -96,25 +108,31 @@ const BLOCK_RANGE_ERRORS = [
 export async function* fetchLogs<abiEvents extends readonly AbiEvent[]>({
   maxBlockRange = 1000n,
   maxRetryCount = 3,
-  publicClient,
-  ...getLogsOpts
+  fromBlock: initialFromBlock,
+  toBlock: initialToBlock,
+  ...opts
 }: FetchLogsOptions<abiEvents>): AsyncGenerator<FetchLogsResult<abiEvents>> {
-  let fromBlock = getLogsOpts.fromBlock;
-  let blockRange = bigIntMin(maxBlockRange, getLogsOpts.toBlock - fromBlock);
+  const getLogs =
+    opts.getLogs ??
+    (async (blockRange): Promise<GetLogsReturnType<undefined, abiEvents, true, BlockNumber, BlockNumber>> =>
+      getAction(
+        opts.publicClient,
+        viem_getLogs,
+        "getLogs",
+      )({ ...blockRange, address: opts.address, events: opts.events, strict: true }));
+
+  let fromBlock = initialFromBlock;
+  let blockRange = bigIntMin(maxBlockRange, initialToBlock - fromBlock);
   let retryCount = 0;
 
-  while (fromBlock <= getLogsOpts.toBlock) {
+  while (fromBlock <= initialToBlock) {
     try {
       const toBlock = fromBlock + blockRange;
       debug(`getting logs for blocks ${fromBlock}-${toBlock} (${blockRange} blocks, ${maxBlockRange} max)`);
-      const logs = await getAction(
-        publicClient,
-        getLogs,
-        "getLogs",
-      )({ ...getLogsOpts, fromBlock, toBlock, strict: true });
+      const logs = await getLogs({ fromBlock, toBlock });
       yield { fromBlock, toBlock, logs };
       fromBlock = toBlock + 1n;
-      blockRange = bigIntMin(maxBlockRange, getLogsOpts.toBlock - fromBlock);
+      blockRange = bigIntMin(maxBlockRange, initialToBlock - fromBlock);
       retryCount = 0;
     } catch (error: unknown) {
       if (!(error instanceof Error)) throw error;

packages/block-logs-stream/src/groupLogsByBlockNumber.ts

@@ -1,5 +1,6 @@
 import { BlockNumber } from "viem";
-import { bigIntSort, isDefined } from "@latticexyz/common/utils";
+import { logSort } from "@latticexyz/common";
+import { bigIntSort, groupBy } from "@latticexyz/common/utils";
 
 type PartialLog = { readonly blockNumber: bigint; readonly logIndex: number };
 
@@ -29,22 +30,12 @@ export function groupLogsByBlockNumber<log extends PartialLog>(
   const blockNumbers = Array.from(new Set(logs.map((log) => log.blockNumber)));
   blockNumbers.sort(bigIntSort);
 
-  const groupedBlocks = blockNumbers
-    .map((blockNumber) => {
-      const blockLogs = logs.filter((log) => log.blockNumber === blockNumber);
-      if (!blockLogs.length) return;
-      blockLogs.sort((a, b) => (a.logIndex < b.logIndex ? -1 : a.logIndex > b.logIndex ? 1 : 0));
+  const sortedLogs = logs.slice().sort(logSort);
+  const groupedBlocks = Array.from(groupBy(sortedLogs, (log) => log.blockNumber).entries())
+    .map(([blockNumber, logs]) => ({ blockNumber, logs }))
+    .filter((block) => block.logs.length > 0);
 
-      if (!blockLogs.length) return;
-
-      return {
-        blockNumber,
-        logs: blockLogs,
-      };
-    })
-    .filter(isDefined);
-
-  const lastBlockNumber = blockNumbers.length > 0 ? blockNumbers[blockNumbers.length - 1] : null;
+  const lastBlockNumber = blockNumbers.at(-1);
 
   if (toBlock != null && (lastBlockNumber == null || toBlock > lastBlockNumber)) {
     groupedBlocks.push({