Merge remote-tracking branch 'origin/main' into holic/postgres-indexe…

…r-stuck

.changeset/chilled-cougars-smash.md

@@ -0,0 +1,23 @@
+---
+"@latticexyz/faucet": minor
+---
+
+New package to run your own faucet service. We'll use this soon for our testnet in place of `@latticexyz/services`.
+
+To run the faucet server:
+
+- Add the package with `pnpm add @latticexyz/faucet`
+- Add a `.env` file that has a `RPC_HTTP_URL` and `FAUCET_PRIVATE_KEY` (or pass the environment variables into the next command)
+- Run `pnpm faucet-server` to start the server
+
+You can also adjust the server's `HOST` (defaults to `0.0.0.0`) and `PORT` (defaults to `3002`). The tRPC routes are accessible under `/trpc`.
+
+To connect a tRPC client, add the package with `pnpm add @latticexyz/faucet` and then use `createClient`:
+
+```ts
+import { createClient } from "@latticexyz/faucet";
+
+const faucet = createClient({ url: "http://localhost:3002/trpc" });
+
+await faucet.mutate.drip({ address: burnerAccount.address });
+```

.changeset/cyan-hats-try.md

@@ -0,0 +1,30 @@
+---
+"@latticexyz/store": major
+"@latticexyz/world": major
+---
+
+We've updated Store events to be "schemaless", meaning there is enough information in each event to only need to operate on the bytes of each record to make an update to that record without having to first decode the record by its schema. This enables new kinds of indexers and sync strategies.
+
+If you've written your own sync logic or are interacting with Store calls directly, this is a breaking change. We have a few more breaking protocol changes upcoming, so you may hold off on upgrading until those land.
+
+If you are using MUD's built-in tooling (table codegen, indexer, store sync, etc.), you don't have to make any changes except upgrading to the latest versions and deploying a fresh World.
+
+- The `data` field in each `StoreSetRecord` and `StoreEphemeralRecord` has been replaced with three new fields: `staticData`, `encodedLengths`, and `dynamicData`. This better reflects the on-chain state and makes it easier to perform modifications to the raw bytes. We recommend storing each of these fields individually in your off-chain storage of choice (indexer, client, etc.).
+
+  ```diff
+  - event StoreSetRecord(bytes32 tableId, bytes32[] keyTuple, bytes data);
+  + event StoreSetRecord(bytes32 tableId, bytes32[] keyTuple, bytes staticData, bytes32 encodedLengths, bytes dynamicData);
+
+  - event StoreEphemeralRecord(bytes32 tableId, bytes32[] keyTuple, bytes data);
+  + event StoreEphemeralRecord(bytes32 tableId, bytes32[] keyTuple, bytes staticData, bytes32 encodedLengths, bytes dynamicData);
+  ```
+
+- The `StoreSetField` event is now replaced by two new events: `StoreSpliceStaticData` and `StoreSpliceDynamicData`. Splicing allows us to perform efficient operations like push and pop, in addition to replacing a field value. We use two events because updating a dynamic-length field also requires updating the record's `encodedLengths` (aka PackedCounter).
+
+  ```diff
+  - event StoreSetField(bytes32 tableId, bytes32[] keyTuple, uint8 fieldIndex, bytes data);
+  + event StoreSpliceStaticData(bytes32 tableId, bytes32[] keyTuple, uint48 start, uint40 deleteCount, bytes data);
+  + event StoreSpliceDynamicData(bytes32 tableId, bytes32[] keyTuple, uint48 start, uint40 deleteCount, bytes data, bytes32 encodedLengths);
+  ```
+
+Similarly, Store setter methods (e.g. `setRecord`) have been updated to reflect the `data` to `staticData`, `encodedLengths`, and `dynamicData` changes. We'll be following up shortly with Store getter method changes for more gas efficient storage reads.

.changeset/fast-zebras-promise.md

@@ -0,0 +1,6 @@
+---
+"@latticexyz/common": minor
+"@latticexyz/protocol-parser": major
+---
+
+`readHex` was moved from `@latticexyz/protocol-parser` to `@latticexyz/common`

.changeset/hot-mice-play.md

@@ -0,0 +1,8 @@
+---
+"@latticexyz/cli": patch
+"@latticexyz/store": patch
+"@latticexyz/world": patch
+---
+
+The `FieldLayout` in table libraries is now generated at compile time instead of dynamically in a table library function.
+This significantly reduces gas cost in all table library functions.

.changeset/hungry-rings-doubt.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/store": patch
+---
+
+Added `Storage.loadField` to optimize loading 32 bytes or less from storage (which is always the case when loading data for static fields).

.changeset/rotten-cats-lay.md

@@ -0,0 +1,9 @@
+---
+"@latticexyz/common": minor
+---
+
+`spliceHex` was added, which has a similar API as JavaScript's [`Array.prototype.splice`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice), but for `Hex` strings.
+
+```ts
+spliceHex("0x123456", 1, 1, "0x0000"); // "0x12000056"
+```

.changeset/short-dragons-shout.md

@@ -0,0 +1,6 @@
+---
+"@latticexyz/store": patch
+"@latticexyz/world": patch
+---
+
+Optimized the `StoreCore` hash function determining the data location to use less gas.

.changeset/shy-sheep-wait.md

@@ -0,0 +1,9 @@
+---
+"@latticexyz/dev-tools": major
+"@latticexyz/store-sync": major
+"create-mud": minor
+---
+
+We've updated Store events to be "schemaless", meaning there is enough information in each event to only need to operate on the bytes of each record to make an update to that record without having to first decode the record by its schema. This enables new kinds of indexers and sync strategies.
+
+As such, we've replaced `blockStorageOperations$` with `storedBlockLogs$`, a stream of simplified Store event logs after they've been synced to the configured storage adapter. These logs may not reflect exactly the events that are on chain when e.g. hydrating from an indexer, but they will still allow the client to "catch up" to the on-chain state of your tables.

.changeset/sour-cycles-warn.md

@@ -0,0 +1,34 @@
+---
+"@latticexyz/cli": patch
+"@latticexyz/store": minor
+"@latticexyz/world": patch
+---
+
+`StoreCore` and `IStore` now expose specific functions for `getStaticField` and `getDynamicField` in addition to the general `getField`.
+Using the specific functions reduces gas overhead because more optimized logic can be executed.
+
+```solidity
+interface IStore {
+  /**
+   * Get a single static field from the given tableId and key tuple, with the given value field layout.
+   * Note: the field value is left-aligned in the returned bytes32, the rest of the word is not zeroed out.
+   * Consumers are expected to truncate the returned value as needed.
+   */
+  function getStaticField(
+    bytes32 tableId,
+    bytes32[] calldata keyTuple,
+    uint8 fieldIndex,
+    FieldLayout fieldLayout
+  ) external view returns (bytes32);
+
+  /**
+   * Get a single dynamic field from the given tableId and key tuple at the given dynamic field index.
+   * (Dynamic field index = field index - number of static fields)
+   */
+  function getDynamicField(
+    bytes32 tableId,
+    bytes32[] memory keyTuple,
+    uint8 dynamicFieldIndex
+  ) external view returns (bytes memory);
+}
+```

.changeset/stale-worms-hunt.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/store": major
+---
+
+Store events now use an `indexed` `tableId`. This adds ~100 gas per write, but means we our sync stack can filter events by table.

.changeset/thin-terms-lay.md

@@ -0,0 +1,23 @@
+---
+"@latticexyz/cli": patch
+"@latticexyz/common": minor
+"@latticexyz/store": minor
+"@latticexyz/world": patch
+---
+
+Generated table libraries now have a set of functions prefixed with `_` that always use their own storage for read/write.
+This saves gas for use cases where the functionality to dynamically determine which `Store` to use for read/write is not needed, e.g. root systems in a `World`, or when using `Store` without `World`.
+
+We decided to continue to always generate a set of functions that dynamically decide which `Store` to use, so that the generated table libraries can still be imported by non-root systems.
+
+```solidity
+library Counter {
+  // Dynamically determine which store to write to based on the context
+  function set(uint32 value) internal;
+
+  // Always write to own storage
+  function _set(uint32 value) internal;
+
+  // ... equivalent functions for all other Store methods
+}
+```

.gitattributes

@@ -1,2 +1,3 @@
 # suppress diffs for codegen in PRs
 **/codegen/**/*.sol linguist-generated=true
+**/test-data/**/*.json linguist-generated=true

.github/workflows/docker.yml

@@ -1,13 +1,7 @@
 name: Docker
 
 on:
-  push:
-    branches:
-      - main
-    tags:
-      # only target a single package tag to avoid running this task for every package on a version release
-      # in the metadata step, we parse out the specific version used in the tag (e.g. `2.0.0-next.5` and `next`)
-      - "@latticexyz/common@*"
+  workflow_call:
   workflow_dispatch:
 
 jobs:
@@ -37,6 +31,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
+          # If this doesn't tag properly within the context of `workflow_call`, try `context: git`
           images: ${{ matrix.image }}
           tags: |
             type=raw,value=latest,enable={{is_default_branch}}

.github/workflows/prerelease.yml

@@ -33,7 +33,7 @@ jobs:
         run: npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}"
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-          
+
       - name: Check for pre.json file existence
         id: check_files
         uses: andstor/[email protected]
@@ -47,10 +47,13 @@ jobs:
         run: npx changeset pre enter next
 
       - name: Create next version PR or publish 🚀
-        uses: changesets/action@v1 
+        uses: changesets/action@v1
         with:
           version: pnpm release:version
           publish: pnpm release:publish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  docker:
+    uses: ./.github/workflows/docker.yml
+    needs: prerelease

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
 
       - name: "Setup"
         uses: ./.github/actions/setup
-        
+
       - name: Clean
         shell: bash
         run: pnpm turbo run clean --force --concurrency 10
@@ -60,3 +60,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
+  docker:
+    uses: ./.github/workflows/docker.yml
+    needs: version

.github/workflows/snapshot.yml

@@ -48,3 +48,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+  docker:
+    uses: ./.github/workflows/docker.yml
+    needs: release-snapshot

Dockerfile

@@ -50,3 +50,7 @@ RUN pnpm run -r build
 FROM mud AS store-indexer
 WORKDIR /app/packages/store-indexer
 EXPOSE 3001
+
+FROM mud AS faucet
+WORKDIR /app/packages/faucet
+EXPOSE 3002

e2e/packages/client-vanilla/src/mud/setupNetwork.ts

@@ -33,7 +33,7 @@ export async function setupNetwork() {
     walletClient: burnerWalletClient,
   });
 
-  const { components, latestBlock$, blockStorageOperations$, waitForTransaction } = await syncToRecs({
+  const { components, latestBlock$, storedBlockLogs$, waitForTransaction } = await syncToRecs({
     world,
     config: mudConfig,
     address: networkConfig.worldAddress as Hex,
@@ -74,7 +74,7 @@ export async function setupNetwork() {
     walletClient: burnerWalletClient,
     worldContract,
     latestBlock$,
-    blockStorageOperations$,
+    storedBlockLogs$,
     waitForTransaction,
   };
 }