feat(store-sync,store-indexer): schemaless indexer (#1965)

.changeset/poor-waves-occur.md

@@ -0,0 +1,27 @@
+---
+"@latticexyz/common": minor
+---
+
+Added `unique` and `groupBy` array helpers to `@latticexyz/common/utils`.
+
+```ts
+import { unique } from "@latticexyz/common/utils";
+
+unique([1, 2, 1, 4, 3, 2]);
+// [1, 2, 4, 3]
+```
+
+```ts
+import { groupBy } from "@latticexyz/common/utils";
+
+const records = [
+  { type: "cat", name: "Bob" },
+  { type: "cat", name: "Spot" },
+  { type: "dog", name: "Rover" },
+];
+Object.fromEntries(groupBy(records, (record) => record.type));
+// {
+//   "cat": [{ type: "cat", name: "Bob" }, { type: "cat", name: "Spot" }],
+//   "dog: [{ type: "dog", name: "Rover" }]
+// }
+```

.changeset/seven-rice-dance.md

@@ -0,0 +1,7 @@
+---
+"@latticexyz/store-indexer": minor
+---
+
+The `findAll` method is now considered deprecated in favor of a new `getLogs` method. This is only implemented in the Postgres indexer for now, with SQLite coming soon. The new `getLogs` method will be an easier and more robust data source to hydrate the client and other indexers and will allow us to add streaming updates from the indexer in the near future.
+
+For backwards compatibility, `findAll` is now implemented on top of `getLogs`, with record key/value decoding done in memory at request time. This may not scale for large databases, so use wisely.

.changeset/wild-moose-smile.md

@@ -0,0 +1,9 @@
+---
+"@latticexyz/store-sync": major
+---
+
+`syncToPostgres` from `@latticexyz/store-sync/postgres` now uses a single table to store all records in their bytes form (`staticData`, `encodedLengths`, and `dynamicData`), more closely mirroring onchain state and enabling more scalability and stability for automatic indexing of many worlds.
+
+The previous behavior, where schemaful SQL tables are created and populated for each MUD table, has been moved to a separate `@latticexyz/store-sync/postgres-decoded` export bundle. This approach is considered less stable and is intended to be used for analytics purposes rather than hydrating clients. Some previous metadata columns on these tables have been removed in favor of the bytes records table as the source of truth for onchain state.
+
+This overhaul is considered breaking and we recommend starting a fresh database when syncing with either of these strategies.

.gitignore

@@ -14,3 +14,5 @@ yarn-error.log
 # We don't want projects created from templates to ignore their lockfiles, but we don't
 # want to check them in here, so we'll ignore them from the root.
 templates/*/pnpm-lock.yaml
+
+.env

e2e/packages/sync-test/package.json

@@ -21,7 +21,7 @@
     "chalk": "^5.2.0",
     "dotenv": "^16.0.3",
     "execa": "^7.1.1",
-    "jsdom": "^22.0.0",
+    "happy-dom": "^12.10.3",
     "typescript": "5.1.6",
     "viem": "1.14.0",
     "vite": "^4.2.1",

e2e/packages/sync-test/vite.config.ts

@@ -2,7 +2,7 @@ import { defineConfig } from "vitest/config";
 
 export default defineConfig({
   test: {
-    environment: "jsdom",
+    environment: "happy-dom",
     testTimeout: 1000 * 60 * 2,
     hookTimeout: 1000 * 60 * 2,
     singleThread: true,