refactor(store-sync): use config namespaces for tables (#2963)

Co-authored-by: alvrs <[email protected]>

.changeset/khaki-wolves-perform.md

@@ -0,0 +1,20 @@
+---
+"@latticexyz/store-sync": patch
+---
+
+Refactored `syncToRecs` and `syncToZustand` to use tables from config namespaces output. This is a precursor for supporting multiple namespaces.
+
+Note for library authors: If you were using `createStorageAdapter` from `@latticexyz/store-sync/recs`, this helper no longer appends MUD's built-in tables from Store and World packages. This behavior was moved into `syncToRecs` for consistency with `syncToZustand` and makes `createStorageAdapter` less opinionated.
+
+You can achieve the previous behavior with:
+
+```diff
+ import { createStorageAdapter } from "@latticexyz/store-sync/recs";
++import { mudTables } from "@latticexyz/store-sync";
+
+ createStorageAdapter({
+-  tables,
++  tables: { ...tables, ...mudTables },
+   ...
+ });
+```

package.json

@@ -37,7 +37,7 @@
     "package.json": "pnpm sort-package-json"
   },
   "devDependencies": {
-    "@ark/attest": "0.9.2",
+    "@arktype/attest": "0.7.5",
     "@changesets/cli": "^2.26.1",
     "@types/node": "^18.15.11",
     "@typescript-eslint/eslint-plugin": "7.1.1",

packages/store-sync/package.json

@@ -62,6 +62,7 @@
     "test:ci": "vitest --run"
   },
   "dependencies": {
+    "@arktype/util": "0.1.0",
     "@latticexyz/block-logs-stream": "workspace:*",
     "@latticexyz/common": "workspace:*",
     "@latticexyz/config": "workspace:*",

packages/store-sync/src/common.test.ts

@@ -0,0 +1,12 @@
+import { describe, it } from "vitest";
+import { attest } from "@arktype/attest";
+import { isDisjoint } from "@arktype/util";
+import storeConfig from "@latticexyz/store/mud.config";
+import worldConfig from "@latticexyz/world/mud.config";
+import { configToTables } from "./configToTables";
+
+describe("mudTables", () => {
+  it("has no overlapping table labels", () => {
+    attest<true, isDisjoint<keyof configToTables<typeof storeConfig>, keyof configToTables<typeof worldConfig>>>();
+  });
+});

packages/store-sync/src/common.ts

@@ -12,16 +12,15 @@ import storeConfig from "@latticexyz/store/mud.config";
 import worldConfig from "@latticexyz/world/mud.config";
 import { Store as StoreConfig } from "@latticexyz/store";
 import { Table as ConfigTable, Schema } from "@latticexyz/config";
+import { configToTables } from "./configToTables";
 
-export const storeTables = storeConfig.tables;
-export type storeTables = typeof storeTables;
+export const mudTables = {
+  ...configToTables(storeConfig),
+  ...configToTables(worldConfig),
+} as const;
+export type mudTables = typeof mudTables;
 
-export const worldTables = worldConfig.tables;
-export type worldTables = typeof worldTables;
-
-export const internalTableIds = [...Object.values(storeTables), ...Object.values(worldTables)].map(
-  (table) => table.tableId,
-);
+export const internalTableIds = Object.values(mudTables).map((table) => table.tableId);
 
 export type ChainId = number;
 export type WorldId = `${ChainId}:${Address}`;
@@ -137,7 +136,7 @@ export type StorageAdapterBlock = { blockNumber: BlockLogs["blockNumber"]; logs:
 export type StorageAdapter = (block: StorageAdapterBlock) => Promise<void>;
 
 export const schemasTable = {
-  ...storeTables.store__Tables,
-  keySchema: getSchemaTypes(getKeySchema(storeTables.store__Tables)),
-  valueSchema: getSchemaTypes(getValueSchema(storeTables.store__Tables)),
+  ...mudTables.Tables,
+  keySchema: getSchemaTypes(getKeySchema(mudTables.Tables)),
+  valueSchema: getSchemaTypes(getValueSchema(mudTables.Tables)),
 };

packages/store-sync/src/configToTables.test.ts

@@ -0,0 +1,44 @@
+import { describe, it } from "vitest";
+import { configToTables } from "./configToTables";
+import { defineWorld } from "@latticexyz/world";
+import { resourceToHex } from "@latticexyz/common";
+import { attest } from "@arktype/attest";
+
+describe("configToTables", () => {
+  it("flattens tables from single namespace", async () => {
+    const config = defineWorld({
+      namespace: "app",
+      tables: {
+        ExceedsResourceNameSizeLimit: "bytes32",
+        Table2: "address",
+      },
+    });
+
+    const tables = configToTables(config);
+
+    attest<"ExceedsResourceNameSizeLimit" | "Table2", keyof typeof tables>();
+
+    attest(tables.ExceedsResourceNameSizeLimit.tableId).equals(
+      resourceToHex({
+        type: "table",
+        namespace: "app",
+        name: "ExceedsResourceN",
+      }),
+    );
+    attest(tables.ExceedsResourceNameSizeLimit.label).equals("ExceedsResourceNameSizeLimit");
+    attest(tables.ExceedsResourceNameSizeLimit.name).equals("ExceedsResourceN");
+
+    attest(tables.Table2.tableId).equals(
+      resourceToHex({
+        type: "table",
+        namespace: "app",
+        name: "Table2",
+      }),
+    );
+    attest(tables.Table2.label).equals("Table2");
+    attest(tables.Table2.name).equals("Table2");
+  });
+
+  // TODO: add test with multiple namespaces
+  // TODO: add test where the label is the same for two tables in different namespaces to make sure TS + runtime agree on which takes precedence
+});

packages/store-sync/src/configToTables.ts

@@ -0,0 +1,25 @@
+import { show } from "@arktype/util";
+import { Store } from "@latticexyz/store";
+
+type flattenedTableKeys<config extends Store> = config extends { readonly namespaces: infer namespaces }
+  ? {
+      [namespaceLabel in keyof namespaces]: namespaces[namespaceLabel] extends { readonly tables: infer tables }
+        ? `${namespaceLabel & string}__${keyof tables & string}`
+        : never;
+    }[keyof namespaces]
+  : never;
+
+// TODO: figure out how TS handles overlapping table labels so we can make runtime match
+
+export type configToTables<config extends Store> = {
+  readonly [key in flattenedTableKeys<config> as key extends `${string}__${infer tableLabel}`
+    ? tableLabel
+    : never]: key extends `${infer namespaceLabel}__${infer tableLabel}`
+    ? config["namespaces"][namespaceLabel]["tables"][tableLabel]
+    : never;
+};
+
+export function configToTables<config extends Store>(config: config): show<configToTables<config>> {
+  const tables = Object.values(config.namespaces).flatMap((namespace) => Object.values(namespace.tables));
+  return Object.fromEntries(tables.map((table) => [table.label, table])) as never;
+}

packages/store-sync/src/index.ts

@@ -1,4 +1,5 @@
 export * from "./common";
+export * from "./configToTables";
 export * from "./createStoreSync";
 export * from "./SyncStep";
 export * from "./isTableRegistrationLog";

packages/store-sync/src/postgres-decoded/syncToPostgres.ts

@@ -4,7 +4,7 @@ import { SyncOptions, SyncResult } from "../common";
 import { createStorageAdapter } from "./createStorageAdapter";
 import { createStoreSync } from "../createStoreSync";
 
-type SyncToPostgresOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
+export type SyncToPostgresOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
   /**
    * [Postgres database object from Drizzle][0].
    *
@@ -15,7 +15,7 @@ type SyncToPostgresOptions<config extends StoreConfig = StoreConfig> = SyncOptio
   startSync?: boolean;
 };
 
-type SyncToPostgresResult = SyncResult & {
+export type SyncToPostgresResult = SyncResult & {
   stopSync: () => void;
 };
 

packages/store-sync/src/postgres/syncToPostgres.ts

@@ -4,7 +4,7 @@ import { SyncOptions, SyncResult } from "../common";
 import { createStorageAdapter } from "./createStorageAdapter";
 import { createStoreSync } from "../createStoreSync";
 
-type SyncToPostgresOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
+export type SyncToPostgresOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
   /**
    * [Postgres database object from Drizzle][0].
    *
@@ -15,7 +15,7 @@ type SyncToPostgresOptions<config extends StoreConfig = StoreConfig> = SyncOptio
   startSync?: boolean;
 };
 
-type SyncToPostgresResult = SyncResult & {
+export type SyncToPostgresResult = SyncResult & {
   stopSync: () => void;
 };
 

packages/store-sync/src/recs/createStorageAdapter.ts

@@ -1,4 +1,4 @@
-import { Table } from "@latticexyz/config";
+import { Tables } from "@latticexyz/config";
 import { debug } from "./debug";
 import { World as RecsWorld, getComponentValue, hasComponent, removeComponent, setComponent } from "@latticexyz/recs";
 import { defineInternalComponents } from "./defineInternalComponents";
@@ -11,28 +11,21 @@ import { logToTable } from "../logToTable";
 import { hexKeyTupleToEntity } from "./hexKeyTupleToEntity";
 import { StorageAdapter, StorageAdapterBlock } from "../common";
 import { singletonEntity } from "./singletonEntity";
-import storeConfig from "@latticexyz/store/mud.config";
-import worldConfig from "@latticexyz/world/mud.config";
 import { tablesToComponents } from "./tablesToComponents";
+import { merge } from "@arktype/util";
 
-const storeTables = storeConfig.tables;
-const worldTables = worldConfig.tables;
-
-export type CreateStorageAdapterOptions<tables extends Record<string, Table>> = {
+export type CreateStorageAdapterOptions<tables extends Tables> = {
   world: RecsWorld;
   tables: tables;
   shouldSkipUpdateStream?: () => boolean;
 };
 
-export type CreateStorageAdapterResult<tables extends Record<string, Table>> = {
+export type CreateStorageAdapterResult<tables extends Tables> = {
   storageAdapter: StorageAdapter;
-  components: tablesToComponents<tables> &
-    tablesToComponents<typeof storeTables> &
-    tablesToComponents<typeof worldTables> &
-    ReturnType<typeof defineInternalComponents>;
+  components: merge<tablesToComponents<tables>, ReturnType<typeof defineInternalComponents>>;
 };
 
-export function createStorageAdapter<tables extends Record<string, Table>>({
+export function createStorageAdapter<tables extends Tables>({
   world,
   tables,
   shouldSkipUpdateStream,
@@ -41,8 +34,6 @@ export function createStorageAdapter<tables extends Record<string, Table>>({
 
   const components = {
     ...tablesToComponents(world, tables),
-    ...tablesToComponents(world, storeTables),
-    ...tablesToComponents(world, worldTables),
     ...defineInternalComponents(world),
   };
 

packages/store-sync/src/recs/syncToRecs.ts

@@ -1,35 +1,38 @@
-import { Table } from "@latticexyz/config";
+import { Tables } from "@latticexyz/config";
 import { Store as StoreConfig } from "@latticexyz/store";
 import { Component as RecsComponent, World as RecsWorld, getComponentValue, setComponent } from "@latticexyz/recs";
-import { SyncOptions, SyncResult } from "../common";
+import { SyncOptions, SyncResult, mudTables } from "../common";
 import { CreateStorageAdapterResult, createStorageAdapter } from "./createStorageAdapter";
 import { createStoreSync } from "../createStoreSync";
 import { singletonEntity } from "./singletonEntity";
 import { SyncStep } from "../SyncStep";
+import { configToTables } from "../configToTables";
+import { merge } from "@arktype/util";
 
-type SyncToRecsOptions<config extends StoreConfig, extraTables extends Record<string, Table>> = SyncOptions & {
+export type SyncToRecsOptions<config extends StoreConfig, extraTables extends Tables> = Omit<SyncOptions, "config"> & {
   world: RecsWorld;
   config: config;
   tables?: extraTables;
   startSync?: boolean;
 };
 
-type SyncToRecsResult<config extends StoreConfig, extraTables extends Record<string, Table>> = SyncResult & {
-  components: CreateStorageAdapterResult<config["tables"] & extraTables>["components"];
+export type SyncToRecsResult<config extends StoreConfig, extraTables extends Tables> = SyncResult & {
+  components: CreateStorageAdapterResult<merge<merge<configToTables<config>, extraTables>, mudTables>>["components"];
   stopSync: () => void;
 };
 
-export async function syncToRecs<config extends StoreConfig, extraTables extends Record<string, Table>>({
+export async function syncToRecs<config extends StoreConfig, extraTables extends Tables = {}>({
   world,
   config,
-  tables: extraTables,
+  tables: extraTables = {} as extraTables,
   startSync = true,
   ...syncOptions
 }: SyncToRecsOptions<config, extraTables>): Promise<SyncToRecsResult<config, extraTables>> {
   const tables = {
-    ...config.tables,
+    ...configToTables(config),
     ...extraTables,
-  } as config["tables"] & extraTables;
+    ...mudTables,
+  };
 
   const { storageAdapter, components } = createStorageAdapter({
     world,
@@ -79,5 +82,5 @@ export async function syncToRecs<config extends StoreConfig, extraTables extends
     ...storeSync,
     components,
     stopSync,
-  };
+  } as never;
 }

packages/store-sync/src/recs/tablesToComponents.ts

@@ -1,15 +1,16 @@
-import { Table } from "@latticexyz/config";
+import { Tables } from "@latticexyz/config";
 import { tableToComponent } from "./tableToComponent";
 import { World } from "@latticexyz/recs";
+import { show } from "@arktype/util";
 
-export type tablesToComponents<tables extends Record<string, Table>> = {
+export type tablesToComponents<tables extends Tables> = {
   [label in keyof tables as tables[label]["label"]]: tableToComponent<tables[label]>;
 };
 
-export function tablesToComponents<tables extends Record<string, Table>>(
+export function tablesToComponents<tables extends Tables>(
   world: World,
   tables: tables,
-): tablesToComponents<tables> {
+): show<tablesToComponents<tables>> {
   return Object.fromEntries(
     Object.entries(tables).map(([, table]) => [table.label, tableToComponent(world, table)]),
   ) as never;

packages/store-sync/src/sqlite/syncToSqlite.ts

@@ -4,7 +4,7 @@ import { SyncOptions, SyncResult } from "../common";
 import { sqliteStorage } from "./sqliteStorage";
 import { createStoreSync } from "../createStoreSync";
 
-type SyncToSqliteOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
+export type SyncToSqliteOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
   /**
    * [SQLite database object from Drizzle][0].
    *
@@ -15,7 +15,7 @@ type SyncToSqliteOptions<config extends StoreConfig = StoreConfig> = SyncOptions
   startSync?: boolean;
 };
 
-type SyncToSqliteResult = SyncResult & {
+export type SyncToSqliteResult = SyncResult & {
   stopSync: () => void;
 };