WIP: stash state lib

packages/config/src/common.ts

@@ -23,10 +23,20 @@ export type Schema = {
   };
 };
 
+export type KeySchema = {
+  readonly [fieldName: string]: {
+    /** the Solidity primitive ABI type */
+    readonly type: StaticAbiType;
+    /** the user defined type or Solidity primitive ABI type */
+    readonly internalType: string;
+  };
+};
+
 export type Table = {
   readonly label: string;
   readonly type: satisfy<ResourceType, "table" | "offchainTable">;
   readonly namespace: string;
+  readonly namespaceLabel: string;
   readonly name: string;
   readonly tableId: Hex;
   readonly schema: Schema;

packages/config/src/exports/index.ts

@@ -4,4 +4,4 @@
  * Be sure we're ready to commit to these being supported and changes made backward compatible!
  */
 
-export type { AbiType, StaticAbiType, DynamicAbiType, Schema, Table, Tables } from "../common";
+export type { AbiType, StaticAbiType, DynamicAbiType, Schema, KeySchema, Table, Tables } from "../common";

packages/protocol-parser/src/common.ts

@@ -1,3 +1,4 @@
+import { Table } from "@latticexyz/config";
 import {
   DynamicAbiType,
   SchemaAbiType,
@@ -51,3 +52,5 @@ export type ValueArgs = {
   encodedLengths: Hex;
   dynamicData: Hex;
 };
+
+export type PartialTable = Pick<Table, "schema" | "key">;

packages/protocol-parser/src/decodeDozerField.test.ts

@@ -0,0 +1,9 @@
+import { describe, expect, it } from "vitest";
+import { decodeDozerField } from "./decodeDozerField";
+
+describe("decodeDozerField", () => {
+  it("should decode numbers to the expected value type", () => {
+    expect(decodeDozerField("uint48", "1")).toBe(1);
+    expect(decodeDozerField("uint56", "1")).toBe(1n);
+  });
+});

packages/protocol-parser/src/decodeDozerField.ts

@@ -0,0 +1,24 @@
+import { AbiType } from "@latticexyz/config";
+import {
+  ArrayAbiType,
+  SchemaAbiTypeToPrimitiveType,
+  arrayToStaticAbiType,
+  schemaAbiTypeToDefaultValue,
+} from "@latticexyz/schema-type/internal";
+
+export function decodeDozerField<abiType extends AbiType>(
+  abiType: abiType,
+  data: string | boolean | string[],
+): SchemaAbiTypeToPrimitiveType<abiType> {
+  const defaultValueType = typeof schemaAbiTypeToDefaultValue[abiType];
+  if (Array.isArray(data)) {
+    return data.map((element) => decodeDozerField(arrayToStaticAbiType(abiType as ArrayAbiType), element)) as never;
+  }
+  if (defaultValueType === "number") {
+    return Number(data) as never;
+  }
+  if (defaultValueType === "bigint") {
+    return BigInt(data) as never;
+  }
+  return data as never;
+}

packages/protocol-parser/src/decodeDozerRecords.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+import { decodeDozerRecords } from "./decodeDozerRecords";
+
+describe("decodeDozerRecord", () => {
+  const schema = {
+    address: { type: "address", internalType: "address" },
+    uint256: { type: "uint256", internalType: "uint256" },
+    uint32: { type: "uint32", internalType: "uint32" },
+    bool: { type: "bool", internalType: "bool" },
+    bytes: { type: "bytes", internalType: "bytes" },
+    string: { type: "string", internalType: "string" },
+    uint32Arr: { type: "uint32[]", internalType: "uint32[]" },
+  } as const;
+
+  it("decodes dozer record", () => {
+    const dozerRecord = [
+      "0x0000000000000000000000000000000000000000",
+      "1234",
+      "1234",
+      true,
+      "0x1234",
+      "hello world",
+      ["1234", "5678"],
+    ];
+    const decodedRecord = {
+      address: "0x0000000000000000000000000000000000000000",
+      uint256: 1234n,
+      uint32: 1234,
+      bool: true,
+      bytes: "0x1234",
+      string: "hello world",
+      uint32Arr: [1234, 5678],
+    };
+
+    const decoded = decodeDozerRecords({ schema, records: [dozerRecord] });
+    expect(decoded).toStrictEqual([decodedRecord]);
+  });
+});

packages/protocol-parser/src/decodeDozerRecords.ts

@@ -0,0 +1,44 @@
+import { Schema } from "@latticexyz/config";
+import { decodeDozerField } from "./decodeDozerField";
+import { getSchemaPrimitives } from "./getSchemaPrimitives";
+
+type DozerQueryHeader = string[];
+type DozerQueryRecord = (string | boolean | string[])[];
+
+// First item in the result is the header
+export type DozerQueryResult = [DozerQueryHeader, ...DozerQueryRecord[]];
+
+export type DecodeDozerRecordsArgs = {
+  schema: Schema;
+  records: DozerQueryResult;
+};
+
+/**
+ * Trim the header row from the query result
+ */
+function trimHeader(result: DozerQueryResult): DozerQueryRecord[] {
+  return result.slice(1);
+}
+
+export type DecodeDozerRecordsResult<schema extends Schema = Schema> = getSchemaPrimitives<schema>[];
+
+export function decodeDozerRecords<schema extends Schema>({
+  schema,
+  records,
+}: DecodeDozerRecordsArgs): DecodeDozerRecordsResult<schema> {
+  const fieldNames = Object.keys(schema);
+  if (records.length > 0 && fieldNames.length !== records[0].length) {
+    throw new Error(
+      `Mismatch between schema and query result.\nSchema: [${fieldNames.join(", ")}]\nQuery result: [${records[0].join(", ")}]`,
+    );
+  }
+
+  return trimHeader(records).map((record) =>
+    Object.fromEntries(
+      Object.keys(schema).map((fieldName, index) => [
+        fieldName,
+        decodeDozerField(schema[fieldName].type, record[index]),
+      ]),
+    ),
+  ) as never;
+}

packages/protocol-parser/src/exports/internal.ts

@@ -25,6 +25,11 @@ export * from "../schemaToHex";
 export * from "../staticDataLength";
 export * from "../valueSchemaToFieldLayoutHex";
 export * from "../valueSchemaToHex";
+export * from "../decodeDozerField";
+export * from "../decodeDozerRecords";
+
+export * from "../getKey";
+export * from "../getValue";
 
 export * from "../getKeySchema";
 export * from "../getValueSchema";

packages/protocol-parser/src/getKey.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vitest";
+import { getKey } from "./getKey";
+
+describe("getKey", () => {
+  it("should return the key fields of the record", () => {
+    const table = {
+      schema: {
+        key1: { type: "uint32", internalType: "uint32" },
+        key2: { type: "uint256", internalType: "uint256" },
+        value1: { type: "string", internalType: "string" },
+        value2: { type: "string", internalType: "string" },
+      },
+      key: ["key1", "key2"],
+    } as const;
+    const record = { key1: 1, key2: 2n, value1: "hello", value2: "world" };
+    const key = getKey(table, record);
+
+    expect(key).toMatchInlineSnapshot(`
+      {
+        "key1": 1,
+        "key2": 2n,
+      }
+    `);
+  });
+});

packages/protocol-parser/src/getKey.ts

@@ -0,0 +1,10 @@
+import { getKeySchema } from "./getKeySchema";
+import { getSchemaPrimitives } from "./getSchemaPrimitives";
+import { PartialTable } from "./common";
+
+export function getKey<table extends PartialTable>(
+  table: table,
+  record: getSchemaPrimitives<table["schema"]>,
+): getSchemaPrimitives<getKeySchema<table>> {
+  return Object.fromEntries(table.key.map((fieldName) => [fieldName, record[fieldName]])) as never;
+}

packages/protocol-parser/src/getKeySchema.ts

@@ -1,11 +1,13 @@
-import { Schema, Table } from "@latticexyz/config";
+import { KeySchema, StaticAbiType, Table } from "@latticexyz/config";
 
 type PartialTable = Pick<Table, "schema" | "key">;
 
 export type getKeySchema<table extends PartialTable> = PartialTable extends table
-  ? Schema
+  ? KeySchema
   : {
-      readonly [fieldName in Extract<keyof table["schema"], table["key"][number]>]: table["schema"][fieldName];
+      readonly [fieldName in Extract<keyof table["schema"], table["key"][number]>]: table["schema"][fieldName] & {
+        type: StaticAbiType;
+      };
     };
 
 export function getKeySchema<table extends PartialTable>(table: table): getKeySchema<table> {

packages/protocol-parser/src/getValue.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vitest";
+import { getValue } from "./getValue";
+
+describe("getValue", () => {
+  it("should return the key fields of the record", () => {
+    const table = {
+      schema: {
+        key1: { type: "uint32", internalType: "uint32" },
+        key2: { type: "uint256", internalType: "uint256" },
+        value1: { type: "string", internalType: "string" },
+        value2: { type: "string", internalType: "string" },
+      },
+      key: ["key1", "key2"],
+    } as const;
+    const record = { key1: 1, key2: 2n, value1: "hello", value2: "world" };
+    const value = getValue(table, record);
+
+    expect(value).toMatchInlineSnapshot(`
+      {
+        "value1": "hello",
+        "value2": "world",
+      }
+    `);
+  });
+});

packages/protocol-parser/src/getValue.ts

@@ -0,0 +1,14 @@
+import { PartialTable } from "./common";
+import { getValueSchema } from "./getValueSchema";
+import { getSchemaPrimitives } from "./getSchemaPrimitives";
+
+export function getValue<table extends PartialTable>(
+  table: table,
+  record: getSchemaPrimitives<table["schema"]>,
+): getSchemaPrimitives<getValueSchema<table>> {
+  return Object.fromEntries(
+    Object.keys(table.schema)
+      .filter((fieldName) => !table.key.includes(fieldName))
+      .map((fieldName) => [fieldName, record[fieldName]]),
+  ) as never;
+}

packages/stash/CHANGELOG.md

@@ -0,0 +1 @@
+# @latticexyz/stash

packages/stash/README.md

@@ -0,0 +1,3 @@
+# Stash
+
+High performance client store and query engine for MUD

packages/stash/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@latticexyz/stash",
+  "version": "2.0.12",
+  "description": "High performance client store and query engine for MUD",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/latticexyz/mud.git",
+    "directory": "packages/stash"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    ".": "./dist/index.js",
+    "./internal": "./dist/internal.js",
+    "./recs": "./dist/recs.js"
+  },
+  "typesVersions": {
+    "*": {
+      "index": [
+        "./dist/index.d.ts"
+      ],
+      "internal": [
+        "./dist/internal.d.ts"
+      ],
+      "recs": [
+        "./dist/recs.d.ts"
+      ]
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "bench": "tsx src/bench.ts",
+    "build": "tsup",
+    "clean": "rimraf dist",
+    "dev": "tsup --watch",
+    "test": "vitest typecheck --run --passWithNoTests && vitest --run --passWithNoTests",
+    "test:ci": "pnpm run test"
+  },
+  "dependencies": {
+    "@arktype/util": "0.0.40",
+    "@latticexyz/config": "workspace:*",
+    "@latticexyz/protocol-parser": "workspace:*",
+    "@latticexyz/recs": "workspace:*",
+    "@latticexyz/schema-type": "workspace:*",
+    "@latticexyz/store": "workspace:*",
+    "mutative": "^1.0.6",
+    "react": "^18.2.0",
+    "rxjs": "7.5.5",
+    "viem": "2.9.20",
+    "zustand": "^4.3.7",
+    "zustand-mutative": "^1.0.1"
+  },
+  "devDependencies": {
+    "@arktype/attest": "0.7.5",
+    "@testing-library/react": "^16.0.0",
+    "@testing-library/react-hooks": "^8.0.1",
+    "@types/react": "18.2.22",
+    "react-dom": "^18.2.0",
+    "tsup": "^6.7.0",
+    "vitest": "0.34.6"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/stash/src/README.md

@@ -0,0 +1,24 @@
+# TODOs
+
+- Set up performance benchmarks for setting records, reading records, running queries, updating records with subscribers
+- Replace objects with Maps for performance ? (https://rehmat-sayany.medium.com/using-map-over-objects-in-javascript-a-performance-benchmark-ff2f351851ed)
+  - could be useful for TableRecords, Keys
+- maybe add option to include records in the query result?
+- Maybe turn `entityKey` into a tagged string? So we could make it the return type of `encodeKey`,
+  and allow us to use Symbol (to reduce memory overhead) or something else later without breaking change
+- add more query fragments - ie GreaterThan, LessThan, Range, etc
+- we might be able to enable different key shapes if we add something like a `keySelector`
+- getKeySchema expects a full table as type, but only needs schema and key
+
+Ideas
+
+- Update streams:
+  - if each query added a new subscriber to the main state, each state update would trigger _every_ subscriber
+  - instead we could add a subscriber per table, which can be subscribed to again, so we only have to iterate through all tables once,
+    and then through all subscribers per table (only those who care about updates to this table)
+- Query fragments:
+  - Instead of pre-defined query types (Has, HasValue, Not, NotValue), could we define fragments in a self-contained way, so
+    it's easy to add a new type of query fragment without changing the core code?
+  - The main complexity is the logic to initialize the initial set with the first query fragment,
+    but it's probably not that critical - we could just run the first fragment on all entities of the first table,
+    unless an initialSet is provided.

packages/stash/src/actions/decodeKey.test.ts

@@ -0,0 +1,28 @@
+import { describe, it } from "vitest";
+import { createStash } from "../createStash";
+import { defineStore } from "@latticexyz/store/config/v2";
+import { setRecord } from "./setRecord";
+import { encodeKey } from "./encodeKey";
+import { attest } from "@ark/attest";
+import { decodeKey } from "./decodeKey";
+
+describe("decodeKey", () => {
+  it("should decode an encoded table key", () => {
+    const config = defineStore({
+      namespace: "namespace1",
+      tables: {
+        table1: {
+          schema: { field1: "string", field2: "uint32", field3: "uint256" },
+          key: ["field2", "field3"],
+        },
+      },
+    });
+    const stash = createStash(config);
+    const table = config.namespaces.namespace1.tables.table1;
+    const key = { field2: 1, field3: 2n };
+    setRecord({ stash, table, key, record: { field1: "hello" } });
+
+    const encodedKey = encodeKey({ table, key });
+    attest<typeof key>(decodeKey({ stash, table, encodedKey })).equals({ field2: 1, field3: 2n });
+  });
+});