feat(protocol-parser): add encoding/decoding functions

packages/protocol-parser/package.json

@@ -24,9 +24,7 @@
   },
   "dependencies": {
     "@latticexyz/common": "workspace:*",
-    "@latticexyz/config": "workspace:*",
     "@latticexyz/schema-type": "workspace:*",
-    "@latticexyz/store": "workspace:*",
     "abitype": "0.8.7",
     "viem": "1.1.7"
   },

packages/protocol-parser/src/common.ts

@@ -1,12 +1,11 @@
-import { Hex } from "viem";
 import { DynamicAbiType, StaticAbiType } from "@latticexyz/schema-type";
 
-export type Schema = Readonly<{
-  staticDataLength: number;
-  staticFields: StaticAbiType[];
-  dynamicFields: DynamicAbiType[];
-  isEmpty: boolean;
-  schemaData: Hex;
-}>;
+export type Schema = {
+  readonly staticFields: readonly StaticAbiType[];
+  readonly dynamicFields: readonly DynamicAbiType[];
+};
 
-export type TableSchema = { keySchema: Schema; valueSchema: Schema; isEmpty: boolean; schemaData: Hex };
+export type TableSchema = {
+  readonly keySchema: Schema; // TODO: refine to set dynamicFields to []
+  readonly valueSchema: Schema;
+};

packages/protocol-parser/src/decodeKeyTuple.test.ts

@@ -1,20 +1,24 @@
 import { describe, expect, it } from "vitest";
 import { decodeKeyTuple } from "./decodeKeyTuple";
-import { abiTypesToSchema } from "./abiTypesToSchema";
+import { Schema } from "./common";
 
 describe("decodeKeyTuple", () => {
   it("can decode bool key tuple", () => {
     expect(
-      decodeKeyTuple(abiTypesToSchema(["bool"]), ["0x0000000000000000000000000000000000000000000000000000000000000000"])
+      decodeKeyTuple({ staticFields: ["bool"], dynamicFields: [] }, [
+        "0x0000000000000000000000000000000000000000000000000000000000000000",
+      ])
     ).toStrictEqual([false]);
     expect(
-      decodeKeyTuple(abiTypesToSchema(["bool"]), ["0x0000000000000000000000000000000000000000000000000000000000000001"])
+      decodeKeyTuple({ staticFields: ["bool"], dynamicFields: [] }, [
+        "0x0000000000000000000000000000000000000000000000000000000000000001",
+      ])
     ).toStrictEqual([true]);
   });
 
   it("can decode complex key tuple", () => {
     expect(
-      decodeKeyTuple(abiTypesToSchema(["uint256", "int32", "bytes16", "address", "bool", "int8"]), [
+      decodeKeyTuple({ staticFields: ["uint256", "int32", "bytes16", "address", "bool", "int8"], dynamicFields: [] }, [
         "0x000000000000000000000000000000000000000000000000000000000000002a",
         "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd6",
         "0x1234000000000000000000000000000000000000000000000000000000000000",

packages/protocol-parser/src/decodeKeyTuple.ts

@@ -4,7 +4,7 @@ import { Schema } from "./common";
 
 // key tuples are encoded in the same way as abi.encode, so we can decode them with viem
 
-export function decodeKeyTuple(keySchema: Schema, keyTuple: Hex[]): StaticPrimitiveType[] {
+export function decodeKeyTuple(keySchema: Schema, keyTuple: readonly Hex[]): StaticPrimitiveType[] {
   return keyTuple.map(
     (key, index) => decodeAbiParameters([{ type: keySchema.staticFields[index] }], key)[0] as StaticPrimitiveType
   );

packages/protocol-parser/src/decodeRecord.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, it } from "vitest";
+import { decodeRecord } from "./decodeRecord";
+
+describe("decodeRecord", () => {
+  it("can decode hex to record values", () => {
+    const schema = { staticFields: ["uint32", "uint128"], dynamicFields: ["uint32[]", "string"] } as const;
+    const values = decodeRecord(
+      schema,
+      "0x0000000100000000000000000000000000000002000000000000130000000008000000000b0000000000000000000000000000000000000300000004736f6d6520737472696e67"
+    );
+    expect(values).toStrictEqual([1, 2n, [3, 4], "some string"]);
+  });
+});

packages/protocol-parser/src/decodeRecord.ts

@@ -0,0 +1,61 @@
+import { StaticPrimitiveType, DynamicPrimitiveType, staticAbiTypeToByteLength } from "@latticexyz/schema-type";
+import { Hex, sliceHex } from "viem";
+import { Schema } from "./common";
+import { decodeDynamicField } from "./decodeDynamicField";
+import { decodeStaticField } from "./decodeStaticField";
+import { hexToPackedCounter } from "./hexToPackedCounter";
+import { staticDataLength } from "./staticDataLength";
+
+export function decodeRecord(schema: Schema, data: Hex): readonly (StaticPrimitiveType | DynamicPrimitiveType)[] {
+  const values: (StaticPrimitiveType | DynamicPrimitiveType)[] = [];
+
+  let bytesOffset = 0;
+  schema.staticFields.forEach((fieldType) => {
+    const fieldByteLength = staticAbiTypeToByteLength[fieldType];
+    const value = decodeStaticField(fieldType, sliceHex(data, bytesOffset, bytesOffset + fieldByteLength));
+    bytesOffset += fieldByteLength;
+    values.push(value);
+  });
+
+  // Warn user if static data length doesn't match the schema, because data corruption might be possible.
+  const schemaStaticDataLength = staticDataLength(schema.staticFields);
+  const actualStaticDataLength = bytesOffset;
+  if (actualStaticDataLength !== schemaStaticDataLength) {
+    console.warn(
+      "Decoded static data length does not match schema's expected static data length. Data may get corrupted. Is `getStaticByteLength` outdated?",
+      {
+        expectedLength: schemaStaticDataLength,
+        actualLength: actualStaticDataLength,
+        bytesOffset,
+      }
+    );
+  }
+
+  if (schema.dynamicFields.length > 0) {
+    const dataLayout = hexToPackedCounter(sliceHex(data, bytesOffset, bytesOffset + 32));
+    bytesOffset += 32;
+
+    schema.dynamicFields.forEach((fieldType, i) => {
+      const dataLength = dataLayout.fieldByteLengths[i];
+      const value = decodeDynamicField(fieldType, sliceHex(data, bytesOffset, bytesOffset + dataLength));
+      bytesOffset += dataLength;
+      values.push(value);
+    });
+
+    // Warn user if dynamic data length doesn't match the schema, because data corruption might be possible.
+    const actualDynamicDataLength = bytesOffset - 32 - actualStaticDataLength;
+    // TODO: refactor this so we don't break for bytes offsets >UINT40
+    if (BigInt(actualDynamicDataLength) !== dataLayout.totalByteLength) {
+      console.warn(
+        "Decoded dynamic data length does not match data layout's expected data length. Data may get corrupted. Did the data layout change?",
+        {
+          expectedLength: dataLayout.totalByteLength,
+          actualLength: actualDynamicDataLength,
+          bytesOffset,
+        }
+      );
+    }
+  }
+
+  return values;
+}

packages/protocol-parser/src/encodeField.ts

@@ -0,0 +1,17 @@
+import { SchemaAbiType, arrayAbiTypeToStaticAbiType, isArrayAbiType } from "@latticexyz/schema-type";
+import { AbiParameterToPrimitiveType } from "abitype";
+import { Hex, encodePacked } from "viem";
+
+export function encodeField<TSchemaAbiType extends SchemaAbiType>(
+  fieldType: TSchemaAbiType,
+  value: AbiParameterToPrimitiveType<{ type: TSchemaAbiType }>
+): Hex {
+  if (isArrayAbiType(fieldType) && Array.isArray(value)) {
+    const staticFieldType = arrayAbiTypeToStaticAbiType(fieldType);
+    return encodePacked(
+      value.map(() => staticFieldType),
+      value
+    );
+  }
+  return encodePacked([fieldType], [value]);
+}

packages/protocol-parser/src/encodeRecord.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from "vitest";
+import { encodeRecord } from "./encodeRecord";
+
+describe("encodeRecord", () => {
+  it("can encode a schema and record values to hex", () => {
+    const schema = { staticFields: ["uint32", "uint128"], dynamicFields: ["uint32[]", "string"] } as const;
+    const hex = encodeRecord(schema, [1, 2n, [3, 4], "some string"]);
+    expect(hex).toBe(
+      "0x0000000100000000000000000000000000000002000000000000130000000008000000000b0000000000000000000000000000000000000300000004736f6d6520737472696e67"
+    );
+  });
+});

packages/protocol-parser/src/encodeRecord.ts

@@ -0,0 +1,28 @@
+import { StaticPrimitiveType, DynamicPrimitiveType } from "@latticexyz/schema-type";
+import { Hex } from "viem";
+import { encodeField } from "./encodeField";
+import { Schema } from "./common";
+
+export function encodeRecord(schema: Schema, values: readonly (StaticPrimitiveType | DynamicPrimitiveType)[]): Hex {
+  const staticValues = values.slice(0, schema.staticFields.length) as readonly StaticPrimitiveType[];
+  const dynamicValues = values.slice(schema.staticFields.length) as readonly DynamicPrimitiveType[];
+
+  const staticData = staticValues
+    .map((value, i) => encodeField(schema.staticFields[i], value).replace(/^0x/, ""))
+    .join("");
+
+  const dynamicDataItems = dynamicValues.map((value, i) =>
+    encodeField(schema.dynamicFields[i], value).replace(/^0x/, "")
+  );
+
+  const dynamicFieldByteLengths = dynamicDataItems.map((value) => value.length / 2);
+  const dynamicTotalByteLength = dynamicFieldByteLengths.reduce((total, length) => total + BigInt(length), 0n);
+
+  const dynamicData = dynamicDataItems.join("");
+
+  const packedCounter = `${encodeField("uint56", dynamicTotalByteLength).replace(/^0x/, "")}${dynamicFieldByteLengths
+    .map((length) => encodeField("uint40", length).replace(/^0x/, ""))
+    .join("")}`.padEnd(64, "0");
+
+  return `0x${staticData}${packedCounter}${dynamicData}`;
+}

packages/protocol-parser/src/hexToPackedCounter.ts

@@ -12,7 +12,7 @@ import { InvalidHexLengthForPackedCounterError, PackedCounterLengthMismatchError
 
 export function hexToPackedCounter(data: Hex): {
   totalByteLength: bigint;
-  fieldByteLengths: number[];
+  fieldByteLengths: readonly number[];
 } {
   if (data.length !== 66) {
     throw new InvalidHexLengthForPackedCounterError(data);
@@ -22,7 +22,7 @@ export function hexToPackedCounter(data: Hex): {
   // TODO: use schema to make sure we only parse as many as we need (rather than zeroes at the end)?
   const fieldByteLengths = decodeDynamicField("uint40[]", sliceHex(data, 7));
 
-  const summedLength = BigInt(fieldByteLengths.reduce((a, b) => a + b, 0));
+  const summedLength = BigInt(fieldByteLengths.reduce((total, length) => total + length, 0));
   if (summedLength !== totalByteLength) {
     throw new PackedCounterLengthMismatchError(data, totalByteLength, summedLength);
   }

packages/protocol-parser/src/hexToSchema.test.ts

@@ -2,28 +2,39 @@ import { describe, expect, it } from "vitest";
 import { hexToSchema } from "./hexToSchema";
 
 describe("hexToSchema", () => {
-  it("decodes schema hex data to schema", () => {
-    expect(hexToSchema("0x0001010060000000000000000000000000000000000000000000000000000000")).toStrictEqual({
-      staticDataLength: 1,
-      staticFields: ["bool"],
-      dynamicFields: [],
-      isEmpty: false,
-      schemaData: "0x0001010060000000000000000000000000000000000000000000000000000000",
-    });
-    expect(hexToSchema("0x0001010160c20000000000000000000000000000000000000000000000000000")).toStrictEqual({
-      staticDataLength: 1,
-      staticFields: ["bool"],
-      dynamicFields: ["bool[]"],
-      isEmpty: false,
-      schemaData: "0x0001010160c20000000000000000000000000000000000000000000000000000",
-    });
-    expect(hexToSchema("0x002402045f2381c3c4c500000000000000000000000000000000000000000000")).toStrictEqual({
-      staticDataLength: 36,
-      staticFields: ["bytes32", "int32"],
-      dynamicFields: ["uint256[]", "address[]", "bytes", "string"],
-      isEmpty: false,
-      schemaData: "0x002402045f2381c3c4c500000000000000000000000000000000000000000000",
-    });
+  it("converts hex to schema", () => {
+    expect(hexToSchema("0x0001010060000000000000000000000000000000000000000000000000000000")).toMatchInlineSnapshot(`
+      {
+        "dynamicFields": [],
+        "staticFields": [
+          "bool",
+        ],
+      }
+    `);
+    expect(hexToSchema("0x0001010160c20000000000000000000000000000000000000000000000000000")).toMatchInlineSnapshot(`
+      {
+        "dynamicFields": [
+          "bool[]",
+        ],
+        "staticFields": [
+          "bool",
+        ],
+      }
+    `);
+    expect(hexToSchema("0x002402045f2381c3c4c500000000000000000000000000000000000000000000")).toMatchInlineSnapshot(`
+      {
+        "dynamicFields": [
+          "uint256[]",
+          "address[]",
+          "bytes",
+          "string",
+        ],
+        "staticFields": [
+          "bytes32",
+          "int32",
+        ],
+      }
+    `);
   });
 
   it("throws if schema hex data is not bytes32", () => {

packages/protocol-parser/src/hexToSchema.ts

@@ -1,5 +1,5 @@
+import { StaticAbiType, DynamicAbiType, schemaAbiTypes, staticAbiTypeToByteLength } from "@latticexyz/schema-type";
 import { Hex, hexToNumber, sliceHex } from "viem";
-import { StaticAbiType, DynamicAbiType, staticAbiTypeToByteLength, schemaAbiTypes } from "@latticexyz/schema-type";
 import { Schema } from "./common";
 import { InvalidHexLengthForSchemaError, SchemaStaticLengthMismatchError } from "./errors";
 
@@ -29,11 +29,5 @@ export function hexToSchema(data: Hex): Schema {
     throw new SchemaStaticLengthMismatchError(data, staticDataLength, actualStaticDataLength);
   }
 
-  return {
-    staticDataLength,
-    staticFields,
-    dynamicFields,
-    isEmpty: data === "0x",
-    schemaData: data,
-  };
+  return { staticFields, dynamicFields };
 }

packages/protocol-parser/src/hexToTableSchema.ts

@@ -8,7 +8,5 @@ export function hexToTableSchema(data: Hex): TableSchema {
   return {
     keySchema,
     valueSchema,
-    isEmpty: data === "0x",
-    schemaData: data,
   };
 }