Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(protocol-parser): add encoding/decoding functions #1084

Merged
merged 10 commits into from
Jun 29, 2023
Merged
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions packages/protocol-parser/package.json
Original file line number Diff line number Diff line change
@@ -24,9 +24,7 @@
},
"dependencies": {
"@latticexyz/common": "workspace:*",
"@latticexyz/config": "workspace:*",
"@latticexyz/schema-type": "workspace:*",
"@latticexyz/store": "workspace:*",
"abitype": "0.8.7",
"viem": "1.1.7"
},
87 changes: 87 additions & 0 deletions packages/protocol-parser/src/Schema.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import { describe, expect, it } from "vitest";
import { Schema } from "./Schema";

describe("schema", () => {
it("can serialize to JSON", () => {
const emptySchema = new Schema([], []);
expect(JSON.stringify(emptySchema)).toMatchInlineSnapshot('"{\\"staticFields\\":[],\\"dynamicFields\\":[]}"');

const mixedSchema = new Schema(["uint8", "address", "bool"], ["bytes32[]", "string", "bytes", "bool[]"]);
expect(JSON.stringify(mixedSchema)).toMatchInlineSnapshot(
'"{\\"staticFields\\":[\\"uint8\\",\\"address\\",\\"bool\\"],\\"dynamicFields\\":[\\"bytes32[]\\",\\"string\\",\\"bytes\\",\\"bool[]\\"]}"'
);
});

it("converts schema to hex", () => {
expect(new Schema(["bool"]).toHex()).toBe("0x0001010060000000000000000000000000000000000000000000000000000000");
expect(new Schema(["bool"], ["bool[]"]).toHex()).toBe(
"0x0001010160c20000000000000000000000000000000000000000000000000000"
);
expect(new Schema(["bytes32", "int32"], ["uint256[]", "address[]", "bytes", "string"]).toHex()).toBe(
"0x002402045f2381c3c4c500000000000000000000000000000000000000000000"
);
});

it("converts hex to schema", () => {
expect(Schema.fromHex("0x0001010060000000000000000000000000000000000000000000000000000000")).toMatchInlineSnapshot(`
Schema {
"dynamicFields": [],
"staticFields": [
"bool",
],
}
`);
expect(Schema.fromHex("0x0001010160c20000000000000000000000000000000000000000000000000000")).toMatchInlineSnapshot(`
Schema {
"dynamicFields": [
"bool[]",
],
"staticFields": [
"bool",
],
}
`);
expect(Schema.fromHex("0x002402045f2381c3c4c500000000000000000000000000000000000000000000")).toMatchInlineSnapshot(`
Schema {
"dynamicFields": [
"uint256[]",
"address[]",
"bytes",
"string",
],
"staticFields": [
"bytes32",
"int32",
],
}
`);
});

it("throws if schema hex data is not bytes32", () => {
expect(() => Schema.fromHex("0x002502045f2381c3c4c5")).toThrow(
'Hex value "0x002502045f2381c3c4c5" has length of 20, but expected length of 64 for a schema.'
);
});

it("throws if schema static field lengths do not match", () => {
expect(() => Schema.fromHex("0x002502045f2381c3c4c500000000000000000000000000000000000000000000")).toThrow(
'Schema "0x002502045f2381c3c4c500000000000000000000000000000000000000000000" static data length (37) did not match the summed length of all static fields (36). Is `staticAbiTypeToByteLength` up to date with Solidity schema types?'
);
});

it("can encode a record values to hex", () => {
const schema = new Schema(["uint32", "uint128"], ["uint32[]", "string"]);
const hex = schema.encodeRecord([1, 2n, [3, 4], "some string"]);
expect(hex).toBe(
"0x0000000100000000000000000000000000000002000000000000130000000008000000000b0000000000000000000000000000000000000300000004736f6d6520737472696e67"
);
});

it("can decode hex to record values", () => {
const schema = new Schema(["uint32", "uint128"], ["uint32[]", "string"]);
const values = schema.decodeRecord(
"0x0000000100000000000000000000000000000002000000000000130000000008000000000b0000000000000000000000000000000000000300000004736f6d6520737472696e67"
);
expect(values).toStrictEqual([1, 2n, [3, 4], "some string"]);
});
});
160 changes: 160 additions & 0 deletions packages/protocol-parser/src/Schema.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
import {
StaticPrimitiveType,
DynamicPrimitiveType,
staticAbiTypeToByteLength,
DynamicAbiType,
StaticAbiType,
schemaAbiTypes,
} from "@latticexyz/schema-type";
import { Hex, hexToNumber, sliceHex } from "viem";
import { decodeDynamicField } from "./decodeDynamicField";
import { decodeStaticField } from "./decodeStaticField";
import { hexToPackedCounter } from "./hexToPackedCounter";
import { InvalidHexLengthForSchemaError, SchemaStaticLengthMismatchError } from "./errors";
import { encodeFieldData } from "./encodeField";

export class Schema {
readonly staticFields: readonly StaticAbiType[];
readonly dynamicFields: readonly DynamicAbiType[];

constructor(staticFields: readonly StaticAbiType[], dynamicFields: readonly DynamicAbiType[] = []) {
// TODO: validate at least one static field
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i believe there can also just be dynamic fields? are we even enforcing that there are any fields?

Copy link
Member Author

@holic holic Jun 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh I think you might be right, will remove the comment

would be good to have tests for no-static-fields and no-dynamic-fields though

// TODO: validate that static fields and dynamic fields are part of the possible abi types
this.staticFields = staticFields;
this.dynamicFields = dynamicFields;
Object.freeze(this);
}

static fromHex(data: Hex): Schema {
if (data.length !== 66) {
throw new InvalidHexLengthForSchemaError(data);
}

const staticDataLength = hexToNumber(sliceHex(data, 0, 2));
const numStaticFields = hexToNumber(sliceHex(data, 2, 3));
const numDynamicFields = hexToNumber(sliceHex(data, 3, 4));
const staticFields: StaticAbiType[] = [];
const dynamicFields: DynamicAbiType[] = [];

for (let i = 4; i < 4 + numStaticFields; i++) {
const schemaTypeIndex = hexToNumber(sliceHex(data, i, i + 1));
staticFields.push(schemaAbiTypes[schemaTypeIndex] as StaticAbiType);
}
for (let i = 4 + numStaticFields; i < 4 + numStaticFields + numDynamicFields; i++) {
const schemaTypeIndex = hexToNumber(sliceHex(data, i, i + 1));
dynamicFields.push(schemaAbiTypes[schemaTypeIndex] as DynamicAbiType);
}

// validate static data length
const actualStaticDataLength = staticFields.reduce(
(acc, fieldType) => acc + staticAbiTypeToByteLength[fieldType],
0
);
if (actualStaticDataLength !== staticDataLength) {
throw new SchemaStaticLengthMismatchError(data, staticDataLength, actualStaticDataLength);
}

return new Schema(staticFields, dynamicFields);
}

staticDataLength(): number {
return this.staticFields.reduce((length, fieldType) => length + staticAbiTypeToByteLength[fieldType], 0);
}

isEmpty(): boolean {
return this.staticFields.length === 0 && this.dynamicFields.length === 0;
}

toHex(): Hex {
const staticSchemaTypes = this.staticFields.map((abiType) => schemaAbiTypes.indexOf(abiType));
const dynamicSchemaTypes = this.dynamicFields.map((abiType) => schemaAbiTypes.indexOf(abiType));
return `0x${[
this.staticDataLength().toString(16).padStart(4, "0"),
this.staticFields.length.toString(16).padStart(2, "0"),
this.dynamicFields.length.toString(16).padStart(2, "0"),
...staticSchemaTypes.map((schemaType) => schemaType.toString(16).padStart(2, "0")),
...dynamicSchemaTypes.map((schemaType) => schemaType.toString(16).padStart(2, "0")),
]
.join("")
.padEnd(64, "0")}`;
}

decodeRecord(data: Hex): readonly (StaticPrimitiveType | DynamicPrimitiveType)[] {
const values: (StaticPrimitiveType | DynamicPrimitiveType)[] = [];

let bytesOffset = 0;
this.staticFields.forEach((fieldType) => {
const fieldByteLength = staticAbiTypeToByteLength[fieldType];
const value = decodeStaticField(fieldType, sliceHex(data, bytesOffset, bytesOffset + fieldByteLength));
bytesOffset += fieldByteLength;
values.push(value);
});

// Warn user if static data length doesn't match the schema, because data corruption might be possible.
const actualStaticDataLength = bytesOffset;
if (actualStaticDataLength !== this.staticDataLength()) {
console.warn(
"Decoded static data length does not match schema's expected static data length. Data may get corrupted. Is `getStaticByteLength` outdated?",
{
expectedLength: this.staticDataLength,
actualLength: actualStaticDataLength,
bytesOffset,
}
);
}

if (this.dynamicFields.length > 0) {
const dataLayout = hexToPackedCounter(sliceHex(data, bytesOffset, bytesOffset + 32));
bytesOffset += 32;

this.dynamicFields.forEach((fieldType, i) => {
const dataLength = dataLayout.fieldByteLengths[i];
const value = decodeDynamicField(fieldType, sliceHex(data, bytesOffset, bytesOffset + dataLength));
bytesOffset += dataLength;
values.push(value);
});

// Warn user if dynamic data length doesn't match the schema, because data corruption might be possible.
const actualDynamicDataLength = bytesOffset - 32 - actualStaticDataLength;
// TODO: refactor this so we don't break for bytes offsets >UINT40
if (BigInt(actualDynamicDataLength) !== dataLayout.totalByteLength) {
console.warn(
"Decoded dynamic data length does not match data layout's expected data length. Data may get corrupted. Did the data layout change?",
{
expectedLength: dataLayout.totalByteLength,
actualLength: actualDynamicDataLength,
bytesOffset,
}
);
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does this TODO mean? is the refactor to make bytesOffset a bigint instead of a number? is there a downside to already do that now?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think since this is new code, I can probably do that now. This was copied over from another area of the codebase, where it was a much bigger refactor.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ahh I remember now: viem's sliceHex operations expect a number, so that's why bytesOffset is a number

moving it to bigint means that I can't use sliceHex unless downcasting, which has the same potential issue.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

uint40 max is 1099511627775, which means we'd need a 1TB string to hit this issue. Feels like we could punt on this for now.

}

return values;
}

decodeField(fieldIndex: number, data: Hex): StaticPrimitiveType | DynamicPrimitiveType {
return fieldIndex < this.staticFields.length
? decodeStaticField(this.staticFields[fieldIndex], data)
: decodeDynamicField(this.dynamicFields[fieldIndex - this.staticFields.length], data);
}

encodeRecord(values: readonly (StaticPrimitiveType | DynamicPrimitiveType)[]): Hex {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh nice, we need this for #1074 too!

const staticValues = values.slice(0, this.staticFields.length) as readonly StaticPrimitiveType[];
const dynamicValues = values.slice(this.staticFields.length) as readonly DynamicPrimitiveType[];

const staticData = staticValues.map((value, i) => encodeFieldData(this.staticFields[i], value)).join("");

const dynamicDataItems = dynamicValues.map((value, i) => encodeFieldData(this.dynamicFields[i], value));

const dynamicFieldByteLengths = dynamicDataItems.map((value) => value.length / 2);
const dynamicTotalByteLength = dynamicFieldByteLengths.reduce((total, length) => total + BigInt(length), 0n);

const dynamicData = dynamicDataItems.join("");

const packedCounter = `${encodeFieldData("uint56", dynamicTotalByteLength)}${dynamicFieldByteLengths
.map((length) => encodeFieldData("uint40", length))
.join("")}`.padEnd(64, "0");

return `0x${staticData}${packedCounter}${dynamicData}`;
}
}
28 changes: 0 additions & 28 deletions packages/protocol-parser/src/abiTypesToSchema.test.ts

This file was deleted.

14 changes: 0 additions & 14 deletions packages/protocol-parser/src/abiTypesToSchema.ts

This file was deleted.

14 changes: 0 additions & 14 deletions packages/protocol-parser/src/abiTypesToSchemaData.test.ts

This file was deleted.

19 changes: 0 additions & 19 deletions packages/protocol-parser/src/abiTypesToSchemaData.ts

This file was deleted.

16 changes: 5 additions & 11 deletions packages/protocol-parser/src/common.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,6 @@
import { Hex } from "viem";
import { DynamicAbiType, StaticAbiType } from "@latticexyz/schema-type";
import { Schema } from "./Schema";

export type Schema = Readonly<{
staticDataLength: number;
staticFields: StaticAbiType[];
dynamicFields: DynamicAbiType[];
isEmpty: boolean;
schemaData: Hex;
}>;

export type TableSchema = { keySchema: Schema; valueSchema: Schema; isEmpty: boolean; schemaData: Hex };
export type TableSchema = {
keySchema: Schema;
valueSchema: Schema;
};
8 changes: 4 additions & 4 deletions packages/protocol-parser/src/decodeKeyTuple.test.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
import { describe, expect, it } from "vitest";
import { decodeKeyTuple } from "./decodeKeyTuple";
import { abiTypesToSchema } from "./abiTypesToSchema";
import { Schema } from "./Schema";

describe("decodeKeyTuple", () => {
it("can decode bool key tuple", () => {
expect(
decodeKeyTuple(abiTypesToSchema(["bool"]), ["0x0000000000000000000000000000000000000000000000000000000000000000"])
decodeKeyTuple(new Schema(["bool"]), ["0x0000000000000000000000000000000000000000000000000000000000000000"])
).toStrictEqual([false]);
expect(
decodeKeyTuple(abiTypesToSchema(["bool"]), ["0x0000000000000000000000000000000000000000000000000000000000000001"])
decodeKeyTuple(new Schema(["bool"]), ["0x0000000000000000000000000000000000000000000000000000000000000001"])
).toStrictEqual([true]);
});

it("can decode complex key tuple", () => {
expect(
decodeKeyTuple(abiTypesToSchema(["uint256", "int32", "bytes16", "address", "bool", "int8"]), [
decodeKeyTuple(new Schema(["uint256", "int32", "bytes16", "address", "bool", "int8"]), [
"0x000000000000000000000000000000000000000000000000000000000000002a",
"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffd6",
"0x1234000000000000000000000000000000000000000000000000000000000000",
4 changes: 2 additions & 2 deletions packages/protocol-parser/src/decodeKeyTuple.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { Hex, decodeAbiParameters } from "viem";
import { StaticPrimitiveType } from "@latticexyz/schema-type";
import { Schema } from "./common";
import { Schema } from "./Schema";

// key tuples are encoded in the same way as abi.encode, so we can decode them with viem

export function decodeKeyTuple(keySchema: Schema, keyTuple: Hex[]): StaticPrimitiveType[] {
export function decodeKeyTuple(keySchema: Schema, keyTuple: readonly Hex[]): StaticPrimitiveType[] {
return keyTuple.map(
(key, index) => decodeAbiParameters([{ type: keySchema.staticFields[index] }], key)[0] as StaticPrimitiveType
);
Loading