feat: protocol-parser in go

.changeset/big-goats-prove.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/services": patch
+---
+
+protocol-parser in Go

packages/services/go.mod

@@ -1,26 +1,27 @@
 module latticexyz/mud/packages/services
 
-go 1.18
+go 1.20
 
 replace github.com/ethereum/go-ethereum v1.10.21 => github.com/ethereum-optimism/op-geth v1.101105.1
 
 require (
+	github.com/andriidski/abiencode-go v0.0.0-20230709002942-ad6b06528e3e
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/dghubble/go-twitter v0.0.0-20220816163853-8a0df96f1e6d
-	github.com/ethereum/go-ethereum v1.10.26
+	github.com/ethereum/go-ethereum v1.12.0
+	github.com/golodash/godash v1.3.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/improbable-eng/grpc-web v0.15.0
 	github.com/jackc/pglogrepl v0.0.0-20221125161640-1e972958ba03
 	github.com/jackc/pgx/v5 v5.3.1
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/keith-turner/ecoji v1.0.0
 	github.com/lib/pq v1.10.2
-	github.com/ompluscator/dynamic-struct v1.4.0
-	github.com/prometheus/client_golang v1.11.1
+	github.com/prometheus/client_golang v1.14.0
 	github.com/umbracle/ethgo v0.1.3
 	go.uber.org/zap v1.22.0
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+	golang.org/x/oauth2 v0.3.0
+	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af
 	google.golang.org/grpc v1.48.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -33,30 +34,31 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/deckarep/golang-set v1.8.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
 	github.com/dghubble/sling v1.4.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-ole/go-ole v1.2.1 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
-	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
@@ -69,7 +71,7 @@ require (
 	golang.org/x/net v0.9.0 // indirect
 	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	nhooyr.io/websocket v1.8.6 // indirect

packages/services/package.json

@@ -41,7 +41,8 @@
     "build:js": "tsup",
     "build:protobuf": "make protoc-ts && prettier --write protobuf/ts/**/*.ts",
     "dev": "tsup --watch",
-    "test": "tsc --noEmit && echo 'todo: add tests'"
+    "test": "pnpm run test:go",
+    "test:go": "go test -v ./..."
   },
   "dependencies": {
     "long": "^5.2.1",

packages/services/pkg/protocol-parser/abiTypesToSchema.go

@@ -0,0 +1,21 @@
+package protocolparser
+
+import schematype "latticexyz/mud/packages/services/pkg/schema-type"
+
+func AbiTypesToSchema(abiTypes []schematype.SchemaType) Schema {
+	staticFields := []schematype.SchemaType{}
+	dynamicFields := []schematype.SchemaType{}
+
+	for _, abiType := range abiTypes {
+		if abiType.IsDynamic() {
+			dynamicFields = append(dynamicFields, abiType)
+		} else {
+			staticFields = append(staticFields, abiType)
+		}
+	}
+
+	return Schema{
+		StaticFields:  staticFields,
+		DynamicFields: dynamicFields,
+	}
+}

packages/services/pkg/protocol-parser/abiTypesToSchema_test.go

@@ -0,0 +1,64 @@
+package protocolparser_test
+
+import (
+	protocolparser "latticexyz/mud/packages/services/pkg/protocol-parser"
+	. "latticexyz/mud/packages/services/pkg/schema-type"
+	"reflect"
+	"testing"
+)
+
+func TestAbiTypesToSchema(testing *testing.T) {
+	abiTypesList := [][]SchemaType{
+		{
+			BOOL,
+		},
+		{
+			BOOL,
+			BOOL_ARRAY,
+		},
+		{
+			BYTES32,
+			INT32,
+			UINT256_ARRAY,
+			ADDRESS_ARRAY,
+			BYTES,
+			STRING,
+		},
+	}
+
+	expectedSchemas := []protocolparser.Schema{
+		{
+			StaticFields: []SchemaType{
+				BOOL,
+			},
+			DynamicFields: []SchemaType{},
+		},
+		{
+			StaticFields: []SchemaType{
+				BOOL,
+			},
+			DynamicFields: []SchemaType{
+				BOOL_ARRAY,
+			},
+		},
+		{
+			StaticFields: []SchemaType{
+				BYTES32,
+				INT32,
+			},
+			DynamicFields: []SchemaType{
+				UINT256_ARRAY,
+				ADDRESS_ARRAY,
+				BYTES,
+				STRING,
+			},
+		},
+	}
+
+	for i, abiTypes := range abiTypesList {
+		schema := protocolparser.AbiTypesToSchema(abiTypes)
+		if !reflect.DeepEqual(schema, expectedSchemas[i]) {
+			testing.Errorf("expected schema to be %v, got %v", expectedSchemas[i], schema)
+		}
+	}
+}

packages/services/pkg/protocol-parser/common.go

@@ -0,0 +1,13 @@
+package protocolparser
+
+import schematype "latticexyz/mud/packages/services/pkg/schema-type"
+
+type Schema struct {
+	StaticFields  []schematype.SchemaType
+	DynamicFields []schematype.SchemaType
+}
+
+type TableSchema struct {
+	KeySchema   Schema
+	ValueSchema Schema
+}

packages/services/pkg/protocol-parser/decodeDynamicField.go

@@ -0,0 +1,137 @@
+package protocolparser
+
+import (
+	. "latticexyz/mud/packages/services/pkg/schema-type"
+
+	"github.com/andriidski/abiencode-go/convert"
+)
+
+func DecodeDynamicField(schemaType SchemaType, data string) interface{} {
+	if schemaType == BYTES {
+		return convert.HexToBytes(data)
+	}
+	if schemaType == STRING {
+		return convert.HexToString(data)
+	}
+
+	if len(data) > 3 && len(data)%2 != 0 {
+		panic(ErrInvalidHexLength)
+	}
+
+	dataSize := (len(data) - 2) / 2
+
+	switch schemaType {
+	case UINT8_ARRAY,
+		UINT16_ARRAY,
+		UINT24_ARRAY,
+		UINT32_ARRAY,
+		UINT40_ARRAY,
+		UINT48_ARRAY,
+		UINT56_ARRAY,
+		UINT64_ARRAY,
+		UINT72_ARRAY,
+		UINT80_ARRAY,
+		UINT88_ARRAY,
+		UINT96_ARRAY,
+		UINT104_ARRAY,
+		UINT112_ARRAY,
+		UINT120_ARRAY,
+		UINT128_ARRAY,
+		UINT136_ARRAY,
+		UINT144_ARRAY,
+		UINT152_ARRAY,
+		UINT160_ARRAY,
+		UINT168_ARRAY,
+		UINT176_ARRAY,
+		UINT184_ARRAY,
+		UINT192_ARRAY,
+		UINT200_ARRAY,
+		UINT208_ARRAY,
+		UINT216_ARRAY,
+		UINT224_ARRAY,
+		UINT232_ARRAY,
+		UINT240_ARRAY,
+		UINT248_ARRAY,
+		UINT256_ARRAY,
+		INT8_ARRAY,
+		INT16_ARRAY,
+		INT24_ARRAY,
+		INT32_ARRAY,
+		INT40_ARRAY,
+		INT48_ARRAY,
+		INT56_ARRAY,
+		INT64_ARRAY,
+		INT72_ARRAY,
+		INT80_ARRAY,
+		INT88_ARRAY,
+		INT96_ARRAY,
+		INT104_ARRAY,
+		INT112_ARRAY,
+		INT120_ARRAY,
+		INT128_ARRAY,
+		INT136_ARRAY,
+		INT144_ARRAY,
+		INT152_ARRAY,
+		INT160_ARRAY,
+		INT168_ARRAY,
+		INT176_ARRAY,
+		INT184_ARRAY,
+		INT192_ARRAY,
+		INT200_ARRAY,
+		INT208_ARRAY,
+		INT216_ARRAY,
+		INT224_ARRAY,
+		INT232_ARRAY,
+		INT240_ARRAY,
+		INT248_ARRAY,
+		INT256_ARRAY,
+		BYTES1_ARRAY,
+		BYTES2_ARRAY,
+		BYTES3_ARRAY,
+		BYTES4_ARRAY,
+		BYTES5_ARRAY,
+		BYTES6_ARRAY,
+		BYTES7_ARRAY,
+		BYTES8_ARRAY,
+		BYTES9_ARRAY,
+		BYTES10_ARRAY,
+		BYTES11_ARRAY,
+		BYTES12_ARRAY,
+		BYTES13_ARRAY,
+		BYTES14_ARRAY,
+		BYTES15_ARRAY,
+		BYTES16_ARRAY,
+		BYTES17_ARRAY,
+		BYTES18_ARRAY,
+		BYTES19_ARRAY,
+		BYTES20_ARRAY,
+		BYTES21_ARRAY,
+		BYTES22_ARRAY,
+		BYTES23_ARRAY,
+		BYTES24_ARRAY,
+		BYTES25_ARRAY,
+		BYTES26_ARRAY,
+		BYTES27_ARRAY,
+		BYTES28_ARRAY,
+		BYTES29_ARRAY,
+		BYTES30_ARRAY,
+		BYTES31_ARRAY,
+		BYTES32_ARRAY,
+		BOOL_ARRAY,
+		ADDRESS_ARRAY:
+		staticAbiType := ArrayAbiTypeToStaticAbiType(schemaType)
+		itemByteLength := staticAbiType.ByteLength()
+		if dataSize%itemByteLength != 0 {
+			panic(ErrInvalidHexLengthForArrayField)
+		}
+		itemCount := dataSize / int(itemByteLength)
+		items := make([]interface{}, itemCount)
+		for i := 0; i < itemCount; i++ {
+			itemData := HexSlice(data, i*itemByteLength, (i+1)*itemByteLength)
+			items[i] = DecodeStaticField(staticAbiType, itemData)
+		}
+		return items
+	default:
+		panic("unsupported type")
+	}
+}