From a7fc7b04c38ba0bc19aa1afe70ee5150760b6524 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Fri, 30 Sep 2022 13:40:01 +0530 Subject: [PATCH 1/9] scale encoding support for map --- pkg/scale/encode.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pkg/scale/encode.go b/pkg/scale/encode.go index 614581fedb..e0240853cd 100644 --- a/pkg/scale/encode.go +++ b/pkg/scale/encode.go @@ -106,6 +106,8 @@ func (es *encodeState) marshal(in interface{}) (err error) { err = es.encodeArray(in) case reflect.Slice: err = es.encodeSlice(in) + case reflect.Map: + err = es.encodeMap(in) default: err = fmt.Errorf("unsupported type: %T", in) } @@ -223,6 +225,35 @@ func (es *encodeState) encodeArray(in interface{}) (err error) { return } +func (es *encodeState) encodeMap(in interface{}) (err error) { + v := reflect.ValueOf(in) + err = es.encodeLength(v.Len()) + if err != nil { + return + } + + for i := v.MapRange(); i.Next(); { + fmt.Println(i.Key(), "\t:", i.Value()) + + key := i.Key() + err = es.marshal(key.Interface()) + if err != nil { + return + } + + mapValue := i.Value() + if !mapValue.CanInterface() { + continue + } + + err = es.marshal(mapValue.Interface()) + if err != nil { + return + } + } + return +} + // encodeBigInt performs the same encoding as encodeInteger, except on a big.Int. // if 2^30 <= n < 2^536 write // [lower 2 bits of first byte = 11] [upper 6 bits of first byte = # of bytes following less 4] From 27ead7c9582b5b02e2463683c81bb5dac480a118 Mon Sep 17 00:00:00 2001 From: Axay sagathiya Date: Mon, 17 Oct 2022 18:14:15 +0530 Subject: [PATCH 2/9] test for encodeMap method. --- pkg/scale/encode_test.go | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/pkg/scale/encode_test.go b/pkg/scale/encode_test.go index fd43c17201..1a9462be71 100644 --- a/pkg/scale/encode_test.go +++ b/pkg/scale/encode_test.go @@ -909,9 +909,25 @@ var ( }, } + mapTests = tests{ + { + name: "testMap1", + in: map[int8][]byte{2: []byte("some string")}, + want: []byte{4, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103}, + }, + { + name: "testMap2", + in: map[int8][]byte{ + 2: []byte("some string"), + 16: []byte("lorem ipsum"), + }, + want: []byte{8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, 105, 112, 115, 117, 109}, + }, + } + allTests = newTests( fixedWidthIntegerTests, variableWidthIntegerTests, stringTests, - boolTests, structTests, sliceTests, arrayTests, + boolTests, structTests, sliceTests, arrayTests, mapTests, varyingDataTypeTests, ) ) @@ -1096,6 +1112,24 @@ func Test_encodeState_encodeArray(t *testing.T) { } } +func Test_encodeState_encodeMap(t *testing.T) { + for _, tt := range mapTests { + t.Run(tt.name, func(t *testing.T) { + buffer := bytes.NewBuffer(nil) + es := &encodeState{ + Writer: buffer, + fieldScaleIndicesCache: cache, + } + if err := es.marshal(tt.in); (err != nil) != tt.wantErr { + t.Errorf("encodeState.encodeMap() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(buffer.Bytes(), tt.want) { + t.Errorf("encodeState.encodeMap() = %v, want %v", buffer.Bytes(), tt.want) + } + }) + } +} + func Test_marshal_optionality(t *testing.T) { var ptrTests tests for i := range allTests { From 868ca44aba385001c95e1f2af0e2c1e2254eb917 Mon Sep 17 00:00:00 2001 From: Axay sagathiya Date: Mon, 17 Oct 2022 18:18:55 +0530 Subject: [PATCH 3/9] scale decoding support for map. --- pkg/scale/decode.go | 30 ++++++++++++++++++++++++++++++ pkg/scale/decode_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/pkg/scale/decode.go b/pkg/scale/decode.go index e10cf4c4e6..3593f5b2ce 100644 --- a/pkg/scale/decode.go +++ b/pkg/scale/decode.go @@ -159,6 +159,8 @@ func (ds *decodeState) unmarshal(dstv reflect.Value) (err error) { err = ds.decodeArray(dstv) case reflect.Slice: err = ds.decodeSlice(dstv) + case reflect.Map: + err = ds.decodeMap(dstv) default: err = fmt.Errorf("unsupported type: %T", in) } @@ -426,6 +428,34 @@ func (ds *decodeState) decodeArray(dstv reflect.Value) (err error) { return } +func (ds *decodeState) decodeMap(dstv reflect.Value) (err error) { + l, err := ds.decodeLength() + if err != nil { + return + } + in := dstv.Interface() + + for i := uint(0); i < l; i++ { + tempKeyType := reflect.TypeOf(in).Key() + tempKey := reflect.New(tempKeyType).Elem() + err = ds.unmarshal(tempKey) + if err != nil { + return + } + + tempElemType := reflect.TypeOf(in).Elem() + tempElem := reflect.New(tempElemType).Elem() + err = ds.unmarshal(tempElem) + if err != nil { + return + } + + dstv.SetMapIndex(tempKey, tempElem) + } + + return +} + // decodeStruct decodes a byte array representing a SCALE tuple. The order of data is // determined by the source tuple in rust, or the struct field order in a go struct func (ds *decodeState) decodeStruct(dstv reflect.Value) (err error) { diff --git a/pkg/scale/decode_test.go b/pkg/scale/decode_test.go index ac644871ce..dfc0f4edf3 100644 --- a/pkg/scale/decode_test.go +++ b/pkg/scale/decode_test.go @@ -132,6 +132,37 @@ func Test_decodeState_decodeSlice(t *testing.T) { } } +func Test_decodeState_decodeMap(t *testing.T) { + mapTests = tests{ + { + name: "testMap1", + in: map[int8][]byte{2: []byte("some string")}, + want: []byte{4, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103}, + }, + { + name: "testMap2", + in: map[int8][]byte{ + 2: []byte("some string"), + 16: []byte("lorem ipsum"), + }, + want: []byte{8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, 105, 112, 115, 117, 109}, + }, + } + + for _, tt := range mapTests { + t.Run(tt.name, func(t *testing.T) { + dst := map[int8][]byte{} + if err := Unmarshal(tt.want, &dst); (err != nil) != tt.wantErr { + t.Errorf("decodeState.unmarshal() error = %v, wantErr %v", err, tt.wantErr) + } + + if !reflect.DeepEqual(dst, tt.in) { + t.Errorf("decodeState.unmarshal() = %v, want %v", dst, tt.in) + } + }) + } +} + func Test_unmarshal_optionality(t *testing.T) { var ptrTests tests for _, t := range append(tests{}, allTests...) { From 12009fe511688a4ef4ce5e5e8ed61053316953eb Mon Sep 17 00:00:00 2001 From: Axay sagathiya Date: Mon, 17 Oct 2022 18:45:24 +0530 Subject: [PATCH 4/9] lint --- pkg/scale/decode_test.go | 5 ++++- pkg/scale/encode_test.go | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/scale/decode_test.go b/pkg/scale/decode_test.go index dfc0f4edf3..a82dccf929 100644 --- a/pkg/scale/decode_test.go +++ b/pkg/scale/decode_test.go @@ -145,7 +145,10 @@ func Test_decodeState_decodeMap(t *testing.T) { 2: []byte("some string"), 16: []byte("lorem ipsum"), }, - want: []byte{8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, 105, 112, 115, 117, 109}, + want: []byte{ + 8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, + 105, 112, 115, 117, 109, + }, }, } diff --git a/pkg/scale/encode_test.go b/pkg/scale/encode_test.go index 1a9462be71..476f651b02 100644 --- a/pkg/scale/encode_test.go +++ b/pkg/scale/encode_test.go @@ -921,8 +921,10 @@ var ( 2: []byte("some string"), 16: []byte("lorem ipsum"), }, - want: []byte{8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, 105, 112, 115, 117, 109}, - }, + want: []byte{ + 8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, + 105, 112, 115, 117, 109, + }}, } allTests = newTests( From 00c8133de04931c8f4d40cb335764fdda3f34dd7 Mon Sep 17 00:00:00 2001 From: Axay sagathiya Date: Tue, 18 Oct 2022 18:07:34 +0530 Subject: [PATCH 5/9] specify the returned err values and fix the test. --- pkg/scale/decode.go | 12 ++++++------ pkg/scale/decode_test.go | 10 +++++++++- pkg/scale/encode.go | 16 +++++++--------- pkg/scale/encode_test.go | 4 +++- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/pkg/scale/decode.go b/pkg/scale/decode.go index 3593f5b2ce..f86d7eb8bd 100644 --- a/pkg/scale/decode.go +++ b/pkg/scale/decode.go @@ -429,31 +429,31 @@ func (ds *decodeState) decodeArray(dstv reflect.Value) (err error) { } func (ds *decodeState) decodeMap(dstv reflect.Value) (err error) { - l, err := ds.decodeLength() + numberOfTuples, err := ds.decodeLength() if err != nil { - return + return fmt.Errorf("decoding length: %w", err) } in := dstv.Interface() - for i := uint(0); i < l; i++ { + for i := uint(0); i < numberOfTuples; i++ { tempKeyType := reflect.TypeOf(in).Key() tempKey := reflect.New(tempKeyType).Elem() err = ds.unmarshal(tempKey) if err != nil { - return + return fmt.Errorf("decoding key %d of %d: %w", i+1, numberOfTuples, err) } tempElemType := reflect.TypeOf(in).Elem() tempElem := reflect.New(tempElemType).Elem() err = ds.unmarshal(tempElem) if err != nil { - return + return fmt.Errorf("decoding value %d of %d: %w", i+1, numberOfTuples, err) } dstv.SetMapIndex(tempKey, tempElem) } - return + return nil } // decodeStruct decodes a byte array representing a SCALE tuple. The order of data is diff --git a/pkg/scale/decode_test.go b/pkg/scale/decode_test.go index a82dccf929..c0c83afcc1 100644 --- a/pkg/scale/decode_test.go +++ b/pkg/scale/decode_test.go @@ -153,6 +153,7 @@ func Test_decodeState_decodeMap(t *testing.T) { } for _, tt := range mapTests { + tt := tt t.Run(tt.name, func(t *testing.T) { dst := map[int8][]byte{} if err := Unmarshal(tt.want, &dst); (err != nil) != tt.wantErr { @@ -201,7 +202,14 @@ func Test_unmarshal_optionality(t *testing.T) { t.Errorf("decodeState.unmarshal() = %s", diff) } default: - dst := reflect.New(reflect.TypeOf(tt.in)).Interface() + var dst interface{} + + if reflect.TypeOf(tt.in).Kind().String() == "map" { + dst = &(map[int8][]byte{}) + } else { + dst = reflect.New(reflect.TypeOf(tt.in)).Interface() + } + if err := Unmarshal(tt.want, &dst); (err != nil) != tt.wantErr { t.Errorf("decodeState.unmarshal() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/scale/encode.go b/pkg/scale/encode.go index e0240853cd..3f73703c23 100644 --- a/pkg/scale/encode.go +++ b/pkg/scale/encode.go @@ -229,29 +229,27 @@ func (es *encodeState) encodeMap(in interface{}) (err error) { v := reflect.ValueOf(in) err = es.encodeLength(v.Len()) if err != nil { - return + return fmt.Errorf("encoding length: %w", err) } - for i := v.MapRange(); i.Next(); { - fmt.Println(i.Key(), "\t:", i.Value()) - - key := i.Key() + for keyValue := v.MapRange(); keyValue.Next(); { + key := keyValue.Key() err = es.marshal(key.Interface()) if err != nil { - return + return fmt.Errorf("encoding map key: %w", err) } - mapValue := i.Value() + mapValue := keyValue.Value() if !mapValue.CanInterface() { continue } err = es.marshal(mapValue.Interface()) if err != nil { - return + return fmt.Errorf("encoding map value: %w", err) } } - return + return nil } // encodeBigInt performs the same encoding as encodeInteger, except on a big.Int. diff --git a/pkg/scale/encode_test.go b/pkg/scale/encode_test.go index 476f651b02..6864c256de 100644 --- a/pkg/scale/encode_test.go +++ b/pkg/scale/encode_test.go @@ -924,7 +924,8 @@ var ( want: []byte{ 8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, 105, 112, 115, 117, 109, - }}, + }, + }, } allTests = newTests( @@ -1116,6 +1117,7 @@ func Test_encodeState_encodeArray(t *testing.T) { func Test_encodeState_encodeMap(t *testing.T) { for _, tt := range mapTests { + tt := tt t.Run(tt.name, func(t *testing.T) { buffer := bytes.NewBuffer(nil) es := &encodeState{ From 10c98f4df0edf3908e2b22308a5efb02ce1ab49a Mon Sep 17 00:00:00 2001 From: Axay sagathiya Date: Wed, 19 Oct 2022 13:10:01 +0530 Subject: [PATCH 6/9] add test case. --- pkg/scale/decode_test.go | 108 ++++++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 14 deletions(-) diff --git a/pkg/scale/decode_test.go b/pkg/scale/decode_test.go index c0c83afcc1..d97dcaac73 100644 --- a/pkg/scale/decode_test.go +++ b/pkg/scale/decode_test.go @@ -132,36 +132,116 @@ func Test_decodeState_decodeSlice(t *testing.T) { } } +// // Rust code to encode a map of string to struct. +// let mut btree_map: BTreeMap = BTreeMap::new(); +// match btree_map.entry("string1".to_string()) { +// Entry::Vacant(entry) => { +// entry.insert(User{ +// active: true, +// username: "lorem".to_string(), +// email: "lorem@ipsum.org".to_string(), +// sign_in_count: 1, +// }); +// () +// }, +// Entry::Occupied(_) => (), +// } +// match btree_map.entry("string2".to_string()) { +// Entry::Vacant(entry) => { +// entry.insert(User{ +// active: false, +// username: "john".to_string(), +// email: "jack@gmail.com".to_string(), +// sign_in_count: 73, +// }); +// () +// }, +// Entry::Occupied(_) => (), +// } +// println!("{:?}", btree_map.encode()); + +type user struct { + Active bool + Username string + Email string + SignInCount uint64 +} + func Test_decodeState_decodeMap(t *testing.T) { - mapTests = tests{ + mapTests1 := []struct { + name string + input []byte + wantErr bool + expectedOutput map[int8][]byte + }{ { - name: "testMap1", - in: map[int8][]byte{2: []byte("some string")}, - want: []byte{4, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103}, + name: "testing a map of int8 to a byte array 1", + input: []byte{4, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103}, + expectedOutput: map[int8][]byte{2: []byte("some string")}, }, { - name: "testMap2", - in: map[int8][]byte{ + name: "testing a map of int8 to a byte array 2", + input: []byte{ + 8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, + 105, 112, 115, 117, 109, + }, + expectedOutput: map[int8][]byte{ 2: []byte("some string"), 16: []byte("lorem ipsum"), }, - want: []byte{ - 8, 2, 44, 115, 111, 109, 101, 32, 115, 116, 114, 105, 110, 103, 16, 44, 108, 111, 114, 101, 109, 32, - 105, 112, 115, 117, 109, + }, + } + + for _, tt := range mapTests1 { + tt := tt + t.Run(tt.name, func(t *testing.T) { + actualOutput := make(map[int8][]byte) + if err := Unmarshal(tt.input, &actualOutput); (err != nil) != tt.wantErr { + t.Errorf("decodeState.unmarshal() error = %v, wantErr %v", err, tt.wantErr) + } + + if !reflect.DeepEqual(actualOutput, tt.expectedOutput) { + t.Errorf("decodeState.unmarshal() = %v, want %v", actualOutput, tt.expectedOutput) + } + }) + } + + mapTests2 := []struct { + name string + input []byte + wantErr bool + expectedOutput map[string]user + }{ + { + name: "testing a map of string to struct", + input: []byte{8, 28, 115, 116, 114, 105, 110, 103, 49, 1, 20, 108, 111, 114, 101, 109, 60, 108, 111, 114, 101, 109, 64, 105, 112, 115, 117, 109, 46, 111, 114, 103, 1, 0, 0, 0, 0, 0, 0, 0, 28, 115, 116, 114, 105, 110, 103, 50, 0, 16, 106, 111, 104, 110, 56, 106, 97, 99, 107, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 73, 0, 0, 0, 0, 0, 0, 0}, //nolint:lll + expectedOutput: map[string]user{ + "string1": { + Active: true, + Username: "lorem", + Email: "lorem@ipsum.org", + SignInCount: 1, + }, + "string2": { + Active: false, + Username: "john", + Email: "jack@gmail.com", + SignInCount: 73, + }, }, }, } - for _, tt := range mapTests { + for _, tt := range mapTests2 { tt := tt t.Run(tt.name, func(t *testing.T) { - dst := map[int8][]byte{} - if err := Unmarshal(tt.want, &dst); (err != nil) != tt.wantErr { + actualOutput := make(map[string]user) + if err := Unmarshal(tt.input, &actualOutput); (err != nil) != tt.wantErr { t.Errorf("decodeState.unmarshal() error = %v, wantErr %v", err, tt.wantErr) } - if !reflect.DeepEqual(dst, tt.in) { - t.Errorf("decodeState.unmarshal() = %v, want %v", dst, tt.in) + if !reflect.DeepEqual(actualOutput, tt.expectedOutput) { + t.Errorf("decodeState.unmarshal() = %v, want %v", actualOutput, tt.expectedOutput) } }) } From da9e32ef384b451780f93aef1bc752717b27749b Mon Sep 17 00:00:00 2001 From: Axay sagathiya Date: Wed, 19 Oct 2022 19:22:48 +0530 Subject: [PATCH 7/9] improve encodeMap method. --- pkg/scale/encode.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/scale/encode.go b/pkg/scale/encode.go index 2e0868adc0..e32696bb60 100644 --- a/pkg/scale/encode.go +++ b/pkg/scale/encode.go @@ -10,6 +10,7 @@ import ( "io" "math/big" "reflect" + "sort" ) // Encoder scale encodes to a given io.Writer. @@ -232,14 +233,21 @@ func (es *encodeState) encodeMap(in interface{}) (err error) { return fmt.Errorf("encoding length: %w", err) } - for keyValue := v.MapRange(); keyValue.Next(); { - key := keyValue.Key() + mapKeys := v.MapKeys() + + sort.Slice(mapKeys, func(i, j int) bool { + keyByteOfI, _ := Marshal(mapKeys[i].Interface()) + keyByteOfJ, _ := Marshal(mapKeys[j].Interface()) + return bytes.Compare(keyByteOfI, keyByteOfJ) < 0 + }) + + for _, key := range mapKeys { err = es.marshal(key.Interface()) if err != nil { return fmt.Errorf("encoding map key: %w", err) } - mapValue := keyValue.Value() + mapValue := v.MapIndex(key) if !mapValue.CanInterface() { continue } From ec37a64a22edf89b5344a57eeaac15eb35305405 Mon Sep 17 00:00:00 2001 From: Axay sagathiya Date: Thu, 20 Oct 2022 21:37:00 +0530 Subject: [PATCH 8/9] remove commented Rust code. --- pkg/scale/decode_test.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/pkg/scale/decode_test.go b/pkg/scale/decode_test.go index d97dcaac73..7439fea561 100644 --- a/pkg/scale/decode_test.go +++ b/pkg/scale/decode_test.go @@ -132,34 +132,6 @@ func Test_decodeState_decodeSlice(t *testing.T) { } } -// // Rust code to encode a map of string to struct. -// let mut btree_map: BTreeMap = BTreeMap::new(); -// match btree_map.entry("string1".to_string()) { -// Entry::Vacant(entry) => { -// entry.insert(User{ -// active: true, -// username: "lorem".to_string(), -// email: "lorem@ipsum.org".to_string(), -// sign_in_count: 1, -// }); -// () -// }, -// Entry::Occupied(_) => (), -// } -// match btree_map.entry("string2".to_string()) { -// Entry::Vacant(entry) => { -// entry.insert(User{ -// active: false, -// username: "john".to_string(), -// email: "jack@gmail.com".to_string(), -// sign_in_count: 73, -// }); -// () -// }, -// Entry::Occupied(_) => (), -// } -// println!("{:?}", btree_map.encode()); - type user struct { Active bool Username string From d69371df009e71a7521a1f0c5e6b8628ff5092ed Mon Sep 17 00:00:00 2001 From: Axay sagathiya Date: Fri, 21 Oct 2022 10:30:56 +0530 Subject: [PATCH 9/9] Revert "remove commented Rust code." This reverts commit ec37a64a22edf89b5344a57eeaac15eb35305405. --- pkg/scale/decode_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pkg/scale/decode_test.go b/pkg/scale/decode_test.go index 7439fea561..d97dcaac73 100644 --- a/pkg/scale/decode_test.go +++ b/pkg/scale/decode_test.go @@ -132,6 +132,34 @@ func Test_decodeState_decodeSlice(t *testing.T) { } } +// // Rust code to encode a map of string to struct. +// let mut btree_map: BTreeMap = BTreeMap::new(); +// match btree_map.entry("string1".to_string()) { +// Entry::Vacant(entry) => { +// entry.insert(User{ +// active: true, +// username: "lorem".to_string(), +// email: "lorem@ipsum.org".to_string(), +// sign_in_count: 1, +// }); +// () +// }, +// Entry::Occupied(_) => (), +// } +// match btree_map.entry("string2".to_string()) { +// Entry::Vacant(entry) => { +// entry.insert(User{ +// active: false, +// username: "john".to_string(), +// email: "jack@gmail.com".to_string(), +// sign_in_count: 73, +// }); +// () +// }, +// Entry::Occupied(_) => (), +// } +// println!("{:?}", btree_map.encode()); + type user struct { Active bool Username string