Separate CTAP2 and RFC 7049 canonical encoding

Have two canonical encoding options:
- CTAP2Canonical: CTAP2 uses bytewise lexicographic order for sorting keys
- Canonical: RFC 7049 uses length-first map key ordering.

Sorting is the same when all map keys are the same type.

Closes: #43

cache.go

@@ -39,20 +39,31 @@ func getDecodingStructType(t reflect.Type) decodingStructType {
 }
 
 type encodingStructType struct {
-	fields            fields
-	canonicalFields   fields
-	err               error
-	toArray           bool
-	omitEmpty         bool
-	hasAnonymousField bool
+	fields                  fields
+	bytewiseCanonicalFields fields
+	lenFirstCanonicalFields fields
+	err                     error
+	toArray                 bool
+	omitEmpty               bool
+	hasAnonymousField       bool
 }
 
-// byCanonical sorts fields by CBOR encoded field name.
-type byCanonical struct {
+type byBytewiseFields struct {
 	fields
 }
 
-func (x byCanonical) Less(i, j int) bool {
+func (x byBytewiseFields) Less(i, j int) bool {
+	return bytes.Compare(x.fields[i].cborName, x.fields[j].cborName) <= 0
+}
+
+type byLengthFirstFields struct {
+	fields
+}
+
+func (x byLengthFirstFields) Less(i, j int) bool {
+	if len(x.fields[i].cborName) != len(x.fields[j].cborName) {
+		return len(x.fields[i].cborName) < len(x.fields[j].cborName)
+	}
 	return bytes.Compare(x.fields[i].cborName, x.fields[j].cborName) <= 0
 }
 
@@ -68,6 +79,8 @@ func getEncodingStructType(t reflect.Type) encodingStructType {
 	var err error
 	var omitEmpty bool
 	var hasAnonymousField bool
+	var hasKeyAsInt bool
+	var hasKeyAsStr bool
 	e := getEncodeState()
 	for i := 0; i < len(flds); i++ {
 		// Get field's encodeFunc
@@ -112,6 +125,12 @@ func getEncodingStructType(t reflect.Type) encodingStructType {
 		if !omitEmpty && flds[i].omitEmpty {
 			omitEmpty = true
 		}
+
+		if flds[i].keyAsInt {
+			hasKeyAsInt = true
+		} else {
+			hasKeyAsStr = true
+		}
 	}
 	putEncodeState(e)
 
@@ -122,11 +141,18 @@ func getEncodingStructType(t reflect.Type) encodingStructType {
 	}
 
 	// Sort fields by canonical order
-	canonicalFields := make(fields, len(flds))
-	copy(canonicalFields, flds)
-	sort.Sort(byCanonical{canonicalFields})
+	bytewiseCanonicalFields := make(fields, len(flds))
+	copy(bytewiseCanonicalFields, flds)
+	sort.Sort(byBytewiseFields{bytewiseCanonicalFields})
+
+	lenFirstCanonicalFields := bytewiseCanonicalFields
+	if hasKeyAsInt && hasKeyAsStr {
+		lenFirstCanonicalFields = make(fields, len(flds))
+		copy(lenFirstCanonicalFields, flds)
+		sort.Sort(byLengthFirstFields{lenFirstCanonicalFields})
+	}
 
-	structType := encodingStructType{fields: flds, canonicalFields: canonicalFields, toArray: toArray, omitEmpty: omitEmpty, hasAnonymousField: hasAnonymousField}
+	structType := encodingStructType{fields: flds, bytewiseCanonicalFields: bytewiseCanonicalFields, lenFirstCanonicalFields: lenFirstCanonicalFields, toArray: toArray, omitEmpty: omitEmpty, hasAnonymousField: hasAnonymousField}
 	encodingStructTypeCache.Store(t, structType)
 	return structType
 }

decode_test.go

@@ -940,9 +940,9 @@ var invalidCBORUnmarshalTests = []struct {
 }
 
 func TestInvalidCBORUnmarshal(t *testing.T) {
-	var i interface{}
 	for _, tc := range invalidCBORUnmarshalTests {
 		t.Run(tc.name, func(t *testing.T) {
+			var i interface{}
 			err := cbor.Unmarshal(tc.cborData, &i)
 			if err == nil {
 				t.Errorf("Unmarshal(0x%0x) expecting error, got nil", tc.cborData)

encode.go

@@ -77,21 +77,35 @@ import (
 // Channel, complex, and functon values cannot be encoded in CBOR.  Attempting
 // to encode such a value causes Marshal to return an UnsupportedTypeError.
 //
-// CBOR cannot represent cyclic data structures and Marshal does not handle them.
+// Canonical CBOR encoding uses the following rules:
+//
+//     1. Integers must be as small as possible.
+//     2. The expression of lengths in major types 2 through 5 must be as short
+//        as possible.
+//     3. The keys in every map must be sorted by the following rules:
+//        *  If two keys have different lengths, the shorter one sorts earlier;
+//		  *  If two keys have the same length, the one with the lower value in
+//           (byte-wise) lexical order sorts earlier.
+//     4. Indefinite-length items must be made into definite-length items.
 //
 // CTAP2 canonical CBOR encoding uses the following rules:
 //
 //     1. Integers must be encoded as small as possible.
 //     2. The representations of any floating-point values are not changed.
-//     3. The expression of lengths in major types 2 through 5 must be as short as possible.
+//     3. The expression of lengths in major types 2 through 5 must be as short
+//        as possible.
 //     4. Indefinite-length items must be made into definite-length items.
 //     5. The keys in every map must be sorted lowest value to highest.
+//        * If the major types are different, the one with the lower value in
+//          numerical order sorts earlier.
+//        * If two keys have different lengths, the shorter one sorts earlier;
+//        * If two keys have the same length, the one with the lower value in
+//          (byte-wise) lexical order sorts earlier.
 //     6. Tags must not be present.
 //
-// Canonical CBOR encoding specified in RFC 7049 section 3.9 consists of CTAP2
-// canonical CBOR encoding rules 1, 3, 4, and 5.
-//
-// Marshal supports RFC 7049 and CTAP2 canonical CBOR encoding.
+// Marshal supports 2 options for canonical encoding:
+// 1. Canonical: Canonical CBOR encoding (RFC 7049)
+// 2. CTAP2Canonical:  CTAP2 canonical CBOR encoding
 func Marshal(v interface{}, encOpts EncOptions) ([]byte, error) {
 	e := getEncodeState()
 
@@ -128,12 +142,19 @@ func (e *UnsupportedTypeError) Error() string {
 type EncOptions struct {
 	// Canonical causes map and struct to be encoded in a predictable sequence
 	// of bytes by sorting map keys or struct fields according to canonical rules:
-	//     - If two keys have different CBOR types, the one with lower value in
-	//       numerical order sorts earlier.
-	//     - If two keys have different lengths, the shorter one sorts earlier.
-	//     - If two keys have the same CBOR type and same length, the one with the
-	//       lower value in (byte-wise) lexical order sorts earlier.
+	//     - If two keys have different lengths, the shorter one sorts earlier;
+	//     - If two keys have the same length, the one with the lower value in
+	//       (byte-wise) lexical order sorts earlier.
 	Canonical bool
+	// CTAP2Canonical uses bytewise lexicographic order of map keys encodings:
+	//     - If the major types are different, the one with the lower value in
+	//       numerical order sorts earlier.
+	//     - If two keys have different lengths, the shorter one sorts earlier;
+	//     - If two keys have the same length, the one with the lower value in
+	//       (byte-wise) lexical order sorts earlier.
+	// Please note that when maps keys have the same data type, "canonical CBOR"
+	// AND "CTAP2 canonical CBOR" render the same sort order.
+	CTAP2Canonical bool
 	// TimeRFC3339 causes time.Time to be encoded as string in RFC3339 format;
 	// otherwise, time.Time is encoded as numerical representation of seconds
 	// since January 1, 1970 UTC.
@@ -299,7 +320,7 @@ type mapEncoder struct {
 }
 
 func (me mapEncoder) encodeMap(e *encodeState, v reflect.Value, opts EncOptions) (int, error) {
-	if opts.Canonical {
+	if opts.Canonical || opts.CTAP2Canonical {
 		return me.encodeMapCanonical(e, v, opts)
 	}
 	if me.kf == nil || me.ef == nil {
@@ -333,18 +354,33 @@ type keyValue struct {
 	keyLen, keyValueLen           int
 }
 
-type byCanonicalKeyValues []keyValue
+type pairs []keyValue
 
-func (x byCanonicalKeyValues) Len() int {
+func (x pairs) Len() int {
 	return len(x)
 }
 
-func (x byCanonicalKeyValues) Swap(i, j int) {
+func (x pairs) Swap(i, j int) {
 	x[i], x[j] = x[j], x[i]
 }
 
-func (x byCanonicalKeyValues) Less(i, j int) bool {
-	return bytes.Compare(x[i].keyCBORData, x[j].keyCBORData) <= 0
+type byBytewiseKeyValues struct {
+	pairs
+}
+
+func (x byBytewiseKeyValues) Less(i, j int) bool {
+	return bytes.Compare(x.pairs[i].keyCBORData, x.pairs[j].keyCBORData) <= 0
+}
+
+type byLengthFirstKeyValues struct {
+	pairs
+}
+
+func (x byLengthFirstKeyValues) Less(i, j int) bool {
+	if len(x.pairs[i].keyCBORData) != len(x.pairs[j].keyCBORData) {
+		return len(x.pairs[i].keyCBORData) < len(x.pairs[j].keyCBORData)
+	}
+	return bytes.Compare(x.pairs[i].keyCBORData, x.pairs[j].keyCBORData) <= 0
 }
 
 var keyValuePool = sync.Pool{}
@@ -406,7 +442,11 @@ func (me mapEncoder) encodeMapCanonical(e *encodeState, v reflect.Value, opts En
 		off += kvs[i].keyValueLen
 	}
 
-	sort.Sort(byCanonicalKeyValues(kvs))
+	if opts.CTAP2Canonical {
+		sort.Sort(byBytewiseKeyValues{kvs})
+	} else {
+		sort.Sort(byLengthFirstKeyValues{kvs})
+	}
 
 	n := encodeTypeAndAdditionalValue(e, byte(cborTypeMap), uint64(len(kvs)))
 	for i := 0; i < len(kvs); i++ {
@@ -476,7 +516,9 @@ func encodeStruct(e *encodeState, v reflect.Value, opts EncOptions) (int, error)
 
 	flds := structType.fields
 	if opts.Canonical {
-		flds = structType.canonicalFields
+		flds = structType.lenFirstCanonicalFields
+	} else if opts.CTAP2Canonical {
+		flds = structType.bytewiseCanonicalFields
 	}
 
 	if !structType.hasAnonymousField && !structType.omitEmpty {

encode_test.go

@@ -1545,3 +1545,81 @@ func TestMarshalUnmarshalStructToArray(t *testing.T) {
 		})
 	}
 }
+
+func TestMapCBORCanonical(t *testing.T) {
+	m := make(map[interface{}]bool)
+	m[10] = true
+	m[100] = true
+	m[-1] = true
+	m["z"] = true
+	m["aa"] = true
+	m[[1]int{100}] = true
+	m[[1]int{-1}] = true
+	m[false] = true
+	wantCborData := hexDecode("a80af520f5f4f51864f5617af58120f5626161f5811864f5")
+	b, err := cbor.Marshal(m, cbor.EncOptions{Canonical: true})
+	if err != nil {
+		t.Errorf("Marshal(%v) returns error %s", m, err)
+	}
+	if !bytes.Equal(b, wantCborData) {
+		t.Errorf("Marshal(%v) = 0x%0x, want 0x%0x", m, b, wantCborData)
+	}
+}
+
+func TestMapCTAP2Canonical(t *testing.T) {
+	m := make(map[interface{}]bool)
+	m[10] = true
+	m[100] = true
+	m[-1] = true
+	m["z"] = true
+	m["aa"] = true
+	m[[1]int{100}] = true
+	m[[1]int{-1}] = true
+	m[false] = true
+	wantCborData := hexDecode("a80af51864f520f5617af5626161f5811864f58120f5f4f5")
+	b, err := cbor.Marshal(m, cbor.EncOptions{CTAP2Canonical: true})
+	if err != nil {
+		t.Errorf("Marshal(%v) returns error %s", m, err)
+	}
+	if !bytes.Equal(b, wantCborData) {
+		t.Errorf("Marshal(%v) = 0x%0x, want 0x%0x", m, b, wantCborData)
+	}
+}
+
+func TestStructCBORCanonical(t *testing.T) {
+	type T struct {
+		A bool `cbor:"10,keyasint"`
+		B bool `cbor:"100,keyasint"`
+		C bool `cbor:"-1,keyasint"`
+		D bool `cbor:"z"`
+		E bool `cbor:"aa"`
+	}
+	wantCborData := hexDecode("a50af420f41864f4617af4626161f4")
+	var v T
+	b, err := cbor.Marshal(v, cbor.EncOptions{Canonical: true})
+	if err != nil {
+		t.Errorf("Marshal(%v) returns error %s", v, err)
+	}
+	if !bytes.Equal(b, wantCborData) {
+		t.Errorf("Marshal(%v) = 0x%0x, want 0x%0x", v, b, wantCborData)
+	}
+}
+
+func TestStructCTAP2Canonical(t *testing.T) {
+	type T struct {
+		A bool `cbor:"10,keyasint"`
+		B bool `cbor:"100,keyasint"`
+		C bool `cbor:"-1,keyasint"`
+		D bool `cbor:"z"`
+		E bool `cbor:"aa"`
+	}
+	wantCborData := hexDecode("a50af41864f420f4617af4626161f4")
+	var v T
+	b, err := cbor.Marshal(v, cbor.EncOptions{CTAP2Canonical: true})
+	if err != nil {
+		t.Errorf("Marshal(%v) returns error %s", v, err)
+	}
+	if !bytes.Equal(b, wantCborData) {
+		t.Errorf("Marshal(%v) = 0x%0x, want 0x%0x", v, b, wantCborData)
+	}
+}