From b1182ec0795d0b2f6f605dc3d36fd8ae5b740368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 5 Jul 2023 13:04:11 +0200 Subject: [PATCH 01/98] CBOR basic value tagging --- .../src/kotlinx/serialization/cbor/Tagged.kt | 48 +++++++ .../serialization/cbor/internal/Encoding.kt | 127 ++++++++++++------ .../serialization/cbor/CborReaderTest.kt | 110 +++++++-------- .../serialization/cbor/CborWriterTest.kt | 26 +++- .../serialization/cbor/SampleClasses.kt | 8 ++ 5 files changed, 215 insertions(+), 104 deletions(-) create mode 100644 formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt new file mode 100644 index 000000000..5d9bee228 --- /dev/null +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt @@ -0,0 +1,48 @@ +package kotlinx.serialization.cbor + +import kotlinx.serialization.* + +/** + * Specifies that a property shall be tagged and serialized as CBOR major type 6: optional semantic tagging + * of other major types. + * For types other than [ByteArray], [ByteString] will have no effect. + * + * Example usage: + * + * ``` + * @Serializable + * data class Data( + * @Tagged(1337uL) + * @ByteString + * val a: ByteArray, // CBOR major type 6 1337(major type 2: a byte string). + * + * @Tagged(1234567uL) + * val b: ByteArray // CBOR major type 6 1234567(major type 4: an array of data items). + * ) + * ``` + * + * See [RFC 7049 2.4. Optional Tagging of Items](https://datatracker.ietf.org/doc/html/rfc7049#section-2.4). + */ +@SerialInfo +@Target(AnnotationTarget.PROPERTY) +@ExperimentalSerializationApi +public annotation class Tagged(vararg val tags: ULong) { + public companion object { + public const val DATE_TIME_STANDARD: ULong = 0u; + public const val DATE_TIME_EPOCH: ULong = 1u; + public const val BIGNUM_POSITIVE: ULong = 2u; + public const val BIGNUM_NEGAIVE: ULong = 3u; + public const val DECIMAL_FRACTION: ULong = 4u; + public const val BIGFLOAT: ULong = 5u; + public const val BASE64_URL: ULong = 21u; + public const val BASE64: ULong = 22u; + public const val BASE16: ULong = 23u; + public const val CBOR_ENCODED_DATA: ULong = 24u; + public const val URI: ULong = 32u; + public const val STRING_BASE64_URL: ULong = 33u; + public const val STRING_BASE64: ULong = 34u; + public const val REGEX: ULong = 35u; + public const val MIME_MESSAGE: ULong = 36u; + public const val CBOR_SELF_DESCRIBE: ULong = 55799u; + } +} diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index b77a18c59..32f859d6f 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -1,7 +1,7 @@ /* * Copyright 2017-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ -@file:OptIn(ExperimentalSerializationApi::class) +@file:OptIn(ExperimentalSerializationApi::class, ExperimentalUnsignedTypes::class) package kotlinx.serialization.cbor.internal @@ -64,6 +64,7 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb get() = cbor.serializersModule private var encodeByteArrayAsByteString = false + private var tags: ULongArray? = null @OptIn(ExperimentalSerializationApi::class) override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { @@ -96,8 +97,14 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { encodeByteArrayAsByteString = descriptor.isByteString(index) + tags = descriptor.getTag(index) val name = descriptor.getElementName(index) encoder.encodeString(name) + tags?.forEach { tag -> + val encodedTag= encoder.composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + } return true } @@ -133,6 +140,8 @@ internal class CborEncoder(private val output: ByteArrayOutput) { fun encodeNull() = output.write(NULL) + internal fun writeByte(byteValue: Int)=output.write(byteValue) + fun encodeBoolean(value: Boolean) = output.write(if (value) TRUE else FALSE) fun encodeNumber(value: Long) = output.write(composeNumber(value)) @@ -171,7 +180,7 @@ internal class CborEncoder(private val output: ByteArrayOutput) { private fun composeNumber(value: Long): ByteArray = if (value >= 0) composePositive(value.toULong()) else composeNegative(value) - private fun composePositive(value: ULong): ByteArray = when (value) { + internal fun composePositive(value: ULong): ByteArray = when (value) { in 0u..23u -> byteArrayOf(value.toByte()) in 24u..UByte.MAX_VALUE.toUInt() -> byteArrayOf(24, value.toByte()) in (UByte.MAX_VALUE.toUInt() + 1u)..UShort.MAX_VALUE.toUInt() -> encodeToByteArray(value, 2, 25) @@ -198,15 +207,16 @@ internal class CborEncoder(private val output: ByteArrayOutput) { } private class CborMapReader(cbor: Cbor, decoder: CborDecoder) : CborListReader(cbor, decoder) { - override fun skipBeginToken() = setSize(decoder.startMap() * 2) + override fun skipBeginToken() = setSize(decoder.startMap(tags) * 2) } private open class CborListReader(cbor: Cbor, decoder: CborDecoder) : CborReader(cbor, decoder) { private var ind = 0 - override fun skipBeginToken() = setSize(decoder.startArray()) + override fun skipBeginToken() = setSize(decoder.startArray(tags)) - override fun decodeElementIndex(descriptor: SerialDescriptor) = if (!finiteMode && decoder.isEnd() || (finiteMode && ind >= size)) CompositeDecoder.DECODE_DONE else ind++ + override fun decodeElementIndex(descriptor: SerialDescriptor) = + if (!finiteMode && decoder.isEnd() || (finiteMode && ind >= size)) CompositeDecoder.DECODE_DONE else ind++ } internal open class CborReader(private val cbor: Cbor, protected val decoder: CborDecoder) : AbstractDecoder() { @@ -218,6 +228,7 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb private var readProperties: Int = 0 private var decodeByteArrayAsByteString = false + protected var tags: ULongArray? = null protected fun setSize(size: Int) { if (size >= 0) { @@ -229,7 +240,7 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb override val serializersModule: SerializersModule get() = cbor.serializersModule - protected open fun skipBeginToken() = setSize(decoder.startMap()) + protected open fun skipBeginToken() = setSize(decoder.startMap(tags)) @OptIn(ExperimentalSerializationApi::class) override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder { @@ -251,12 +262,12 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb val knownIndex: Int while (true) { if (isDone()) return CompositeDecoder.DECODE_DONE - val elemName = decoder.nextString() + val elemName = decoder.nextString(tags) readProperties++ val index = descriptor.getElementIndex(elemName) if (index == CompositeDecoder.UNKNOWN_NAME) { - decoder.skipElement() + decoder.skipElement(tags) } else { knownIndex = index break @@ -265,45 +276,46 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb knownIndex } else { if (isDone()) return CompositeDecoder.DECODE_DONE - val elemName = decoder.nextString() + val elemName = decoder.nextString(tags) readProperties++ descriptor.getElementIndexOrThrow(elemName) } decodeByteArrayAsByteString = descriptor.isByteString(index) + tags = descriptor.getTag(index) return index } @OptIn(ExperimentalSerializationApi::class) override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { + return if (decodeByteArrayAsByteString && deserializer.descriptor == ByteArraySerializer().descriptor) { @Suppress("UNCHECKED_CAST") - decoder.nextByteString() as T + decoder.nextByteString(tags) as T } else { - decodeByteArrayAsByteString = decodeByteArrayAsByteString || deserializer.descriptor.isInlineByteString() super.decodeSerializableValue(deserializer) } } - override fun decodeString() = decoder.nextString() + override fun decodeString() = decoder.nextString(tags) override fun decodeNotNullMark(): Boolean = !decoder.isNull() - override fun decodeDouble() = decoder.nextDouble() - override fun decodeFloat() = decoder.nextFloat() + override fun decodeDouble() = decoder.nextDouble(tags) + override fun decodeFloat() = decoder.nextFloat(tags) - override fun decodeBoolean() = decoder.nextBoolean() + override fun decodeBoolean() = decoder.nextBoolean(tags) - override fun decodeByte() = decoder.nextNumber().toByte() - override fun decodeShort() = decoder.nextNumber().toShort() - override fun decodeChar() = decoder.nextNumber().toInt().toChar() - override fun decodeInt() = decoder.nextNumber().toInt() - override fun decodeLong() = decoder.nextNumber() + override fun decodeByte() = decoder.nextNumber(tags).toByte() + override fun decodeShort() = decoder.nextNumber(tags).toShort() + override fun decodeChar() = decoder.nextNumber(tags).toInt().toChar() + override fun decodeInt() = decoder.nextNumber(tags).toInt() + override fun decodeLong() = decoder.nextNumber(tags) - override fun decodeNull() = decoder.nextNull() + override fun decodeNull() = decoder.nextNull(tags) override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = - enumDescriptor.getElementIndexOrThrow(decoder.nextString()) + enumDescriptor.getElementIndexOrThrow(decoder.nextString(tags)) private fun isDone(): Boolean = !finiteMode && decoder.isEnd() || (finiteMode && readProperties >= size) } @@ -329,14 +341,17 @@ internal class CborDecoder(private val input: ByteArrayInput) { fun isNull() = curByte == NULL - fun nextNull(): Nothing? { - skipOverTags() + fun nextNull(tag: ULong?) = nextNull(tag?.let { ulongArrayOf(it) }) + fun nextNull(tags: ULongArray?): Nothing? { + processTags(tags) skipByte(NULL) return null } - fun nextBoolean(): Boolean { - skipOverTags() + fun nextBoolean(tag: ULong?) = nextBoolean(tag?.let { ulongArrayOf(it) }) + + fun nextBoolean(tags: ULongArray?): Boolean { + processTags(tags) val ans = when (curByte) { TRUE -> true FALSE -> false @@ -346,12 +361,21 @@ internal class CborDecoder(private val input: ByteArrayInput) { return ans } - fun startArray() = startSized(BEGIN_ARRAY, HEADER_ARRAY, "array") + fun startArray(tag: ULong?) = startArray(tag?.let { ulongArrayOf(it) }) - fun startMap() = startSized(BEGIN_MAP, HEADER_MAP, "map") + fun startArray(tags: ULongArray?) = startSized(tags, BEGIN_ARRAY, HEADER_ARRAY, "array") - private fun startSized(unboundedHeader: Int, boundedHeaderMask: Int, collectionType: String): Int { - skipOverTags() + fun startMap(tag: ULong?) = startMap(tag?.let { ulongArrayOf(it) }) + + fun startMap(tags: ULongArray?) = startSized(tags, BEGIN_MAP, HEADER_MAP, "map") + + private fun startSized( + tags: ULongArray?, + unboundedHeader: Int, + boundedHeaderMask: Int, + collectionType: String + ): Int { + processTags(tags) if (curByte == unboundedHeader) { skipByte(unboundedHeader) return -1 @@ -367,8 +391,10 @@ internal class CborDecoder(private val input: ByteArrayInput) { fun end() = skipByte(BREAK) - fun nextByteString(): ByteArray { - skipOverTags() + fun nextByteString(tag: ULong?) = nextByteString(tag?.let { ulongArrayOf(it) }) + fun nextByteString(tags: ULongArray?): ByteArray { + + processTags(tags) if ((curByte and 0b111_00000) != HEADER_BYTE_STRING.toInt()) throw CborDecodingException("start of byte string", curByte) val arr = readBytes() @@ -376,8 +402,9 @@ internal class CborDecoder(private val input: ByteArrayInput) { return arr } - fun nextString(): String { - skipOverTags() + fun nextString(tag: ULong?) = nextString(tag?.let { ulongArrayOf(it) }) + fun nextString(tags: ULongArray?): String { + processTags(tags) if ((curByte and 0b111_00000) != HEADER_STRING.toInt()) throw CborDecodingException("start of string", curByte) val arr = readBytes() @@ -395,15 +422,21 @@ internal class CborDecoder(private val input: ByteArrayInput) { input.readExactNBytes(strLen) } - private fun skipOverTags() { + private fun processTags(tags: ULongArray?) { + var index = 0 while ((curByte and 0b111_00000) == HEADER_TAG) { - readNumber() // This is the tag number + val readTag = readNumber().toULong() // This is the tag number + tags?.let { + if (index++ > it.size) throw CborDecodingException("More tags found than the ${it.size} tags specified.") + if (readTag != it[index - 1]) throw CborDecodingException("CBOR tag $readTag does not match expected tag $it") + } readByte() } } - fun nextNumber(): Long { - skipOverTags() + fun nextNumber(tag: ULong?): Long = nextNumber(tag?.let { ulongArrayOf(it) }) + fun nextNumber(tags: ULongArray?): Long { + processTags(tags) val res = readNumber() readByte() return res @@ -446,8 +479,10 @@ internal class CborDecoder(private val input: ByteArrayInput) { return array } - fun nextFloat(): Float { - skipOverTags() + fun nextFloat(tag: ULong?) = nextFloat(tag?.let { ulongArrayOf(it) }) + + fun nextFloat(tags: ULongArray?): Float { + processTags(tags) val res = when (curByte) { NEXT_FLOAT -> Float.fromBits(readInt()) NEXT_HALF -> floatFromHalfBits(readShort()) @@ -505,10 +540,10 @@ internal class CborDecoder(private val input: ByteArrayInput) { * been skipped, the "length stack" is [pruned][prune]. For indefinite length elements, a special marker is added to * the "length stack" which is only popped from the "length stack" when a CBOR [break][isEnd] is encountered. */ - fun skipElement() { + fun skipElement(tags: ULongArray?) { val lengthStack = mutableListOf() - skipOverTags() + processTags(tags) do { if (isEof()) throw CborDecodingException("Unexpected EOF while skipping element") @@ -524,7 +559,7 @@ internal class CborDecoder(private val input: ByteArrayInput) { val length = elementLength() if (header == HEADER_ARRAY || header == HEADER_MAP) { if (length > 0) lengthStack.add(length) - skipOverTags() + processTags(tags) } else { input.skip(length) prune(lengthStack) @@ -535,6 +570,8 @@ internal class CborDecoder(private val input: ByteArrayInput) { } while (lengthStack.isNotEmpty()) } + fun skipElement(singleTag: ULong?) = skipElement(singleTag?.let { ulongArrayOf(it) }) + /** * Removes an item from the top of the [lengthStack], cascading the removal if the item represents the last item * (i.e. a length value of `1`) at its stack depth. @@ -639,6 +676,10 @@ private fun SerialDescriptor.isByteString(index: Int): Boolean { return getElementAnnotations(index).find { it is ByteString } != null } +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.getTag(index: Int): ULongArray? { + return (getElementAnnotations(index).find { it is Tagged } as Tagged?)?.tags +} private fun SerialDescriptor.isInlineByteString(): Boolean { // inline item classes should only have 1 item return isInline && isByteString(0) diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt index f615d5eda..199b8da2f 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt @@ -21,32 +21,32 @@ class CborReaderTest { @Test fun testDecodeIntegers() { withDecoder("0C1903E8") { - assertEquals(12L, nextNumber()) - assertEquals(1000L, nextNumber()) + assertEquals(12L, nextNumber(null)) + assertEquals(1000L, nextNumber(null)) } withDecoder("203903e7") { - assertEquals(-1L, nextNumber()) - assertEquals(-1000L, nextNumber()) + assertEquals(-1L, nextNumber(null)) + assertEquals(-1000L, nextNumber(null)) } } @Test fun testDecodeStrings() { withDecoder("6568656C6C6F") { - assertEquals("hello", nextString()) + assertEquals("hello", nextString(null)) } withDecoder("7828737472696E672074686174206973206C6F6E676572207468616E2032332063686172616374657273") { - assertEquals("string that is longer than 23 characters", nextString()) + assertEquals("string that is longer than 23 characters", nextString(null)) } } @Test fun testDecodeDoubles() { withDecoder("fb7e37e43c8800759c") { - assertEquals(1e+300, nextDouble()) + assertEquals(1e+300, nextDouble(null)) } withDecoder("fa47c35000") { - assertEquals(100000.0f, nextFloat()) + assertEquals(100000.0f, nextFloat(null)) } } @@ -104,7 +104,7 @@ class CborReaderTest { withDecoder(input = "5F44aabbccdd43eeff99FF") { assertEquals( expected = "aabbccddeeff99", - actual = HexConverter.printHexBinary(nextByteString(), lowerCase = true) + actual = HexConverter.printHexBinary(nextByteString(null), lowerCase = true) ) } } @@ -251,13 +251,13 @@ class CborReaderTest { withDecoder("a461611bffffffffffffffff616220616342cafe61646b48656c6c6f20776f726c64") { expectMap(size = 4) expect("a") - skipElement() // unsigned(18446744073709551615) + skipElement(null) // unsigned(18446744073709551615) expect("b") - skipElement() // negative(0) + skipElement(null) // negative(0) expect("c") - skipElement() // "\xCA\xFE" + skipElement(null) // "\xCA\xFE" expect("d") - skipElement() // "Hello world" + skipElement(null) // "Hello world" expectEof() } } @@ -282,9 +282,9 @@ class CborReaderTest { withDecoder("a2616140616260") { expectMap(size = 2) expect("a") - skipElement() // bytes(0) + skipElement(null) // bytes(0) expect("b") - skipElement() // text(0) + skipElement(null) // text(0) expectEof() } } @@ -318,9 +318,9 @@ class CborReaderTest { withDecoder("a26161830118ff1a000100006162a26178676b6f746c696e7861796d73657269616c697a6174696f6e") { expectMap(size = 2) expect("a") - skipElement() // [1, 255, 65536] + skipElement(null) // [1, 255, 65536] expect("b") - skipElement() // {"x": "kotlinx", "y": "serialization"} + skipElement(null) // {"x": "kotlinx", "y": "serialization"} expectEof() } } @@ -343,9 +343,9 @@ class CborReaderTest { withDecoder("a26161806162a0") { expectMap(size = 2) expect("a") - skipElement() // [1, 255, 65536] + skipElement(null) // [1, 255, 65536] expect("b") - skipElement() // {"x": "kotlinx", "y": "serialization"} + skipElement(null) // {"x": "kotlinx", "y": "serialization"} expectEof() } } @@ -401,13 +401,13 @@ class CborReaderTest { withDecoder("a461615f42cafe43010203ff61627f6648656c6c6f2065776f726c64ff61639f676b6f746c696e786d73657269616c697a6174696f6eff6164bf613101613202613303ff") { expectMap(size = 4) expect("a") - skipElement() // "\xCA\xFE\x01\x02\x03" + skipElement(null) // "\xCA\xFE\x01\x02\x03" expect("b") - skipElement() // "Hello world" + skipElement(null) // "Hello world" expect("c") - skipElement() // ["kotlinx", "serialization"] + skipElement(null) // ["kotlinx", "serialization"] expect("d") - skipElement() // {"1": 1, "2": 2, "3": 3} + skipElement(null) // {"1": 1, "2": 2, "3": 3} expectEof() } } @@ -445,13 +445,13 @@ class CborReaderTest { withDecoder("A46161CC1BFFFFFFFFFFFFFFFFD822616220D8386163D84E42CAFE6164D85ACC6B48656C6C6F20776F726C64") { expectMap(size = 4) expect("a") - skipElement() // unsigned(18446744073709551615) - expect("b") - skipElement() // negative(0) - expect("c") - skipElement() // "\xCA\xFE" + skipElement(12uL) // unsigned(18446744073709551615) + expect("b", 34uL) + skipElement(null) // negative(0) + expect("c", 56uL) + skipElement(78uL) // "\xCA\xFE" expect("d") - skipElement() // "Hello world" + skipElement(ulongArrayOf(90uL, 12uL)) // "Hello world" expectEof() } } @@ -679,11 +679,11 @@ class CborReaderTest { * */ withDecoder("8468756E746167676564C0687461676765642D30D8F56A7461676765642D323435D930396C7461676765642D3132333435") { - assertEquals(4, startArray()) - assertEquals("untagged", nextString()) - assertEquals("tagged-0", nextString()) - assertEquals("tagged-245", nextString()) - assertEquals("tagged-12345", nextString()) + assertEquals(4, startArray(null)) + assertEquals("untagged", nextString(null)) + assertEquals("tagged-0", nextString(0u)) + assertEquals("tagged-245", nextString(245uL)) + assertEquals("tagged-12345", nextString(12345uL)) } } @@ -704,13 +704,13 @@ class CborReaderTest { * FB 401999999999999A # primitive(4618891777831180698) */ withDecoder("86187BC01A0001E240D8F51A000F423FD930393831D822FB3FE161F9F01B866ED90237FB401999999999999A") { - assertEquals(6, startArray()) - assertEquals(123, nextNumber()) - assertEquals(123456, nextNumber()) - assertEquals(999999, nextNumber()) - assertEquals(-50, nextNumber()) - assertEquals(0.54321, nextDouble(), 0.00001) - assertEquals(6.4, nextDouble(), 0.00001) + assertEquals(6, startArray(null)) + assertEquals(123, nextNumber(null)) + assertEquals(123456, nextNumber(0uL)) + assertEquals(999999, nextNumber(245uL)) + assertEquals(-50, nextNumber(12345uL)) + assertEquals(0.54321, nextDouble(34uL), 0.00001) + assertEquals(6.4, nextDouble(567uL), 0.00001) } } @@ -738,26 +738,26 @@ class CborReaderTest { * 31323334353637 # "1234567" */ withDecoder("A2636D6170D87BA16874686973206D61706D69732074616767656420313233656172726179DA0012D687836A74686973206172726179696973207461676765646731323334353637") { - assertEquals(2, startMap()) - assertEquals("map", nextString()) - assertEquals(1, startMap()) - assertEquals("this map", nextString()) - assertEquals("is tagged 123", nextString()) - assertEquals("array", nextString()) - assertEquals(3, startArray()) - assertEquals("this array", nextString()) - assertEquals("is tagged", nextString()) - assertEquals("1234567", nextString()) + assertEquals(2, startMap(null)) + assertEquals("map", nextString(null)) + assertEquals(1, startMap(123uL)) + assertEquals("this map", nextString(null)) + assertEquals("is tagged 123", nextString(null)) + assertEquals("array", nextString(null)) + assertEquals(3, startArray(1234567uL)) + assertEquals("this array", nextString(null)) + assertEquals("is tagged", nextString(null)) + assertEquals("1234567", nextString(null)) } } } -private fun CborDecoder.expect(expected: String) { - assertEquals(expected, actual = nextString(), "string") +private fun CborDecoder.expect(expected: String, currentTag:ULong?=null) { + assertEquals(expected, actual = nextString(currentTag), "string") } -private fun CborDecoder.expectMap(size: Int) { - assertEquals(size, actual = startMap(), "map size") +private fun CborDecoder.expectMap(size: Int, currentTag: ULong?=null) { + assertEquals(size, actual = startMap(currentTag), "map size") } private fun CborDecoder.expectEof() { diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt index da7b12874..c283ae223 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt @@ -33,6 +33,20 @@ class CbrWriterTest { ) } + @Test + fun writeTaggedClass() { + val test = WithTags( + a = 18446744073709551615uL, + b = -0, + c = byteArrayOf(0xC.toByte(), 0xA.toByte(), 0xF.toByte(), 0xE.toByte()), + d = "Hello World" + ) + assertEquals( + "bf637374726d48656c6c6f2c20776f726c64216169182a686e756c6c61626c65f6646c6973749f61616162ff636d6170bf01f502f4ff65696e6e6572bf6161636c6f6cff6a696e6e6572734c6973749fbf6161636b656bffff6a62797465537472696e6742cafe696279746541727261799f383521ffff", + Cbor.encodeToHexString(WithTags.serializer(), test) + ) + } + @Test fun writeManyNumbers() { val test = NumberTypesUmbrella( @@ -102,24 +116,24 @@ class CbrWriterTest { @Test fun testWriteCustomByteString() { assertEquals( - expected = "bf617843112233ff", - actual = Cbor.encodeToHexString(TypeWithCustomByteString(CustomByteString(0x11, 0x22, 0x33))) + expected = "bf617843112233ff", + actual = Cbor.encodeToHexString(TypeWithCustomByteString(CustomByteString(0x11, 0x22, 0x33))) ) } @Test fun testWriteNullableCustomByteString() { assertEquals( - expected = "bf617843112233ff", - actual = Cbor.encodeToHexString(TypeWithNullableCustomByteString(CustomByteString(0x11, 0x22, 0x33))) + expected = "bf617843112233ff", + actual = Cbor.encodeToHexString(TypeWithNullableCustomByteString(CustomByteString(0x11, 0x22, 0x33))) ) } @Test fun testWriteNullCustomByteString() { assertEquals( - expected = "bf6178f6ff", - actual = Cbor.encodeToHexString(TypeWithNullableCustomByteString(null)) + expected = "bf6178f6ff", + actual = Cbor.encodeToHexString(TypeWithNullableCustomByteString(null)) ) } diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt index e4418f474..5a5054517 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt @@ -113,6 +113,14 @@ data class TypeWithCustomByteString(@ByteString val x: CustomByteString) @Serializable data class TypeWithNullableCustomByteString(@ByteString val x: CustomByteString?) +@Serializable +data class WithTags( + @Tagged(12uL) val a: ULong, + val b: Int, + @ByteString val c: ByteArray, + @Tagged(90uL, 12uL) val d: String +) + @JvmInline @Serializable value class ValueClassWithByteString(@ByteString val x: ByteArray) From 47d62338052340268301d38f87a10b3f6f2606a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 5 Jul 2023 14:20:11 +0200 Subject: [PATCH 02/98] CBOR: handle tagged keys --- .../src/kotlinx/serialization/cbor/Tagged.kt | 7 ++- .../serialization/cbor/internal/Encoding.kt | 55 +++++++++++++----- .../serialization/cbor/CborWriterTest.kt | 12 ++-- .../serialization/cbor/SampleClasses.kt | 56 +++++++++++++------ 4 files changed, 94 insertions(+), 36 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt index 5d9bee228..6581f00df 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt @@ -26,7 +26,7 @@ import kotlinx.serialization.* @SerialInfo @Target(AnnotationTarget.PROPERTY) @ExperimentalSerializationApi -public annotation class Tagged(vararg val tags: ULong) { +public annotation class Tagged(@OptIn(ExperimentalUnsignedTypes::class) vararg val tags: ULong) { public companion object { public const val DATE_TIME_STANDARD: ULong = 0u; public const val DATE_TIME_EPOCH: ULong = 1u; @@ -46,3 +46,8 @@ public annotation class Tagged(vararg val tags: ULong) { public const val CBOR_SELF_DESCRIBE: ULong = 55799u; } } + +@SerialInfo +@Target(AnnotationTarget.PROPERTY) +@ExperimentalSerializationApi +public annotation class KeyTags(@OptIn(ExperimentalUnsignedTypes::class) vararg val tags: ULong) \ No newline at end of file diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 32f859d6f..deb52d067 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -64,7 +64,6 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb get() = cbor.serializersModule private var encodeByteArrayAsByteString = false - private var tags: ULongArray? = null @OptIn(ExperimentalSerializationApi::class) override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { @@ -97,11 +96,18 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { encodeByteArrayAsByteString = descriptor.isByteString(index) - tags = descriptor.getTag(index) val name = descriptor.getElementName(index) + + descriptor.getKeyTags(index)?.forEach { tag -> + val encodedTag = encoder.composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + } + encoder.encodeString(name) - tags?.forEach { tag -> - val encodedTag= encoder.composePositive(tag) + + descriptor.getValueTags(index)?.forEach { tag -> + val encodedTag = encoder.composePositive(tag) encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } } @@ -140,7 +146,7 @@ internal class CborEncoder(private val output: ByteArrayOutput) { fun encodeNull() = output.write(NULL) - internal fun writeByte(byteValue: Int)=output.write(byteValue) + internal fun writeByte(byteValue: Int) = output.write(byteValue) fun encodeBoolean(value: Boolean) = output.write(if (value) TRUE else FALSE) @@ -258,17 +264,21 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb } override fun decodeElementIndex(descriptor: SerialDescriptor): Int { + val index = if (cbor.ignoreUnknownKeys) { val knownIndex: Int while (true) { if (isDone()) return CompositeDecoder.DECODE_DONE - val elemName = decoder.nextString(tags) + val (elemName, tags) = decoder.nextTaggedString() readProperties++ val index = descriptor.getElementIndex(elemName) if (index == CompositeDecoder.UNKNOWN_NAME) { decoder.skipElement(tags) } else { + descriptor.getKeyTags(index)?.let { keyTags -> + if (!(keyTags contentEquals tags)) throw CborDecodingException("CBOR tags $tags do not match declared tags $keyTags") + } knownIndex = index break } @@ -276,13 +286,17 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb knownIndex } else { if (isDone()) return CompositeDecoder.DECODE_DONE - val elemName = decoder.nextString(tags) + val (elemName, tags) = decoder.nextTaggedString() readProperties++ - descriptor.getElementIndexOrThrow(elemName) + descriptor.getElementIndexOrThrow(elemName).also { index -> + descriptor.getKeyTags(index)?.let { keyTags -> + if (!(keyTags contentEquals tags)) throw CborDecodingException("CBOR tags $tags do not match declared tags $keyTags") + } + } } decodeByteArrayAsByteString = descriptor.isByteString(index) - tags = descriptor.getTag(index) + tags = descriptor.getValueTags(index) return index } @@ -403,14 +417,19 @@ internal class CborDecoder(private val input: ByteArrayInput) { } fun nextString(tag: ULong?) = nextString(tag?.let { ulongArrayOf(it) }) - fun nextString(tags: ULongArray?): String { - processTags(tags) + fun nextString(tags: ULongArray?) = nextTaggedString(tags).first + + //used to r + fun nextTaggedString() = nextTaggedString(null) + + private fun nextTaggedString(tags: ULongArray?): Pair { + val collectedTags = processTags(tags) if ((curByte and 0b111_00000) != HEADER_STRING.toInt()) throw CborDecodingException("start of string", curByte) val arr = readBytes() val ans = arr.decodeToString() readByte() - return ans + return ans to collectedTags } private fun readBytes(): ByteArray = @@ -422,16 +441,19 @@ internal class CborDecoder(private val input: ByteArrayInput) { input.readExactNBytes(strLen) } - private fun processTags(tags: ULongArray?) { + private fun processTags(tags: ULongArray?): ULongArray? { var index = 0 + val collectedTags = mutableListOf() while ((curByte and 0b111_00000) == HEADER_TAG) { val readTag = readNumber().toULong() // This is the tag number + collectedTags += readTag tags?.let { if (index++ > it.size) throw CborDecodingException("More tags found than the ${it.size} tags specified.") if (readTag != it[index - 1]) throw CborDecodingException("CBOR tag $readTag does not match expected tag $it") } readByte() } + return if (collectedTags.isEmpty()) null else collectedTags.toULongArray() } fun nextNumber(tag: ULong?): Long = nextNumber(tag?.let { ulongArrayOf(it) }) @@ -677,7 +699,7 @@ private fun SerialDescriptor.isByteString(index: Int): Boolean { } @OptIn(ExperimentalSerializationApi::class) -private fun SerialDescriptor.getTag(index: Int): ULongArray? { +private fun SerialDescriptor.getValueTags(index: Int): ULongArray? { return (getElementAnnotations(index).find { it is Tagged } as Tagged?)?.tags } private fun SerialDescriptor.isInlineByteString(): Boolean { @@ -686,6 +708,11 @@ private fun SerialDescriptor.isInlineByteString(): Boolean { } +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.getKeyTags(index: Int): ULongArray? { + return (getElementAnnotations(index).find { it is KeyTags } as KeyTags?)?.tags +} + private val normalizeBaseBits = SINGLE_PRECISION_NORMALIZE_BASE.toBits() diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt index c283ae223..2560c141e 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt @@ -36,15 +36,17 @@ class CbrWriterTest { @Test fun writeTaggedClass() { val test = WithTags( - a = 18446744073709551615uL, - b = -0, - c = byteArrayOf(0xC.toByte(), 0xA.toByte(), 0xF.toByte(), 0xE.toByte()), + a = 0xFFFFFFFuL, + b = -1, + c = byteArrayOf(0xCA.toByte(), 0xFE.toByte()), d = "Hello World" ) + val encoded = Cbor.encodeToHexString(WithTags.serializer(), test) assertEquals( - "bf637374726d48656c6c6f2c20776f726c64216169182a686e756c6c61626c65f6646c6973749f61616162ff636d6170bf01f502f4ff65696e6e6572bf6161636c6f6cff6a696e6e6572734c6973749fbf6161636b656bffff6a62797465537472696e6742cafe696279746541727261799f383521ffff", - Cbor.encodeToHexString(WithTags.serializer(), test) + "bf6161cc1a0fffffffd822616220d8386163d84e42cafe6164d85acc6b48656c6c6f20576f726c64ff", + encoded ) + assertEquals(test, Cbor.decodeFromHexString(WithTags.serializer(), encoded)) } @Test diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt index 5a5054517..92693b124 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt @@ -15,15 +15,15 @@ data class Simple(val a: String) @Serializable data class TypesUmbrella( - val str: String, - val i: Int, - val nullable: Double?, - val list: List, - val map: Map, - val inner: Simple, - val innersList: List, - @ByteString val byteString: ByteArray, - val byteArray: ByteArray + val str: String, + val i: Int, + val nullable: Double?, + val list: List, + val map: Map, + val inner: Simple, + val innersList: List, + @ByteString val byteString: ByteArray, + val byteArray: ByteArray ) { override fun equals(other: Any?): Boolean { if (this === other) return true @@ -60,12 +60,12 @@ data class TypesUmbrella( @Serializable data class NumberTypesUmbrella( - val int: Int, - val long: Long, - val float: Float, - val double: Double, - val boolean: Boolean, - val char: Char + val int: Int, + val long: Long, + val float: Float, + val double: Double, + val boolean: Boolean, + val char: Char ) @Serializable @@ -116,10 +116,34 @@ data class TypeWithNullableCustomByteString(@ByteString val x: CustomByteString? @Serializable data class WithTags( @Tagged(12uL) val a: ULong, + @KeyTags(34uL) val b: Int, + @KeyTags(56uL) + @Tagged(78uL) @ByteString val c: ByteArray, @Tagged(90uL, 12uL) val d: String -) +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + + other as WithTags + + if (a != other.a) return false + if (b != other.b) return false + if (!c.contentEquals(other.c)) return false + return d == other.d + } + + override fun hashCode(): Int { + var result = a.hashCode() + result = 31 * result + b + result = 31 * result + c.contentHashCode() + result = 31 * result + d.hashCode() + return result + } +} + @JvmInline @Serializable From 87bcda33552d20384e6992e400bdf22240306961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 5 Jul 2023 14:37:38 +0200 Subject: [PATCH 03/98] CBOR: make verifying and writing tags optional --- .../src/kotlinx/serialization/cbor/Cbor.kt | 64 +++++++++++++++++-- .../serialization/cbor/internal/Encoding.kt | 35 ++++++---- 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index 9e76a8fbd..e6bc02a8b 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -5,9 +5,6 @@ package kotlinx.serialization.cbor import kotlinx.serialization.* -import kotlinx.serialization.builtins.* -import kotlinx.serialization.cbor.internal.ByteArrayInput -import kotlinx.serialization.cbor.internal.ByteArrayOutput import kotlinx.serialization.cbor.internal.* import kotlinx.serialization.modules.* @@ -27,18 +24,28 @@ import kotlinx.serialization.modules.* * @param encodeDefaults specifies whether default values of Kotlin properties are encoded. * False by default; meaning that properties with values equal to defaults will be elided. * @param ignoreUnknownKeys specifies if unknown CBOR elements should be ignored (skipped) when decoding. + * @param writeKeyTags Specifies whether tags set using the [KeyTags] annotation should be written (or omitted) + * @param writeValueTags Specifies whether tags set using the [Tagged] annotation should be written (or omitted) + * @param verifyKeyTags Specifies whether tags preceding map keys (i.e. properties) should be matched against the + * [KeyTags] annotation during the deserialization process. Useful for lenient parsing + * @param verifyValueTags Specifies whether tags preceding values should be matched against the [Tagged] + * annotation during the deserialization process. Useful for lenient parsing. */ @ExperimentalSerializationApi public sealed class Cbor( internal val encodeDefaults: Boolean, internal val ignoreUnknownKeys: Boolean, + internal val writeKeyTags: Boolean, + internal val writeValueTags: Boolean, + internal val verifyKeyTags: Boolean, + internal val verifyValueTags: Boolean, override val serializersModule: SerializersModule ) : BinaryFormat { /** * The default instance of [Cbor] */ - public companion object Default : Cbor(false, false, EmptySerializersModule()) + public companion object Default : Cbor(false, false, true, true, true, true, EmptySerializersModule()) override fun encodeToByteArray(serializer: SerializationStrategy, value: T): ByteArray { val output = ByteArrayOutput() @@ -55,8 +62,23 @@ public sealed class Cbor( } @OptIn(ExperimentalSerializationApi::class) -private class CborImpl(encodeDefaults: Boolean, ignoreUnknownKeys: Boolean, serializersModule: SerializersModule) : - Cbor(encodeDefaults, ignoreUnknownKeys, serializersModule) +private class CborImpl( + encodeDefaults: Boolean, ignoreUnknownKeys: Boolean, + writeKeyTags: Boolean, + writeValueTags: Boolean, + verifyKeyTags: Boolean, + verifyValueTags: Boolean, + serializersModule: SerializersModule +) : + Cbor( + encodeDefaults, + ignoreUnknownKeys, + writeKeyTags, + writeValueTags, + verifyKeyTags, + verifyValueTags, + serializersModule + ) /** * Creates an instance of [Cbor] configured from the optionally given [Cbor instance][from] @@ -66,7 +88,15 @@ private class CborImpl(encodeDefaults: Boolean, ignoreUnknownKeys: Boolean, seri public fun Cbor(from: Cbor = Cbor, builderAction: CborBuilder.() -> Unit): Cbor { val builder = CborBuilder(from) builder.builderAction() - return CborImpl(builder.encodeDefaults, builder.ignoreUnknownKeys, builder.serializersModule) + return CborImpl( + builder.encodeDefaults, + builder.ignoreUnknownKeys, + builder.writeKeyTags, + builder.writeValueTags, + builder.verifyKeyTags, + builder.verifyValueTags, + builder.serializersModule + ) } /** @@ -87,6 +117,26 @@ public class CborBuilder internal constructor(cbor: Cbor) { */ public var ignoreUnknownKeys: Boolean = cbor.ignoreUnknownKeys + /** + * Specifies whether tags set using the [KeyTags] annotation should be written (or omitted) + */ + public var writeKeyTags: Boolean = cbor.writeKeyTags + + /** + * Specifies whether tags set using the [Tagged] annotation should be written (or omitted) + */ + public var writeValueTags: Boolean = cbor.writeKeyTags + + /** + * Specifies whether tags preceding map keys (i.e. properties) should be matched against the [KeyTags] annotation during the deserialization process + */ + public var verifyKeyTags: Boolean = cbor.verifyKeyTags + + /** + * Specifies whether tags preceding values should be matched against the [Tagged] annotation during the deserialization process + */ + public var verifyValueTags: Boolean = cbor.verifyValueTags + /** * Module with contextual and polymorphic serializers to be used in the resulting [Cbor] instance. */ diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index deb52d067..37eba80ef 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -98,18 +98,22 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb encodeByteArrayAsByteString = descriptor.isByteString(index) val name = descriptor.getElementName(index) - descriptor.getKeyTags(index)?.forEach { tag -> - val encodedTag = encoder.composePositive(tag) - encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() - encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + if (cbor.writeKeyTags) { + descriptor.getKeyTags(index)?.forEach { tag -> + val encodedTag = encoder.composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + } } encoder.encodeString(name) - descriptor.getValueTags(index)?.forEach { tag -> - val encodedTag = encoder.composePositive(tag) - encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() - encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + if (cbor.writeValueTags) { + descriptor.getValueTags(index)?.forEach { tag -> + val encodedTag = encoder.composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + } } return true } @@ -276,8 +280,10 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb if (index == CompositeDecoder.UNKNOWN_NAME) { decoder.skipElement(tags) } else { - descriptor.getKeyTags(index)?.let { keyTags -> - if (!(keyTags contentEquals tags)) throw CborDecodingException("CBOR tags $tags do not match declared tags $keyTags") + if (cbor.verifyKeyTags) { + descriptor.getKeyTags(index)?.let { keyTags -> + if (!(keyTags contentEquals tags)) throw CborDecodingException("CBOR tags $tags do not match declared tags $keyTags") + } } knownIndex = index break @@ -289,14 +295,16 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb val (elemName, tags) = decoder.nextTaggedString() readProperties++ descriptor.getElementIndexOrThrow(elemName).also { index -> - descriptor.getKeyTags(index)?.let { keyTags -> - if (!(keyTags contentEquals tags)) throw CborDecodingException("CBOR tags $tags do not match declared tags $keyTags") + if (cbor.verifyKeyTags) { + descriptor.getKeyTags(index)?.let { keyTags -> + if (!(keyTags contentEquals tags)) throw CborDecodingException("CBOR tags $tags do not match declared tags $keyTags") + } } } } decodeByteArrayAsByteString = descriptor.isByteString(index) - tags = descriptor.getValueTags(index) + tags = if (cbor.verifyValueTags) descriptor.getValueTags(index) else null return index } @@ -407,7 +415,6 @@ internal class CborDecoder(private val input: ByteArrayInput) { fun nextByteString(tag: ULong?) = nextByteString(tag?.let { ulongArrayOf(it) }) fun nextByteString(tags: ULongArray?): ByteArray { - processTags(tags) if ((curByte and 0b111_00000) != HEADER_BYTE_STRING.toInt()) throw CborDecodingException("start of byte string", curByte) From d3eac6b779251f7b2e7f3ba9d67aeaa52f6de0a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 5 Jul 2023 18:19:55 +0200 Subject: [PATCH 04/98] CBOR: fix decoding value tags, add tests --- .../serialization/cbor/internal/Encoding.kt | 67 +++-- .../serialization/cbor/CborReaderTest.kt | 172 ++++++++++-- .../serialization/cbor/CborTaggedTest.kt | 244 ++++++++++++++++++ .../serialization/cbor/CborWriterTest.kt | 16 -- 4 files changed, 443 insertions(+), 56 deletions(-) create mode 100644 formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 37eba80ef..14256e42e 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -363,16 +363,16 @@ internal class CborDecoder(private val input: ByteArrayInput) { fun isNull() = curByte == NULL - fun nextNull(tag: ULong?) = nextNull(tag?.let { ulongArrayOf(it) }) - fun nextNull(tags: ULongArray?): Nothing? { + fun nextNull(tag: ULong) = nextNull(ulongArrayOf(tag)) + fun nextNull(tags: ULongArray? = null): Nothing? { processTags(tags) skipByte(NULL) return null } - fun nextBoolean(tag: ULong?) = nextBoolean(tag?.let { ulongArrayOf(it) }) + fun nextBoolean(tag: ULong) = nextBoolean(ulongArrayOf(tag)) - fun nextBoolean(tags: ULongArray?): Boolean { + fun nextBoolean(tags: ULongArray? = null): Boolean { processTags(tags) val ans = when (curByte) { TRUE -> true @@ -383,13 +383,13 @@ internal class CborDecoder(private val input: ByteArrayInput) { return ans } - fun startArray(tag: ULong?) = startArray(tag?.let { ulongArrayOf(it) }) + fun startArray(tag: ULong) = startArray(ulongArrayOf(tag)) - fun startArray(tags: ULongArray?) = startSized(tags, BEGIN_ARRAY, HEADER_ARRAY, "array") + fun startArray(tags: ULongArray? = null) = startSized(tags, BEGIN_ARRAY, HEADER_ARRAY, "array") - fun startMap(tag: ULong?) = startMap(tag?.let { ulongArrayOf(it) }) + fun startMap(tag: ULong) = startMap(ulongArrayOf(tag)) - fun startMap(tags: ULongArray?) = startSized(tags, BEGIN_MAP, HEADER_MAP, "map") + fun startMap(tags: ULongArray? = null) = startSized(tags, BEGIN_MAP, HEADER_MAP, "map") private fun startSized( tags: ULongArray?, @@ -413,8 +413,8 @@ internal class CborDecoder(private val input: ByteArrayInput) { fun end() = skipByte(BREAK) - fun nextByteString(tag: ULong?) = nextByteString(tag?.let { ulongArrayOf(it) }) - fun nextByteString(tags: ULongArray?): ByteArray { + fun nextByteString(tag: ULong) = nextByteString(ulongArrayOf(tag)) + fun nextByteString(tags: ULongArray? = null): ByteArray { processTags(tags) if ((curByte and 0b111_00000) != HEADER_BYTE_STRING.toInt()) throw CborDecodingException("start of byte string", curByte) @@ -423,8 +423,8 @@ internal class CborDecoder(private val input: ByteArrayInput) { return arr } - fun nextString(tag: ULong?) = nextString(tag?.let { ulongArrayOf(it) }) - fun nextString(tags: ULongArray?) = nextTaggedString(tags).first + fun nextString(tag: ULong) = nextString(ulongArrayOf(tag)) + fun nextString(tags: ULongArray? = null) = nextTaggedString(tags).first //used to r fun nextTaggedString() = nextTaggedString(null) @@ -460,11 +460,28 @@ internal class CborDecoder(private val input: ByteArrayInput) { } readByte() } - return if (collectedTags.isEmpty()) null else collectedTags.toULongArray() + + return (if (collectedTags.isEmpty()) null else collectedTags.toULongArray()).also { collected -> + tags?.let { + if (!it.contentEquals(collected)) throw CborDecodingException( + "CBOR tags ${ + collected?.joinToString( + prefix = "[", + postfix = "]" + ) { it.toString() } + } do not match expected tags ${ + it.joinToString( + prefix = "[", + postfix = "]" + ) { it.toString() } + }" + ) + } + } } - fun nextNumber(tag: ULong?): Long = nextNumber(tag?.let { ulongArrayOf(it) }) - fun nextNumber(tags: ULongArray?): Long { + fun nextNumber(tag: ULong): Long = nextNumber(ulongArrayOf(tag)) + fun nextNumber(tags: ULongArray? = null): Long { processTags(tags) val res = readNumber() readByte() @@ -508,9 +525,9 @@ internal class CborDecoder(private val input: ByteArrayInput) { return array } - fun nextFloat(tag: ULong?) = nextFloat(tag?.let { ulongArrayOf(it) }) + fun nextFloat(tag: ULong) = nextFloat(ulongArrayOf(tag)) - fun nextFloat(tags: ULongArray?): Float { + fun nextFloat(tags: ULongArray? = null): Float { processTags(tags) val res = when (curByte) { NEXT_FLOAT -> Float.fromBits(readInt()) @@ -521,8 +538,10 @@ internal class CborDecoder(private val input: ByteArrayInput) { return res } - fun nextDouble(): Double { - skipOverTags() + fun nextDouble(tag: ULong) = nextDouble(ulongArrayOf(tag)) + + fun nextDouble(tags: ULongArray? = null): Double { + processTags(tags) val res = when (curByte) { NEXT_DOUBLE -> Double.fromBits(readLong()) NEXT_FLOAT -> Float.fromBits(readInt()).toDouble() @@ -569,7 +588,7 @@ internal class CborDecoder(private val input: ByteArrayInput) { * been skipped, the "length stack" is [pruned][prune]. For indefinite length elements, a special marker is added to * the "length stack" which is only popped from the "length stack" when a CBOR [break][isEnd] is encountered. */ - fun skipElement(tags: ULongArray?) { + fun skipElement(tags: ULongArray? = null) { val lengthStack = mutableListOf() processTags(tags) @@ -684,8 +703,10 @@ internal class CborDecoder(private val input: ByteArrayInput) { private fun SerialDescriptor.getElementIndexOrThrow(name: String): Int { val index = getElementIndex(name) if (index == CompositeDecoder.UNKNOWN_NAME) - throw SerializationException("$serialName does not contain element with name '$name." + - " You can enable 'CborBuilder.ignoreUnknownKeys' property to ignore unknown keys") + throw SerializationException( + "$serialName does not contain element with name '$name." + + " You can enable 'CborBuilder.ignoreUnknownKeys' property to ignore unknown keys" + ) return index } @@ -742,6 +763,7 @@ private fun floatFromHalfBits(bits: Short): Float { exp = SINGLE_PRECISION_MAX_EXPONENT mant = halfMant } + 0 -> { if (halfMant == 0) { // if exponent and mantissa are zero - value is zero @@ -754,6 +776,7 @@ private fun floatFromHalfBits(bits: Short): Float { return if (negative) -res else res } } + else -> { // normalized value exp = (halfExp + (SINGLE_PRECISION_EXPONENT_BIAS - HALF_PRECISION_EXPONENT_BIAS)) diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt index 199b8da2f..75929176a 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt @@ -69,16 +69,18 @@ class CborReaderTest { HexConverter.parseHexBinary("cafe") ) // with maps, lists & strings of indefinite length - assertEquals(test, Cbor.decodeFromHexString( - TypesUmbrella.serializer(), - "bf637374726d48656c6c6f2c20776f726c64216169182a686e756c6c61626c65f6646c6973749f61616162ff636d6170bf01f502f4ff65696e6e6572bf6161636c6f6cff6a696e6e6572734c6973749fbf6161636b656bffff6a62797465537472696e675f42cafeff696279746541727261799f383521ffff" - ) + assertEquals( + test, Cbor.decodeFromHexString( + TypesUmbrella.serializer(), + "bf637374726d48656c6c6f2c20776f726c64216169182a686e756c6c61626c65f6646c6973749f61616162ff636d6170bf01f502f4ff65696e6e6572bf6161636c6f6cff6a696e6e6572734c6973749fbf6161636b656bffff6a62797465537472696e675f42cafeff696279746541727261799f383521ffff" + ) ) // with maps, lists & strings of definite length - assertEquals(test, Cbor.decodeFromHexString( - TypesUmbrella.serializer(), - "a9646c6973748261616162686e756c6c61626c65f6636d6170a202f401f56169182a6a696e6e6572734c69737481a16161636b656b637374726d48656c6c6f2c20776f726c642165696e6e6572a16161636c6f6c6a62797465537472696e6742cafe6962797465417272617982383521" - ) + assertEquals( + test, Cbor.decodeFromHexString( + TypesUmbrella.serializer(), + "a9646c6973748261616162686e756c6c61626c65f6636d6170a202f401f56169182a6a696e6e6572734c69737481a16161636b656b637374726d48656c6c6f2c20776f726c642165696e6e6572a16161636c6f6c6a62797465537472696e6742cafe6962797465417272617982383521" + ) ) } @@ -419,6 +421,50 @@ class CborReaderTest { */ @Test fun testSkipTags() { + /* + * A4 # map(4) + * 61 # text(1) + * 61 # "a" + * CC # tag(12) + * 1B FFFFFFFFFFFFFFFF # unsigned(18446744073709551615) + * D8 22 # tag(34) + * 61 # text(1) + * 62 # "b" + * 20 # negative(0) + * D8 38 # tag(56) + * 61 # text(1) + * 63 # "c" + * D8 4E # tag(78) + * 42 # bytes(2) + * CAFE # "\xCA\xFE" + * 61 # text(1) + * 64 # "d" + * D8 5A # tag(90) + * CC # tag(12) + * 6B # text(11) + * 48656C6C6F20776F726C64 # "Hello world" + */ + withDecoder("A46161CC1BFFFFFFFFFFFFFFFFD822616220D8386163D84E42CAFE6164D85ACC6B48656C6C6F20776F726C64") { + expectMap(size = 4) + expect("a") + skipElement() // unsigned(18446744073709551615) + expect("b") + skipElement() // negative(0) + expect("c") + skipElement() // "\xCA\xFE" + expect("d") + skipElement() // "Hello world" + expectEof() + } + } + + /** + * Tests that skipping unknown keys also skips over associated tags. + * + * Includes tags on the key, tags on the value, and tags on both key and value. + */ + @Test + fun testVerifyTags() { /* * A4 # map(4) * 61 # text(1) @@ -612,24 +658,24 @@ class CborReaderTest { @Test fun testReadCustomByteString() { assertEquals( - expected = TypeWithCustomByteString(CustomByteString(0x11, 0x22, 0x33)), - actual = Cbor.decodeFromHexString("bf617843112233ff") + expected = TypeWithCustomByteString(CustomByteString(0x11, 0x22, 0x33)), + actual = Cbor.decodeFromHexString("bf617843112233ff") ) } @Test fun testReadNullableCustomByteString() { assertEquals( - expected = TypeWithNullableCustomByteString(CustomByteString(0x11, 0x22, 0x33)), - actual = Cbor.decodeFromHexString("bf617843112233ff") + expected = TypeWithNullableCustomByteString(CustomByteString(0x11, 0x22, 0x33)), + actual = Cbor.decodeFromHexString("bf617843112233ff") ) } @Test fun testReadNullCustomByteString() { assertEquals( - expected = TypeWithNullableCustomByteString(null), - actual = Cbor.decodeFromHexString("bf6178f6ff") + expected = TypeWithNullableCustomByteString(null), + actual = Cbor.decodeFromHexString("bf6178f6ff") ) } @@ -663,6 +709,32 @@ class CborReaderTest { @Test fun testIgnoresTagsOnStrings() { + /* + * 84 # array(4) + * 68 # text(8) + * 756E746167676564 # "untagged" + * C0 # tag(0) + * 68 # text(8) + * 7461676765642D30 # "tagged-0" + * D8 F5 # tag(245) + * 6A # text(10) + * 7461676765642D323435 # "tagged-244" + * D9 3039 # tag(12345) + * 6C # text(12) + * 7461676765642D3132333435 # "tagged-12345" + * + */ + withDecoder("8468756E746167676564C0687461676765642D30D8F56A7461676765642D323435D930396C7461676765642D3132333435") { + assertEquals(4, startArray()) + assertEquals("untagged", nextString()) + assertEquals("tagged-0", nextString()) + assertEquals("tagged-245", nextString()) + assertEquals("tagged-12345", nextString()) + } + } + + @Test + fun testVerifyTagsOnStrings() { /* * 84 # array(4) * 68 # text(8) @@ -689,6 +761,33 @@ class CborReaderTest { @Test fun testIgnoresTagsOnNumbers() { + /* + * 86 # array(6) + * 18 7B # unsigned(123) + * C0 # tag(0) + * 1A 0001E240 # unsigned(123456) + * D8 F5 # tag(245) + * 1A 000F423F # unsigned(999999) + * D9 3039 # tag(12345) + * 38 31 # negative(49) + * D8 22 # tag(34) + * FB 3FE161F9F01B866E # primitive(4603068020252444270) + * D9 0237 # tag(567) + * FB 401999999999999A # primitive(4618891777831180698) + */ + withDecoder("86187BC01A0001E240D8F51A000F423FD930393831D822FB3FE161F9F01B866ED90237FB401999999999999A") { + assertEquals(6, startArray()) + assertEquals(123, nextNumber()) + assertEquals(123456, nextNumber()) + assertEquals(999999, nextNumber()) + assertEquals(-50, nextNumber()) + assertEquals(0.54321, nextDouble(), 0.00001) + assertEquals(6.4, nextDouble(), 0.00001) + } + } + + @Test + fun testVerifiesTagsOnNumbers() { /* * 86 # array(6) * 18 7B # unsigned(123) @@ -716,6 +815,43 @@ class CborReaderTest { @Test fun testIgnoresTagsOnArraysAndMaps() { + /* + * A2 # map(2) + * 63 # text(3) + * 6D6170 # "map" + * D8 7B # tag(123) + * A1 # map(1) + * 68 # text(8) + * 74686973206D6170 # "this map" + * 6D # text(13) + * 69732074616767656420313233 # "is tagged 123" + * 65 # text(5) + * 6172726179 # "array" + * DA 0012D687 # tag(1234567) + * 83 # array(3) + * 6A # text(10) + * 74686973206172726179 # "this array" + * 69 # text(9) + * 697320746167676564 # "is tagged" + * 67 # text(7) + * 31323334353637 # "1234567" + */ + withDecoder("A2636D6170D87BA16874686973206D61706D69732074616767656420313233656172726179DA0012D687836A74686973206172726179696973207461676765646731323334353637") { + assertEquals(2, startMap()) + assertEquals("map", nextString()) + assertEquals(1, startMap()) + assertEquals("this map", nextString()) + assertEquals("is tagged 123", nextString()) + assertEquals("array", nextString()) + assertEquals(3, startArray()) + assertEquals("this array", nextString()) + assertEquals("is tagged", nextString()) + assertEquals("1234567", nextString()) + } + } + + @Test + fun testVerifiesTagsOnArraysAndMaps() { /* * A2 # map(2) * 63 # text(3) @@ -752,12 +888,12 @@ class CborReaderTest { } } -private fun CborDecoder.expect(expected: String, currentTag:ULong?=null) { - assertEquals(expected, actual = nextString(currentTag), "string") +private fun CborDecoder.expect(expected: String, tag: ULong? = null) { + assertEquals(expected, actual = nextString(tag?.let { ulongArrayOf(it) }), "string") } -private fun CborDecoder.expectMap(size: Int, currentTag: ULong?=null) { - assertEquals(size, actual = startMap(currentTag), "map size") +private fun CborDecoder.expectMap(size: Int, tag: ULong? = null) { + assertEquals(size, actual = startMap(tag?.let { ulongArrayOf(it) }), "map size") } private fun CborDecoder.expectEof() { diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt new file mode 100644 index 000000000..743254598 --- /dev/null +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt @@ -0,0 +1,244 @@ +/* + * Copyright 2017-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.serialization.cbor + +import kotlinx.serialization.* +import kotlinx.serialization.cbor.internal.* +import kotlin.test.* + + +@Serializable +data class DataWithTags( + @Tagged(12uL) + val a: ULong, + + @KeyTags(34uL) + val b: Int, + + @KeyTags(56uL) + @Tagged(78uL) + @ByteString val c: ByteArray, + + @Tagged(90uL, 12uL) + val d: String +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + + other as DataWithTags + + if (a != other.a) return false + if (b != other.b) return false + if (!c.contentEquals(other.c)) return false + return d == other.d + } + + override fun hashCode(): Int { + var result = a.hashCode() + result = 31 * result + b + result = 31 * result + c.contentHashCode() + result = 31 * result + d.hashCode() + return result + } +} + +class CborTaggedTest { + + private val reference = DataWithTags( + a = 0xFFFFFFFuL, + b = -1, + c = byteArrayOf(0xCA.toByte(), 0xFE.toByte()), + d = "Hello World" + ) + + /* + * BF # map(*) + * 61 # text(1) + * 61 # "a" + * CC # tag(12) + * 1A 0FFFFFFF # unsigned(268435455) + * D8 22 # tag(34) + * 61 # text(1) + * 62 # "b" + * 20 # negative(0) + * D8 38 # tag(56) + * 61 # text(1) + * 63 # "c" + * D8 4E # tag(78) + * 42 # bytes(2) + * CAFE # "\xCA\xFE" + * 61 # text(1) + * 64 # "d" + * D8 5A # tag(90) + * CC # tag(12) + * 6B # text(11) + * 48656C6C6F20576F726C64 # "Hello World" + * FF # primitive(*) + */ + private val referenceHexString = + "bf6161cc1a0fffffffd822616220d8386163d84e42cafe6164d85acc6b48656c6c6f20576f726c64ff" + + /* + * BF # map(*) + * 61 # text(1) + * 61 # "a" + * CC # tag(12) + * 1A 0FFFFFFF # unsigned(268435455) + * 61 # text(1) + * 62 # "b" + * 20 # negative(0) + * 61 # text(1) + * 63 # "c" + * D8 4E # tag(78) + * 42 # bytes(2) + * CAFE # "\xCA\xFE" + * 61 # text(1) + * 64 # "d" + * D8 5A # tag(90) + * CC # tag(12) + * 6B # text(11) + * 48656C6C6F20576F726C64 # "Hello World" + * FF # primitive(*) + */ + private val noKeyTags = "bf6161cc1a0fffffff6162206163d84e42cafe6164d85acc6b48656c6c6f20576f726c64ff" + + /* + * BF # map(*) + * 61 # text(1) + * 61 # "a" + * 1A 0FFFFFFF # unsigned(268435455) + * D8 22 # tag(34) + * 61 # text(1) + * 62 # "b" + * 20 # negative(0) + * D8 38 # tag(56) + * 61 # text(1) + * 63 # "c" + * 42 # bytes(2) + * CAFE # "\xCA\xFE" + * 61 # text(1) + * 64 # "d" + * 6B # text(11) + * 48656C6C6F20576F726C64 # "Hello World" + * FF # primitive(*) + */ + private val noValueTags = "bf61611a0fffffffd822616220d838616342cafe61646b48656c6c6f20576f726c64ff" + + /* + * BF # map(*) + * 61 # text(1) + * 61 # "a" + * 1A 0FFFFFFF # unsigned(268435455) + * 61 # text(1) + * 62 # "b" + * 20 # negative(0) + * 61 # text(1) + * 63 # "c" + * 42 # bytes(2) + * CAFE # "\xCA\xFE" + * 61 # text(1) + * 64 # "d" + * 6B # text(11) + * 48656C6C6F20576F726C64 # "Hello World" + * FF # primitive(*) + * + */ + private val noTags = "bf61611a0fffffff616220616342cafe61646b48656c6c6f20576f726c64ff" + + @Test + fun writeReadVerifyTaggedClass() { + assertEquals(referenceHexString, Cbor.encodeToHexString(DataWithTags.serializer(), reference)) + assertEquals(reference, Cbor.decodeFromHexString(DataWithTags.serializer(), referenceHexString)) + } + + @Test + fun writeReadUntaggedKeys() { + assertEquals(noKeyTags, Cbor { writeKeyTags = false }.encodeToHexString(DataWithTags.serializer(), reference)) + assertEquals(reference, Cbor { verifyKeyTags = false }.decodeFromHexString(noKeyTags)) + assertEquals(reference, Cbor { verifyKeyTags = false }.decodeFromHexString(referenceHexString)) + assertFailsWith(CborDecodingException::class) { Cbor.decodeFromHexString(DataWithTags.serializer(), noKeyTags) } + assertFailsWith(CborDecodingException::class) { + Cbor { + verifyKeyTags = false + }.decodeFromHexString(DataWithTags.serializer(), noValueTags) + } + } + + @Test + fun writeReadUntaggedValues() { + assertEquals( + noValueTags, + Cbor { writeValueTags = false }.encodeToHexString(DataWithTags.serializer(), reference) + ) + assertEquals(reference, Cbor { verifyValueTags = false }.decodeFromHexString(noValueTags)) + assertEquals(reference, Cbor { verifyValueTags = false }.decodeFromHexString(referenceHexString)) + + assertFailsWith(CborDecodingException::class) { + Cbor.decodeFromHexString( + DataWithTags.serializer(), + noValueTags + ) + } + + assertFailsWith(CborDecodingException::class) { + Cbor { verifyValueTags = false }.decodeFromHexString( + DataWithTags.serializer(), + noKeyTags + ) + } + + } + + @Test + fun writeReadUntaggedEverything() { + assertEquals( + noTags, + Cbor { + writeValueTags = false + writeKeyTags = false + }.encodeToHexString(DataWithTags.serializer(), reference) + ) + assertEquals(reference, Cbor { + verifyKeyTags = false + verifyValueTags = false + }.decodeFromHexString(noTags)) + + assertEquals(reference, Cbor { + verifyKeyTags = false + verifyValueTags = false + }.decodeFromHexString(noKeyTags)) + + assertEquals(reference, Cbor { + verifyKeyTags = false + verifyValueTags = false + }.decodeFromHexString(noValueTags)) + + assertEquals(reference, Cbor { + verifyKeyTags = false + verifyValueTags = false + }.decodeFromHexString(referenceHexString)) + + assertFailsWith(CborDecodingException::class) { + Cbor.decodeFromHexString( + DataWithTags.serializer(), + noTags + ) + } + + } + + @Test + fun wrongTags() { + val wrongTag55ForPropertyC = "A46161CC1A0FFFFFFFD822616220D8376163D84E42CAFE6164D85ACC6B48656C6C6F20576F726C64" + assertFailsWith(CborDecodingException::class, message = "CBOR tags [55] do not match expected tags [56]") { + Cbor.decodeFromHexString( + DataWithTags.serializer(), + wrongTag55ForPropertyC + ) + } + assertEquals(reference, Cbor { verifyKeyTags = false }.decodeFromHexString(wrongTag55ForPropertyC)) + } +} \ No newline at end of file diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt index 2560c141e..d54a9c73e 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt @@ -33,22 +33,6 @@ class CbrWriterTest { ) } - @Test - fun writeTaggedClass() { - val test = WithTags( - a = 0xFFFFFFFuL, - b = -1, - c = byteArrayOf(0xCA.toByte(), 0xFE.toByte()), - d = "Hello World" - ) - val encoded = Cbor.encodeToHexString(WithTags.serializer(), test) - assertEquals( - "bf6161cc1a0fffffffd822616220d8386163d84e42cafe6164d85acc6b48656c6c6f20576f726c64ff", - encoded - ) - assertEquals(test, Cbor.decodeFromHexString(WithTags.serializer(), encoded)) - } - @Test fun writeManyNumbers() { val test = NumberTypesUmbrella( From 2788ec237ad25568a80118b8fbbc18ed1236021e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 5 Jul 2023 18:22:12 +0200 Subject: [PATCH 05/98] =?UTF-8?q?CBOR:=20refactor=20Tagged=20=E2=86=92=20V?= =?UTF-8?q?alueTags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commonMain/src/kotlinx/serialization/cbor/Cbor.kt | 8 ++++---- .../serialization/cbor/{Tagged.kt => ValueTags.kt} | 2 +- .../src/kotlinx/serialization/cbor/internal/Encoding.kt | 2 +- .../src/kotlinx/serialization/cbor/CborTaggedTest.kt | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) rename formats/cbor/commonMain/src/kotlinx/serialization/cbor/{Tagged.kt => ValueTags.kt} (94%) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index e6bc02a8b..9d8479442 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -25,10 +25,10 @@ import kotlinx.serialization.modules.* * False by default; meaning that properties with values equal to defaults will be elided. * @param ignoreUnknownKeys specifies if unknown CBOR elements should be ignored (skipped) when decoding. * @param writeKeyTags Specifies whether tags set using the [KeyTags] annotation should be written (or omitted) - * @param writeValueTags Specifies whether tags set using the [Tagged] annotation should be written (or omitted) + * @param writeValueTags Specifies whether tags set using the [ValueTags] annotation should be written (or omitted) * @param verifyKeyTags Specifies whether tags preceding map keys (i.e. properties) should be matched against the * [KeyTags] annotation during the deserialization process. Useful for lenient parsing - * @param verifyValueTags Specifies whether tags preceding values should be matched against the [Tagged] + * @param verifyValueTags Specifies whether tags preceding values should be matched against the [ValueTags] * annotation during the deserialization process. Useful for lenient parsing. */ @ExperimentalSerializationApi @@ -123,7 +123,7 @@ public class CborBuilder internal constructor(cbor: Cbor) { public var writeKeyTags: Boolean = cbor.writeKeyTags /** - * Specifies whether tags set using the [Tagged] annotation should be written (or omitted) + * Specifies whether tags set using the [ValueTags] annotation should be written (or omitted) */ public var writeValueTags: Boolean = cbor.writeKeyTags @@ -133,7 +133,7 @@ public class CborBuilder internal constructor(cbor: Cbor) { public var verifyKeyTags: Boolean = cbor.verifyKeyTags /** - * Specifies whether tags preceding values should be matched against the [Tagged] annotation during the deserialization process + * Specifies whether tags preceding values should be matched against the [ValueTags] annotation during the deserialization process */ public var verifyValueTags: Boolean = cbor.verifyValueTags diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/ValueTags.kt similarity index 94% rename from formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt rename to formats/cbor/commonMain/src/kotlinx/serialization/cbor/ValueTags.kt index 6581f00df..5dc2e6bb8 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tagged.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/ValueTags.kt @@ -26,7 +26,7 @@ import kotlinx.serialization.* @SerialInfo @Target(AnnotationTarget.PROPERTY) @ExperimentalSerializationApi -public annotation class Tagged(@OptIn(ExperimentalUnsignedTypes::class) vararg val tags: ULong) { +public annotation class ValueTags(@OptIn(ExperimentalUnsignedTypes::class) vararg val tags: ULong) { public companion object { public const val DATE_TIME_STANDARD: ULong = 0u; public const val DATE_TIME_EPOCH: ULong = 1u; diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 14256e42e..50f23ac18 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -728,7 +728,7 @@ private fun SerialDescriptor.isByteString(index: Int): Boolean { @OptIn(ExperimentalSerializationApi::class) private fun SerialDescriptor.getValueTags(index: Int): ULongArray? { - return (getElementAnnotations(index).find { it is Tagged } as Tagged?)?.tags + return (getElementAnnotations(index).find { it is ValueTags } as ValueTags?)?.tags } private fun SerialDescriptor.isInlineByteString(): Boolean { // inline item classes should only have 1 item diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt index 743254598..6d3f9ada4 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt @@ -11,17 +11,17 @@ import kotlin.test.* @Serializable data class DataWithTags( - @Tagged(12uL) + @ValueTags(12uL) val a: ULong, @KeyTags(34uL) val b: Int, @KeyTags(56uL) - @Tagged(78uL) + @ValueTags(78uL) @ByteString val c: ByteArray, - @Tagged(90uL, 12uL) + @ValueTags(90uL, 12uL) val d: String ) { override fun equals(other: Any?): Boolean { From c620fdb4cae0447ba215efb357291b35cb65555d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Mon, 10 Jul 2023 16:01:42 +0200 Subject: [PATCH 06/98] CBOR: Fixes (docs, whitespace, names, superflous params) --- .../cbor/{ValueTags.kt => Tags.kt} | 23 +++++++-- .../serialization/cbor/internal/Encoding.kt | 2 +- .../serialization/cbor/CborReaderTest.kt | 50 +++++++++---------- 3 files changed, 45 insertions(+), 30 deletions(-) rename formats/cbor/commonMain/src/kotlinx/serialization/cbor/{ValueTags.kt => Tags.kt} (73%) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/ValueTags.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tags.kt similarity index 73% rename from formats/cbor/commonMain/src/kotlinx/serialization/cbor/ValueTags.kt rename to formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tags.kt index 5dc2e6bb8..55dcaae96 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/ValueTags.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Tags.kt @@ -5,23 +5,22 @@ import kotlinx.serialization.* /** * Specifies that a property shall be tagged and serialized as CBOR major type 6: optional semantic tagging * of other major types. - * For types other than [ByteArray], [ByteString] will have no effect. * * Example usage: * * ``` * @Serializable * data class Data( - * @Tagged(1337uL) + * @ValueTags(1337uL) * @ByteString * val a: ByteArray, // CBOR major type 6 1337(major type 2: a byte string). * - * @Tagged(1234567uL) + * @ValueTags(1234567uL) * val b: ByteArray // CBOR major type 6 1234567(major type 4: an array of data items). * ) * ``` * - * See [RFC 7049 2.4. Optional Tagging of Items](https://datatracker.ietf.org/doc/html/rfc7049#section-2.4). + * See [RFC 8949 3.4. Tagging of Items](https://datatracker.ietf.org/doc/html/rfc8949#name-tagging-of-items). */ @SerialInfo @Target(AnnotationTarget.PROPERTY) @@ -47,6 +46,22 @@ public annotation class ValueTags(@OptIn(ExperimentalUnsignedTypes::class) varar } } +/** + * Specifies that a key (i.e. a property identifier) shall be tagged and serialized as CBOR major type 6: optional + * semantic tagging of other major types. + * + * Example usage: + * + * ``` + * @Serializable + * data class Data( + * @KeyTags(34uL) + * val b: Int = -1 // results in the CBOR equivalent of 34("b"): -1 + * ) + * ``` + * + * See [RFC 8949 3.4. Tagging of Items](https://datatracker.ietf.org/doc/html/rfc8949#name-tagging-of-items). + */ @SerialInfo @Target(AnnotationTarget.PROPERTY) @ExperimentalSerializationApi diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 50f23ac18..4b4e85abf 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -426,7 +426,7 @@ internal class CborDecoder(private val input: ByteArrayInput) { fun nextString(tag: ULong) = nextString(ulongArrayOf(tag)) fun nextString(tags: ULongArray? = null) = nextTaggedString(tags).first - //used to r + //used for reading the tag names and names of tagged keys (of maps, and serialized classes) fun nextTaggedString() = nextTaggedString(null) private fun nextTaggedString(tags: ULongArray?): Pair { diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt index 75929176a..d24970876 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt @@ -21,32 +21,32 @@ class CborReaderTest { @Test fun testDecodeIntegers() { withDecoder("0C1903E8") { - assertEquals(12L, nextNumber(null)) - assertEquals(1000L, nextNumber(null)) + assertEquals(12L, nextNumber()) + assertEquals(1000L, nextNumber()) } withDecoder("203903e7") { - assertEquals(-1L, nextNumber(null)) - assertEquals(-1000L, nextNumber(null)) + assertEquals(-1L, nextNumber()) + assertEquals(-1000L, nextNumber()) } } @Test fun testDecodeStrings() { withDecoder("6568656C6C6F") { - assertEquals("hello", nextString(null)) + assertEquals("hello", nextString()) } withDecoder("7828737472696E672074686174206973206C6F6E676572207468616E2032332063686172616374657273") { - assertEquals("string that is longer than 23 characters", nextString(null)) + assertEquals("string that is longer than 23 characters", nextString()) } } @Test fun testDecodeDoubles() { withDecoder("fb7e37e43c8800759c") { - assertEquals(1e+300, nextDouble(null)) + assertEquals(1e+300, nextDouble()) } withDecoder("fa47c35000") { - assertEquals(100000.0f, nextFloat(null)) + assertEquals(100000.0f, nextFloat()) } } @@ -106,7 +106,7 @@ class CborReaderTest { withDecoder(input = "5F44aabbccdd43eeff99FF") { assertEquals( expected = "aabbccddeeff99", - actual = HexConverter.printHexBinary(nextByteString(null), lowerCase = true) + actual = HexConverter.printHexBinary(nextByteString(), lowerCase = true) ) } } @@ -253,13 +253,13 @@ class CborReaderTest { withDecoder("a461611bffffffffffffffff616220616342cafe61646b48656c6c6f20776f726c64") { expectMap(size = 4) expect("a") - skipElement(null) // unsigned(18446744073709551615) + skipElement() // unsigned(18446744073709551615) expect("b") - skipElement(null) // negative(0) + skipElement() // negative(0) expect("c") - skipElement(null) // "\xCA\xFE" + skipElement() // "\xCA\xFE" expect("d") - skipElement(null) // "Hello world" + skipElement() // "Hello world" expectEof() } } @@ -284,9 +284,9 @@ class CborReaderTest { withDecoder("a2616140616260") { expectMap(size = 2) expect("a") - skipElement(null) // bytes(0) + skipElement() // bytes(0) expect("b") - skipElement(null) // text(0) + skipElement() // text(0) expectEof() } } @@ -320,9 +320,9 @@ class CborReaderTest { withDecoder("a26161830118ff1a000100006162a26178676b6f746c696e7861796d73657269616c697a6174696f6e") { expectMap(size = 2) expect("a") - skipElement(null) // [1, 255, 65536] + skipElement() // [1, 255, 65536] expect("b") - skipElement(null) // {"x": "kotlinx", "y": "serialization"} + skipElement() // {"x": "kotlinx", "y": "serialization"} expectEof() } } @@ -345,9 +345,9 @@ class CborReaderTest { withDecoder("a26161806162a0") { expectMap(size = 2) expect("a") - skipElement(null) // [1, 255, 65536] + skipElement() // [1, 255, 65536] expect("b") - skipElement(null) // {"x": "kotlinx", "y": "serialization"} + skipElement() // {"x": "kotlinx", "y": "serialization"} expectEof() } } @@ -403,13 +403,13 @@ class CborReaderTest { withDecoder("a461615f42cafe43010203ff61627f6648656c6c6f2065776f726c64ff61639f676b6f746c696e786d73657269616c697a6174696f6eff6164bf613101613202613303ff") { expectMap(size = 4) expect("a") - skipElement(null) // "\xCA\xFE\x01\x02\x03" + skipElement() // "\xCA\xFE\x01\x02\x03" expect("b") - skipElement(null) // "Hello world" + skipElement() // "Hello world" expect("c") - skipElement(null) // ["kotlinx", "serialization"] + skipElement() // ["kotlinx", "serialization"] expect("d") - skipElement(null) // {"1": 1, "2": 2, "3": 3} + skipElement() // {"1": 1, "2": 2, "3": 3} expectEof() } } @@ -493,7 +493,7 @@ class CborReaderTest { expect("a") skipElement(12uL) // unsigned(18446744073709551615) expect("b", 34uL) - skipElement(null) // negative(0) + skipElement(null) // negative(0); explicitly setting parameter to null for clearer semantics expect("c", 56uL) skipElement(78uL) // "\xCA\xFE" expect("d") @@ -736,7 +736,7 @@ class CborReaderTest { @Test fun testVerifyTagsOnStrings() { /* - * 84 # array(4) + * 84 # array(4) * 68 # text(8) * 756E746167676564 # "untagged" * C0 # tag(0) From 0d1b31bbfe357b9faf39e6a5412dff633ae17bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Tue, 18 Jul 2023 21:03:47 +0200 Subject: [PATCH 07/98] CBOR: serialize cbor to tree --- .../src/kotlinx/serialization/cbor/Cbor.kt | 3 ++ .../serialization/cbor/internal/Encoding.kt | 54 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index 9d8479442..653a21816 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -48,6 +48,9 @@ public sealed class Cbor( public companion object Default : Cbor(false, false, true, true, true, true, EmptySerializersModule()) override fun encodeToByteArray(serializer: SerializationStrategy, value: T): ByteArray { + val tree =CborTree(this) + tree.encodeSerializableValue(serializer,value) + val output = ByteArrayOutput() val dumper = CborWriter(this, CborEncoder(output)) dumper.encodeSerializableValue(serializer, value) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 4b4e85abf..e95de0e8d 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -58,6 +58,60 @@ private open class CborListWriter(cbor: Cbor, encoder: CborEncoder) : CborWriter override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean = true } +internal class CborTree(private val cbor: Cbor) : AbstractEncoder() { + + class Node(val descriptor: SerialDescriptor?, var data: Any?, val parent: Node?) { + val children = mutableListOf() + + override fun toString(): String { + return "(${descriptor?.serialName}:${descriptor?.kind}, $data, ${children.joinToString { it.toString() }})" + } + } + + private var currentNode = Node(null, null, null) + val root = currentNode + override val serializersModule: SerializersModule + get() = cbor.serializersModule + + override fun shouldEncodeElementDefault(descriptor: SerialDescriptor, index: Int): Boolean = cbor.encodeDefaults + + override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder { + println("Begin Structure for ${descriptor.serialName}") + if (currentNode == root) + currentNode = Node(descriptor, null, currentNode).apply { currentNode.children += this } + else { + currentNode = currentNode.children.last() + // currentNode.parent?.children?.removeLast() + } + return this + } + + override fun endStructure(descriptor: SerialDescriptor) { + println("End Structure for ${descriptor.serialName}") + currentNode = currentNode.parent ?: throw SerializationException("Root node reached!") + + if (currentNode.parent == null) + println(currentNode) + + } + + override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { + println("EncodeElement for ${descriptor.getElementDescriptor(index)}") + currentNode.children += Node(descriptor.getElementDescriptor(index), null, currentNode) + return true + } + + override fun encodeNull() { + println("EncodeNull") + } + + override fun encodeValue(value: Any) { + println("EncodeValue $value") + currentNode.children.last().data = value + } +} + + // Writes class as map [fieldName, fieldValue] internal open class CborWriter(private val cbor: Cbor, protected val encoder: CborEncoder) : AbstractEncoder() { override val serializersModule: SerializersModule From 4f82ec87ea23b0db2c03b98aea75e15ecb7b6d74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 19 Jul 2023 08:24:25 +0200 Subject: [PATCH 08/98] CBOR: tree fixes --- .../src/kotlinx/serialization/cbor/Cbor.kt | 5 +- .../serialization/cbor/internal/Encoding.kt | 50 +++++++++++++------ 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index 653a21816..14bd7e890 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -48,8 +48,9 @@ public sealed class Cbor( public companion object Default : Cbor(false, false, true, true, true, true, EmptySerializersModule()) override fun encodeToByteArray(serializer: SerializationStrategy, value: T): ByteArray { - val tree =CborTree(this) - tree.encodeSerializableValue(serializer,value) + val tree =CborTree(this).pass1Accumulate(serializer,value) + tree.pass2PruneNulls() + println(tree) val output = ByteArrayOutput() val dumper = CborWriter(this, CborEncoder(output)) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index e95de0e8d..87e82ffcd 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -60,25 +60,38 @@ private open class CborListWriter(cbor: Cbor, encoder: CborEncoder) : CborWriter internal class CborTree(private val cbor: Cbor) : AbstractEncoder() { - class Node(val descriptor: SerialDescriptor?, var data: Any?, val parent: Node?) { + class Node( + val descriptor: SerialDescriptor?, + var data: Any?, + val parent: Node?, + val index: Int, + val name: String? + ) { val children = mutableListOf() override fun toString(): String { return "(${descriptor?.serialName}:${descriptor?.kind}, $data, ${children.joinToString { it.toString() }})" } + + internal fun pass2PruneNulls() { + if (descriptor?.kind == StructureKind.CLASS || descriptor?.kind == StructureKind.OBJECT) { + children.removeAll { it.data == null && it.children.isEmpty() } + children.forEach { it.pass2PruneNulls() } + } + } } - private var currentNode = Node(null, null, null) - val root = currentNode + private var currentNode = Node(null, null, null, -1, null) + val root: Node get() = currentNode.children.first() override val serializersModule: SerializersModule get() = cbor.serializersModule override fun shouldEncodeElementDefault(descriptor: SerialDescriptor, index: Int): Boolean = cbor.encodeDefaults override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder { - println("Begin Structure for ${descriptor.serialName}") - if (currentNode == root) - currentNode = Node(descriptor, null, currentNode).apply { currentNode.children += this } + //println("Begin Structure for ${descriptor.serialName}") + if (currentNode.parent == null) + currentNode = Node(descriptor, null, currentNode, -1, null).apply { currentNode.children += this } else { currentNode = currentNode.children.last() // currentNode.parent?.children?.removeLast() @@ -87,28 +100,37 @@ internal class CborTree(private val cbor: Cbor) : AbstractEncoder() { } override fun endStructure(descriptor: SerialDescriptor) { - println("End Structure for ${descriptor.serialName}") + //println("End Structure for ${descriptor.serialName}") currentNode = currentNode.parent ?: throw SerializationException("Root node reached!") - if (currentNode.parent == null) - println(currentNode) - } override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { - println("EncodeElement for ${descriptor.getElementDescriptor(index)}") - currentNode.children += Node(descriptor.getElementDescriptor(index), null, currentNode) + // println("EncodeElement for ${descriptor.getElementDescriptor(index)}") + currentNode.children += Node( + descriptor.getElementDescriptor(index), + null, + currentNode, + index, + descriptor.getElementName(index) + ) return true } override fun encodeNull() { - println("EncodeNull") + // println("EncodeNull") } override fun encodeValue(value: Any) { - println("EncodeValue $value") + //println("EncodeValue $value") currentNode.children.last().data = value } + + fun pass1Accumulate(serializer: SerializationStrategy, value: T): Node { + encodeSerializableValue(serializer, value) + return root + + } } From 135666601968c86d76255a6b48435b38e3f3acb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 19 Jul 2023 17:16:34 +0200 Subject: [PATCH 09/98] CBOR: frankensteined data tree --- .../src/kotlinx/serialization/cbor/Cbor.kt | 13 +- .../serialization/cbor/internal/Encoding.kt | 256 ++++++++++++------ 2 files changed, 176 insertions(+), 93 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index 14bd7e890..27ceb65d3 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -48,14 +48,19 @@ public sealed class Cbor( public companion object Default : Cbor(false, false, true, true, true, true, EmptySerializersModule()) override fun encodeToByteArray(serializer: SerializationStrategy, value: T): ByteArray { - val tree =CborTree(this).pass1Accumulate(serializer,value) - tree.pass2PruneNulls() - println(tree) + // val tree = CborTree(this).pass1Accumulate(serializer, value) + // tree.pass2PruneNulls() + // println(tree) val output = ByteArrayOutput() val dumper = CborWriter(this, CborEncoder(output)) dumper.encodeSerializableValue(serializer, value) - return output.toByteArray() + return output.toByteArray().also {bytes-> + println(bytes.joinToString(separator = "", prefix = "", postfix = "") { + it.toUByte().toString(16).let { if (it.length == 1) "0$it" else it } + }) + + } } override fun decodeFromByteArray(deserializer: DeserializationStrategy, bytes: ByteArray): T { diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 87e82ffcd..2bca1e0f6 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -46,6 +46,7 @@ private const val SINGLE_PRECISION_MAX_EXPONENT = 0xFF private const val SINGLE_PRECISION_NORMALIZE_BASE = 0.5f +/* // Differs from List only in start byte private class CborMapWriter(cbor: Cbor, encoder: CborEncoder) : CborListWriter(cbor, encoder) { override fun writeBeginToken() = encoder.startMap() @@ -57,13 +58,19 @@ private open class CborListWriter(cbor: Cbor, encoder: CborEncoder) : CborWriter override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean = true } +*/ + -internal class CborTree(private val cbor: Cbor) : AbstractEncoder() { +// Writes class as map [fieldName, fieldValue] +internal open class CborWriter(private val cbor: Cbor, protected val encoder: CborEncoder) : AbstractEncoder() { - class Node( + + var encodeByteArrayAsByteString = false + + inner class Node( val descriptor: SerialDescriptor?, var data: Any?, - val parent: Node?, + var parent: Node?, val index: Int, val name: String? ) { @@ -73,148 +80,204 @@ internal class CborTree(private val cbor: Cbor) : AbstractEncoder() { return "(${descriptor?.serialName}:${descriptor?.kind}, $data, ${children.joinToString { it.toString() }})" } - internal fun pass2PruneNulls() { - if (descriptor?.kind == StructureKind.CLASS || descriptor?.kind == StructureKind.OBJECT) { - children.removeAll { it.data == null && it.children.isEmpty() } - children.forEach { it.pass2PruneNulls() } - } - } - } + fun encode() { + encodeElementPreamble() + if (parent?.descriptor?.isByteString(index) != true) { + if (children.isNotEmpty()) { + if (descriptor != null) + when (descriptor.kind) { + StructureKind.LIST, is PolymorphicKind -> encoder.startArray() + else -> encoder.startMap() + } - private var currentNode = Node(null, null, null, -1, null) - val root: Node get() = currentNode.children.first() - override val serializersModule: SerializersModule - get() = cbor.serializersModule - override fun shouldEncodeElementDefault(descriptor: SerialDescriptor, index: Int): Boolean = cbor.encodeDefaults + } - override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder { - //println("Begin Structure for ${descriptor.serialName}") - if (currentNode.parent == null) - currentNode = Node(descriptor, null, currentNode, -1, null).apply { currentNode.children += this } - else { - currentNode = currentNode.children.last() - // currentNode.parent?.children?.removeLast() - } - return this - } + children.forEach { it.encode() } + if (children.isNotEmpty() && descriptor != null) encoder.end() - override fun endStructure(descriptor: SerialDescriptor) { - //println("End Structure for ${descriptor.serialName}") - currentNode = currentNode.parent ?: throw SerializationException("Root node reached!") + } + data?.let { + it as ByteArray; it.forEach { encoder.writeByte(it.toInt()) } + } - } + } - override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { - // println("EncodeElement for ${descriptor.getElementDescriptor(index)}") - currentNode.children += Node( - descriptor.getElementDescriptor(index), - null, - currentNode, - index, - descriptor.getElementName(index) - ) - return true - } + fun encodeElementPreamble() { - override fun encodeNull() { - // println("EncodeNull") - } - override fun encodeValue(value: Any) { - //println("EncodeValue $value") - currentNode.children.last().data = value + if (cbor.writeKeyTags) { + parent?.descriptor?.getKeyTags(index)?.forEach { tag -> + val encodedTag = encoder.composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + } + } + if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) //TODO polymorphicKind? + name?.let { encoder.encodeString(it) } + + if (cbor.writeValueTags) { + parent?.descriptor?.getValueTags(index)?.forEach { tag -> + val encodedTag = encoder.composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + } + } + + } } - fun pass1Accumulate(serializer: SerializationStrategy, value: T): Node { - encodeSerializableValue(serializer, value) - return root - } -} + private var currentNode = Node(null, null, null, -1, null) + val root: Node get() = currentNode.children.first().apply { parent = null } -// Writes class as map [fieldName, fieldValue] -internal open class CborWriter(private val cbor: Cbor, protected val encoder: CborEncoder) : AbstractEncoder() { override val serializersModule: SerializersModule get() = cbor.serializersModule - private var encodeByteArrayAsByteString = false + //private var encodeByteArrayAsByteString = false @OptIn(ExperimentalSerializationApi::class) override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { if (encodeByteArrayAsByteString && serializer.descriptor == ByteArraySerializer().descriptor) { - encoder.encodeByteString(value as ByteArray) + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeByteString(value as ByteArray) }.toByteArray() } else { - encodeByteArrayAsByteString = encodeByteArrayAsByteString || serializer.descriptor.isInlineByteString() - super.encodeSerializableValue(serializer, value) } } override fun shouldEncodeElementDefault(descriptor: SerialDescriptor, index: Int): Boolean = cbor.encodeDefaults - protected open fun writeBeginToken() = encoder.startMap() + // protected open fun writeBeginToken() = encoder.startMap() //todo: Write size of map or array if known @OptIn(ExperimentalSerializationApi::class) override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder { + if (currentNode.parent == null) + currentNode = Node( + descriptor, + null, + currentNode, + -1, + null + ).apply { currentNode.children += this } + else { + currentNode = currentNode.children.last() + } + /* val writer = when (descriptor.kind) { StructureKind.LIST, is PolymorphicKind -> CborListWriter(cbor, encoder) StructureKind.MAP -> CborMapWriter(cbor, encoder) else -> CborWriter(cbor, encoder) } + writer = this writer.writeBeginToken() - return writer + return writer*/ + return this + } + + override fun endStructure(descriptor: SerialDescriptor) { + // encoder.end() + currentNode = currentNode.parent ?: throw SerializationException("Root node reached!") + if (currentNode.parent == null) { + currentNode.encode() + } } - override fun endStructure(descriptor: SerialDescriptor) = encoder.end() override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { encodeByteArrayAsByteString = descriptor.isByteString(index) - val name = descriptor.getElementName(index) + /* val name = descriptor.getElementName(index) + + if (cbor.writeKeyTags) { + descriptor.getKeyTags(index)?.forEach { tag -> + val encodedTag = encoder.composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + } + } + + encoder.encodeString(name) + + if (cbor.writeValueTags) { + descriptor.getValueTags(index)?.forEach { tag -> + val encodedTag = encoder.composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + } + } + */ + currentNode.children += Node( + descriptor.getElementDescriptor(index), + null, + currentNode, + index, + descriptor.getElementName(index) + ) + return true + } - if (cbor.writeKeyTags) { - descriptor.getKeyTags(index)?.forEach { tag -> - val encodedTag = encoder.composePositive(tag) - encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() - encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } - } - } + override fun encodeString(value: String) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeString(value) }.toByteArray() - encoder.encodeString(name) + } - if (cbor.writeValueTags) { - descriptor.getValueTags(index)?.forEach { tag -> - val encodedTag = encoder.composePositive(tag) - encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() - encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } - } - } - return true + override fun encodeFloat(value: Float) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeFloat(value) }.toByteArray() } - override fun encodeString(value: String) = encoder.encodeString(value) + override fun encodeDouble(value: Double) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeDouble(value) }.toByteArray() + } - override fun encodeFloat(value: Float) = encoder.encodeFloat(value) - override fun encodeDouble(value: Double) = encoder.encodeDouble(value) + override fun encodeChar(value: Char) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.code.toLong()) }.toByteArray() + } - override fun encodeChar(value: Char) = encoder.encodeNumber(value.code.toLong()) - override fun encodeByte(value: Byte) = encoder.encodeNumber(value.toLong()) - override fun encodeShort(value: Short) = encoder.encodeNumber(value.toLong()) - override fun encodeInt(value: Int) = encoder.encodeNumber(value.toLong()) - override fun encodeLong(value: Long) = encoder.encodeNumber(value) + override fun encodeByte(value: Byte) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + } - override fun encodeBoolean(value: Boolean) = encoder.encodeBoolean(value) + override fun encodeShort(value: Short) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + } + + override fun encodeInt(value: Int) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + } - override fun encodeNull() = encoder.encodeNull() + override fun encodeLong(value: Long) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value) }.toByteArray() + } + + override fun encodeBoolean(value: Boolean) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeBoolean(value) }.toByteArray() + } + + override fun encodeNull() { + currentNode.children.last().data = ByteArrayOutput().also { CborEncoder(it).encodeNull() }.toByteArray() + } @OptIn(ExperimentalSerializationApi::class) // KT-46731 override fun encodeEnum( enumDescriptor: SerialDescriptor, index: Int - ) = - encoder.encodeString(enumDescriptor.getElementName(index)) + ) { + currentNode.children.last().data = + ByteArrayOutput().also { CborEncoder(it).encodeString(enumDescriptor.getElementName(index)) } + .toByteArray() + } + } // For details of representation, see https://tools.ietf.org/html/rfc7049#section-2.1 @@ -817,6 +880,21 @@ private fun SerialDescriptor.getKeyTags(index: Int): ULongArray? { return (getElementAnnotations(index).find { it is KeyTags } as KeyTags?)?.tags } +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.isByteString(): Boolean { + return annotations.find { it is ByteString } != null +} + +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.getValueTags(): ULongArray? { + return (annotations.find { it is ValueTags } as ValueTags?)?.tags +} + +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.getKeyTags(): ULongArray? { + return (annotations.find { it is KeyTags } as KeyTags?)?.tags +} + private val normalizeBaseBits = SINGLE_PRECISION_NORMALIZE_BASE.toBits() From 76afa35f2095b518aaaf8c64a8720e00893aa948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 19 Jul 2023 17:31:15 +0200 Subject: [PATCH 10/98] CBOR: fix direct encoding of primitives --- .../src/kotlinx/serialization/cbor/Cbor.kt | 1 + .../serialization/cbor/internal/Encoding.kt | 77 ++++++++++++------- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index 27ceb65d3..92dab0a5c 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -55,6 +55,7 @@ public sealed class Cbor( val output = ByteArrayOutput() val dumper = CborWriter(this, CborEncoder(output)) dumper.encodeSerializableValue(serializer, value) + dumper.root.encode() return output.toByteArray().also {bytes-> println(bytes.joinToString(separator = "", prefix = "", postfix = "") { it.toUByte().toString(16).let { if (it.length == 1) "0$it" else it } diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 2bca1e0f6..0c064f247 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -97,13 +97,14 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb if (children.isNotEmpty() && descriptor != null) encoder.end() } + //byteStrings are encoded into the data already data?.let { it as ByteArray; it.forEach { encoder.writeByte(it.toInt()) } } } - fun encodeElementPreamble() { + private fun encodeElementPreamble() { if (cbor.writeKeyTags) { @@ -114,7 +115,7 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb } } if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) //TODO polymorphicKind? - name?.let { encoder.encodeString(it) } + name?.let { encoder.encodeString(it) } //indieces are put into the name field. we don't want to write those, as it would result in double writes if (cbor.writeValueTags) { parent?.descriptor?.getValueTags(index)?.forEach { tag -> @@ -129,7 +130,7 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb private var currentNode = Node(null, null, null, -1, null) - val root: Node get() = currentNode.children.first().apply { parent = null } + val root: Node get() = currentNode//.children.first().apply { parent = null } override val serializersModule: SerializersModule @@ -180,9 +181,9 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb override fun endStructure(descriptor: SerialDescriptor) { // encoder.end() currentNode = currentNode.parent ?: throw SerializationException("Root node reached!") - if (currentNode.parent == null) { + /*if (currentNode.parent == null) { currentNode.encode() - } + }*/ } @@ -219,53 +220,73 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb } override fun encodeString(value: String) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeString(value) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeString(value) }.toByteArray() + } } override fun encodeFloat(value: Float) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeFloat(value) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeFloat(value) }.toByteArray() + } } override fun encodeDouble(value: Double) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeDouble(value) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeDouble(value) }.toByteArray() + } } override fun encodeChar(value: Char) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.code.toLong()) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.code.toLong()) }.toByteArray() + } } override fun encodeByte(value: Byte) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + } } override fun encodeShort(value: Short) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + } } override fun encodeInt(value: Int) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() + } } override fun encodeLong(value: Long) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeNumber(value) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeNumber(value) }.toByteArray() + } } override fun encodeBoolean(value: Boolean) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeBoolean(value) }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeBoolean(value) }.toByteArray() + } } override fun encodeNull() { - currentNode.children.last().data = ByteArrayOutput().also { CborEncoder(it).encodeNull() }.toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = ByteArrayOutput().also { CborEncoder(it).encodeNull() }.toByteArray() + } } @OptIn(ExperimentalSerializationApi::class) // KT-46731 @@ -273,9 +294,11 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb enumDescriptor: SerialDescriptor, index: Int ) { - currentNode.children.last().data = - ByteArrayOutput().also { CborEncoder(it).encodeString(enumDescriptor.getElementName(index)) } - .toByteArray() + (currentNode.children.lastOrNull() ?: currentNode).apply { + data = + ByteArrayOutput().also { CborEncoder(it).encodeString(enumDescriptor.getElementName(index)) } + .toByteArray() + } } } From ce9eefd24f7f42437bdfb7fc9408c587856d9fc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 19 Jul 2023 17:52:31 +0200 Subject: [PATCH 11/98] CBOR: write definite lengths --- .../serialization/cbor/internal/Encoding.kt | 79 ++++++------------- 1 file changed, 23 insertions(+), 56 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 0c064f247..b9b66a296 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -46,21 +46,6 @@ private const val SINGLE_PRECISION_MAX_EXPONENT = 0xFF private const val SINGLE_PRECISION_NORMALIZE_BASE = 0.5f -/* -// Differs from List only in start byte -private class CborMapWriter(cbor: Cbor, encoder: CborEncoder) : CborListWriter(cbor, encoder) { - override fun writeBeginToken() = encoder.startMap() -} - -// Writes all elements consequently, except size - CBOR supports maps and arrays of indefinite length -private open class CborListWriter(cbor: Cbor, encoder: CborEncoder) : CborWriter(cbor, encoder) { - override fun writeBeginToken() = encoder.startArray() - - override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean = true -} -*/ - - // Writes class as map [fieldName, fieldValue] internal open class CborWriter(private val cbor: Cbor, protected val encoder: CborEncoder) : AbstractEncoder() { @@ -86,18 +71,19 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb if (children.isNotEmpty()) { if (descriptor != null) when (descriptor.kind) { - StructureKind.LIST, is PolymorphicKind -> encoder.startArray() - else -> encoder.startMap() + StructureKind.LIST, is PolymorphicKind -> encoder.startArray(children.size.toULong()) + is StructureKind.MAP -> encoder.startMap((children.size/2).toULong()) + else -> encoder.startMap(children.size.toULong()) } } children.forEach { it.encode() } - if (children.isNotEmpty() && descriptor != null) encoder.end() + //if (children.isNotEmpty() && descriptor != null) encoder.end() } - //byteStrings are encoded into the data already + //byteStrings are encoded into the data already, as are primitives data?.let { it as ByteArray; it.forEach { encoder.writeByte(it.toInt()) } } @@ -130,14 +116,12 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb private var currentNode = Node(null, null, null, -1, null) - val root: Node get() = currentNode//.children.first().apply { parent = null } + val root: Node get() = currentNode override val serializersModule: SerializersModule get() = cbor.serializersModule - //private var encodeByteArrayAsByteString = false - @OptIn(ExperimentalSerializationApi::class) override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { if (encodeByteArrayAsByteString && serializer.descriptor == ByteArraySerializer().descriptor) { @@ -150,7 +134,6 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb override fun shouldEncodeElementDefault(descriptor: SerialDescriptor, index: Int): Boolean = cbor.encodeDefaults - // protected open fun writeBeginToken() = encoder.startMap() //todo: Write size of map or array if known @OptIn(ExperimentalSerializationApi::class) @@ -166,49 +149,17 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb else { currentNode = currentNode.children.last() } - /* - val writer = when (descriptor.kind) { - StructureKind.LIST, is PolymorphicKind -> CborListWriter(cbor, encoder) - StructureKind.MAP -> CborMapWriter(cbor, encoder) - else -> CborWriter(cbor, encoder) - } - writer = this - writer.writeBeginToken() - return writer*/ return this } override fun endStructure(descriptor: SerialDescriptor) { - // encoder.end() currentNode = currentNode.parent ?: throw SerializationException("Root node reached!") - /*if (currentNode.parent == null) { - currentNode.encode() - }*/ + } override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { encodeByteArrayAsByteString = descriptor.isByteString(index) - /* val name = descriptor.getElementName(index) - - if (cbor.writeKeyTags) { - descriptor.getKeyTags(index)?.forEach { tag -> - val encodedTag = encoder.composePositive(tag) - encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() - encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } - } - } - - encoder.encodeString(name) - - if (cbor.writeValueTags) { - descriptor.getValueTags(index)?.forEach { tag -> - val encodedTag = encoder.composePositive(tag) - encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() - encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } - } - } - */ currentNode.children += Node( descriptor.getElementDescriptor(index), null, @@ -219,6 +170,8 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb return true } + //If any of the following functions are called for serializing raw primitives (i.e. something other than a class, + // list, map or array, no children exist and the root node needs the data override fun encodeString(value: String) { (currentNode.children.lastOrNull() ?: currentNode).apply { data = @@ -307,7 +260,21 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb internal class CborEncoder(private val output: ByteArrayOutput) { fun startArray() = output.write(BEGIN_ARRAY) + + fun startArray(size: ULong) { + val encodedNumber = composePositive(size) + encodedNumber[0] = encodedNumber[0] or HEADER_ARRAY.toUByte().toByte() + encodedNumber.forEach { writeByte(it.toUByte().toInt()) } + } fun startMap() = output.write(BEGIN_MAP) + + fun startMap(size: ULong) { + val encodedNumber = composePositive(size) + encodedNumber[0] = encodedNumber[0] or HEADER_MAP.toUByte().toByte() + encodedNumber.forEach { writeByte(it.toUByte().toInt()) } + } + + fun end() = output.write(BREAK) fun encodeNull() = output.write(NULL) From 8b328f4a4e66761691b7f8ada2b13d997256662f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Wed, 19 Jul 2023 18:11:38 +0200 Subject: [PATCH 12/98] CBOR: experiment with pruning null properties --- .../src/kotlinx/serialization/cbor/Cbor.kt | 32 +++-- .../serialization/cbor/internal/Encoding.kt | 32 ++++- .../serialization/cbor/CborDefLenTest.kt | 31 ++++ .../serialization/cbor/CborReaderTest.kt | 5 + .../cbor/CborRootLevelNullsTest.kt | 6 +- .../serialization/cbor/CborTaggedTest.kt | 134 +++++++++++++++++- .../serialization/cbor/CborWriterTest.kt | 46 ++++++ 7 files changed, 260 insertions(+), 26 deletions(-) create mode 100644 formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborDefLenTest.kt diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index 92dab0a5c..998d670ca 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -39,29 +39,23 @@ public sealed class Cbor( internal val writeValueTags: Boolean, internal val verifyKeyTags: Boolean, internal val verifyValueTags: Boolean, + internal val explicitNulls: Boolean, + internal val writeDefiniteLengths: Boolean, override val serializersModule: SerializersModule ) : BinaryFormat { /** * The default instance of [Cbor] */ - public companion object Default : Cbor(false, false, true, true, true, true, EmptySerializersModule()) + public companion object Default : Cbor(false, false, true, true, true, true, true, false, EmptySerializersModule()) override fun encodeToByteArray(serializer: SerializationStrategy, value: T): ByteArray { - // val tree = CborTree(this).pass1Accumulate(serializer, value) - // tree.pass2PruneNulls() - // println(tree) - val output = ByteArrayOutput() val dumper = CborWriter(this, CborEncoder(output)) dumper.encodeSerializableValue(serializer, value) - dumper.root.encode() - return output.toByteArray().also {bytes-> - println(bytes.joinToString(separator = "", prefix = "", postfix = "") { - it.toUByte().toString(16).let { if (it.length == 1) "0$it" else it } - }) + dumper.encode() + return output.toByteArray() - } } override fun decodeFromByteArray(deserializer: DeserializationStrategy, bytes: ByteArray): T { @@ -78,6 +72,8 @@ private class CborImpl( writeValueTags: Boolean, verifyKeyTags: Boolean, verifyValueTags: Boolean, + encodeNullProperties: Boolean, + writeDefiniteLengths: Boolean, serializersModule: SerializersModule ) : Cbor( @@ -87,6 +83,8 @@ private class CborImpl( writeValueTags, verifyKeyTags, verifyValueTags, + encodeNullProperties, + writeDefiniteLengths, serializersModule ) @@ -105,6 +103,8 @@ public fun Cbor(from: Cbor = Cbor, builderAction: CborBuilder.() -> Unit): Cbor builder.writeValueTags, builder.verifyKeyTags, builder.verifyValueTags, + builder.explicitNulls, + builder.writeDefiniteLengths, builder.serializersModule ) } @@ -147,6 +147,16 @@ public class CborBuilder internal constructor(cbor: Cbor) { */ public var verifyValueTags: Boolean = cbor.verifyValueTags + /** + * Specifies whether `null` values should be encoded for nullable properties + */ + public var explicitNulls: Boolean = cbor.explicitNulls + + /** + * specifies whether structures (maps, object, lists, etc.) should be encoded using definite length encoding + */ + public var writeDefiniteLengths: Boolean = cbor.writeDefiniteLengths + /** * Module with contextual and polymorphic serializers to be used in the resulting [Cbor] instance. */ diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index b9b66a296..cc5efe2cc 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -66,21 +66,36 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb } fun encode() { + val childNodes = + if (!cbor.explicitNulls && (descriptor?.kind is StructureKind.CLASS || descriptor?.kind is StructureKind.OBJECT)) { + children.filterNot { (it.data as ByteArray?)?.contentEquals(byteArrayOf(NULL.toByte())) == true } + } else children + encodeElementPreamble() + if (parent?.descriptor?.isByteString(index) != true) { - if (children.isNotEmpty()) { + if (children.isNotEmpty()) { //TODO: base this on structurekind not on emptyiness if (descriptor != null) when (descriptor.kind) { - StructureKind.LIST, is PolymorphicKind -> encoder.startArray(children.size.toULong()) - is StructureKind.MAP -> encoder.startMap((children.size/2).toULong()) - else -> encoder.startMap(children.size.toULong()) + StructureKind.LIST, is PolymorphicKind -> { + if (cbor.writeDefiniteLengths) encoder.startArray(childNodes.size.toULong()) + else encoder.startArray() + } + + is StructureKind.MAP -> { + if (cbor.writeDefiniteLengths) encoder.startMap((childNodes.size / 2).toULong()) else encoder.startMap() + } + + else -> { + if (cbor.writeDefiniteLengths) encoder.startMap(childNodes.size.toULong()) else encoder.startMap() + } } } - children.forEach { it.encode() } - //if (children.isNotEmpty() && descriptor != null) encoder.end() + childNodes.forEach { it.encode() } + if (children.isNotEmpty() && descriptor != null && !cbor.writeDefiniteLengths) encoder.end() } //byteStrings are encoded into the data already, as are primitives @@ -116,7 +131,9 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb private var currentNode = Node(null, null, null, -1, null) - val root: Node get() = currentNode + fun encode() { + currentNode.encode() + } override val serializersModule: SerializersModule @@ -266,6 +283,7 @@ internal class CborEncoder(private val output: ByteArrayOutput) { encodedNumber[0] = encodedNumber[0] or HEADER_ARRAY.toUByte().toByte() encodedNumber.forEach { writeByte(it.toUByte().toInt()) } } + fun startMap() = output.write(BEGIN_MAP) fun startMap(size: ULong) { diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborDefLenTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborDefLenTest.kt new file mode 100644 index 000000000..f50e403b5 --- /dev/null +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborDefLenTest.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2017-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.serialization.cbor + +import kotlinx.serialization.* +import kotlin.test.* + + +class CborDefLenTest { + @Test + fun writeComplicatedClass() { + val test = TypesUmbrella( + "Hello, world!", + 42, + null, + listOf("a", "b"), + mapOf(1 to true, 2 to false), + Simple("lol"), + listOf(Simple("kek")), + HexConverter.parseHexBinary("cafe"), + HexConverter.parseHexBinary("cafe") + ) + assertEquals( + "a9637374726d48656c6c6f2c20776f726c64216169182a686e756c6c61626c65f6646c6973748261616162636d6170a201f502f465696e6e6572a16161636c6f6c6a696e6e6572734c69737481a16161636b656b6a62797465537472696e6742cafe6962797465417272617982383521", + Cbor { writeDefiniteLengths = true }.encodeToHexString(TypesUmbrella.serializer(), test) + ) + } + +} \ No newline at end of file diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt index d24970876..82b3094d4 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt @@ -141,6 +141,11 @@ class CborReaderTest { ) } + @Test + fun testNullables() { + Cbor { explicitNulls = false }.decodeFromHexString("a0") + } + /** * CBOR hex data represents serialized versions of [TypesUmbrella] (which does **not** have a root property 'a') so * decoding to [Simple] (which has the field 'a') is expected to fail. diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborRootLevelNullsTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborRootLevelNullsTest.kt index 3e5483489..3872ca559 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborRootLevelNullsTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborRootLevelNullsTest.kt @@ -15,7 +15,9 @@ class CborRootLevelNullsTest { @Test fun testNull() { val obj: Simple? = null - val content = Cbor.encodeToByteArray(Simple.serializer().nullable, obj) - assertTrue(content.contentEquals(byteArrayOf(0xf6.toByte()))) + listOf(Cbor, Cbor { explicitNulls = false; writeDefiniteLengths = true }).forEach { + val content = it.encodeToByteArray(Simple.serializer().nullable, obj) + assertTrue(content.contentEquals(byteArrayOf(0xf6.toByte()))) + } } } diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt index 6d3f9ada4..c30ae6e74 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt @@ -81,6 +81,32 @@ class CborTaggedTest { private val referenceHexString = "bf6161cc1a0fffffffd822616220d8386163d84e42cafe6164d85acc6b48656c6c6f20576f726c64ff" + /* + * A4 # map(4) + * 61 # text(1) + * 61 # "a" + * CC # tag(12) + * 1A 0FFFFFFF # unsigned(268435455) + * D8 22 # tag(34) + * 61 # text(1) + * 62 # "b" + * 20 # negative(0) + * D8 38 # tag(56) + * 61 # text(1) + * 63 # "c" + * D8 4E # tag(78) + * 42 # bytes(2) + * CAFE # "\xCA\xFE" + * 61 # text(1) + * 64 # "d" + * D8 5A # tag(90) + * CC # tag(12) + * 6B # text(11) + * 48656C6C6F20576F726C64 # "Hello World" + */ + private val referenceHexStringDefLen = + "a46161cc1a0fffffffd822616220d8386163d84e42cafe6164d85acc6b48656c6c6f20576f726c64" + /* * BF # map(*) * 61 # text(1) @@ -105,6 +131,31 @@ class CborTaggedTest { */ private val noKeyTags = "bf6161cc1a0fffffff6162206163d84e42cafe6164d85acc6b48656c6c6f20576f726c64ff" + /* + * A4 # map(4) + * 61 # text(1) + * 61 # "a" + * CC # tag(12) + * 1A 0FFFFFFF # unsigned(268435455) + * 61 # text(1) + * 62 # "b" + * 20 # negative(0) + * 61 # text(1) + * 63 # "c" + * D8 4E # tag(78) + * 42 # bytes(2) + * CAFE # "\xCA\xFE" + * 61 # text(1) + * 64 # "d" + * D8 5A # tag(90) + * CC # tag(12) + * 6B # text(11) + * 48656C6C6F20576F726C64 # "Hello World" + * + * + */ + private val noKeyTagsDefLen = "a46161cc1a0fffffff6162206163d84e42cafe6164d85acc6b48656c6c6f20576f726c64" + /* * BF # map(*) * 61 # text(1) @@ -148,16 +199,49 @@ class CborTaggedTest { */ private val noTags = "bf61611a0fffffff616220616342cafe61646b48656c6c6f20576f726c64ff" + /* + * A4 # map(4) + * 61 # text(1) + * 61 # "a" + * 1A 0FFFFFFF # unsigned(268435455) + * 61 # text(1) + * 62 # "b" + * 20 # negative(0) + * 61 # text(1) + * 63 # "c" + * 42 # bytes(2) + * CAFE # "\xCA\xFE" + * 61 # text(1) + * 64 # "d" + * 6B # text(11) + * 48656C6C6F20576F726C64 # "Hello World" + * + */ + private val noTagsDefLen = "a461611a0fffffff616220616342cafe61646b48656c6c6f20576f726c64" + @Test fun writeReadVerifyTaggedClass() { assertEquals(referenceHexString, Cbor.encodeToHexString(DataWithTags.serializer(), reference)) + assertEquals( + referenceHexStringDefLen, + Cbor { writeDefiniteLengths = true }.encodeToHexString(DataWithTags.serializer(), reference) + ) assertEquals(reference, Cbor.decodeFromHexString(DataWithTags.serializer(), referenceHexString)) + assertEquals(reference, Cbor.decodeFromHexString(DataWithTags.serializer(), referenceHexStringDefLen)) } @Test fun writeReadUntaggedKeys() { assertEquals(noKeyTags, Cbor { writeKeyTags = false }.encodeToHexString(DataWithTags.serializer(), reference)) + assertEquals( + noKeyTagsDefLen, + Cbor { writeKeyTags = false;writeDefiniteLengths = true }.encodeToHexString( + DataWithTags.serializer(), + reference + ) + ) assertEquals(reference, Cbor { verifyKeyTags = false }.decodeFromHexString(noKeyTags)) + assertEquals(reference, Cbor { verifyKeyTags = false }.decodeFromHexString(noKeyTagsDefLen)) assertEquals(reference, Cbor { verifyKeyTags = false }.decodeFromHexString(referenceHexString)) assertFailsWith(CborDecodingException::class) { Cbor.decodeFromHexString(DataWithTags.serializer(), noKeyTags) } assertFailsWith(CborDecodingException::class) { @@ -201,11 +285,37 @@ class CborTaggedTest { writeKeyTags = false }.encodeToHexString(DataWithTags.serializer(), reference) ) + assertEquals( + noTagsDefLen, + Cbor { + writeValueTags = false + writeKeyTags = false + writeDefiniteLengths = true + }.encodeToHexString(DataWithTags.serializer(), reference) + ) + assertEquals(reference, Cbor { verifyKeyTags = false verifyValueTags = false }.decodeFromHexString(noTags)) + assertEquals(reference, Cbor { + verifyKeyTags = false + verifyValueTags = false + }.decodeFromHexString(noTagsDefLen)) + + assertEquals(reference, Cbor { + verifyKeyTags = false + verifyValueTags = false + writeDefiniteLengths = true + }.decodeFromHexString(noTags)) + + assertEquals(reference, Cbor { + verifyKeyTags = false + verifyValueTags = false + writeDefiniteLengths = true + }.decodeFromHexString(noTagsDefLen)) + assertEquals(reference, Cbor { verifyKeyTags = false verifyValueTags = false @@ -233,12 +343,24 @@ class CborTaggedTest { @Test fun wrongTags() { val wrongTag55ForPropertyC = "A46161CC1A0FFFFFFFD822616220D8376163D84E42CAFE6164D85ACC6B48656C6C6F20576F726C64" - assertFailsWith(CborDecodingException::class, message = "CBOR tags [55] do not match expected tags [56]") { - Cbor.decodeFromHexString( - DataWithTags.serializer(), - wrongTag55ForPropertyC - ) + listOf( + Cbor, + Cbor { writeDefiniteLengths = true }, + Cbor { writeDefiniteLengths = true;explicitNulls = false }, + Cbor { explicitNulls = false }).forEach { cbor -> + + assertFailsWith(CborDecodingException::class, message = "CBOR tags [55] do not match expected tags [56]") { + Cbor.decodeFromHexString( + DataWithTags.serializer(), + wrongTag55ForPropertyC + ) + } + } + listOf( + Cbor { verifyKeyTags = false }, + Cbor { verifyKeyTags = false;writeDefiniteLengths = true }, + Cbor { verifyKeyTags = false;explicitNulls = false }).forEach { cbor -> + assertEquals(reference, cbor.decodeFromHexString(wrongTag55ForPropertyC)) } - assertEquals(reference, Cbor { verifyKeyTags = false }.decodeFromHexString(wrongTag55ForPropertyC)) } } \ No newline at end of file diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt index d54a9c73e..793cc5a04 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt @@ -33,6 +33,25 @@ class CbrWriterTest { ) } + @Test + fun writeComplicatedClassDefLen() { + val test = TypesUmbrella( + "Hello, world!", + 42, + null, + listOf("a", "b"), + mapOf(1 to true, 2 to false), + Simple("lol"), + listOf(Simple("kek")), + HexConverter.parseHexBinary("cafe"), + HexConverter.parseHexBinary("cafe") + ) + assertEquals( + "a9637374726d48656c6c6f2c20776f726c64216169182a686e756c6c61626c65f6646c6973748261616162636d6170a201f502f465696e6e6572a16161636c6f6c6a696e6e6572734c69737481a16161636b656b6a62797465537472696e6742cafe6962797465417272617982383521", + Cbor { writeDefiniteLengths = true }.encodeToHexString(TypesUmbrella.serializer(), test) + ) + } + @Test fun writeManyNumbers() { val test = NumberTypesUmbrella( @@ -99,6 +118,33 @@ class CbrWriterTest { ) } + @Test + fun testOmitNullForNullableByteString() { + /* BF # map(*) + * FF # primitive(*) + */ + assertEquals( + expected = "bfff", + actual = Cbor { explicitNulls = false }.encodeToHexString( + serializer = NullableByteString.serializer(), + value = NullableByteString(byteString = null) + ) + ) + } + + @Test + fun testOmitNullDefLenForNullableByteString() { + /* A0 # map(0) + */ + assertEquals( + expected = "a0", + actual = Cbor { explicitNulls = false;writeDefiniteLengths = true }.encodeToHexString( + serializer = NullableByteString.serializer(), + value = NullableByteString(byteString = null) + ) + ) + } + @Test fun testWriteCustomByteString() { assertEquals( From ad97cbbaa2f0eed90dab08a10515db823f2b98a4 Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Tue, 25 Jul 2023 11:32:53 +0200 Subject: [PATCH 13/98] CBOR: Catch errors on finding annotations --- .../src/kotlinx/serialization/cbor/internal/Encoding.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index cc5efe2cc..13ee4e7c4 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -870,12 +870,12 @@ private fun Iterable.flatten(): ByteArray { @OptIn(ExperimentalSerializationApi::class) private fun SerialDescriptor.isByteString(index: Int): Boolean { - return getElementAnnotations(index).find { it is ByteString } != null + return kotlin.runCatching { getElementAnnotations(index).find { it is ByteString } != null }.getOrDefault(false) } @OptIn(ExperimentalSerializationApi::class) private fun SerialDescriptor.getValueTags(index: Int): ULongArray? { - return (getElementAnnotations(index).find { it is ValueTags } as ValueTags?)?.tags + return kotlin.runCatching { (getElementAnnotations(index).find { it is ValueTags } as ValueTags?)?.tags }.getOrNull() } private fun SerialDescriptor.isInlineByteString(): Boolean { // inline item classes should only have 1 item @@ -885,7 +885,7 @@ private fun SerialDescriptor.isInlineByteString(): Boolean { @OptIn(ExperimentalSerializationApi::class) private fun SerialDescriptor.getKeyTags(index: Int): ULongArray? { - return (getElementAnnotations(index).find { it is KeyTags } as KeyTags?)?.tags + return kotlin.runCatching { (getElementAnnotations(index).find { it is KeyTags } as KeyTags?)?.tags }.getOrNull() } @OptIn(ExperimentalSerializationApi::class) From ef1f77d01fb01e98b3fbf5102a7404d17d091baf Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Thu, 13 Jul 2023 14:23:16 +0200 Subject: [PATCH 14/98] CBOR: Implement labels to use as map keys --- .../kotlinx/serialization/cbor/SerialLabel.kt | 8 +++++ .../serialization/cbor/internal/Encoding.kt | 36 +++++++++++++++---- .../serialization/cbor/CborSerialLabelTest.kt | 34 ++++++++++++++++++ 3 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 formats/cbor/commonMain/src/kotlinx/serialization/cbor/SerialLabel.kt create mode 100644 formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/SerialLabel.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/SerialLabel.kt new file mode 100644 index 000000000..b712d8eed --- /dev/null +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/SerialLabel.kt @@ -0,0 +1,8 @@ +package kotlinx.serialization.cbor + +import kotlinx.serialization.* + +@SerialInfo +@Target(AnnotationTarget.PROPERTY) +@ExperimentalSerializationApi +public annotation class SerialLabel(val label: Long) \ No newline at end of file diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 13ee4e7c4..f090af85d 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -57,7 +57,8 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb var data: Any?, var parent: Node?, val index: Int, - val name: String? + val name: String?, + val label: Long? = null, ) { val children = mutableListOf() @@ -107,7 +108,6 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb private fun encodeElementPreamble() { - if (cbor.writeKeyTags) { parent?.descriptor?.getKeyTags(index)?.forEach { tag -> val encodedTag = encoder.composePositive(tag) @@ -115,8 +115,14 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } } } - if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) //TODO polymorphicKind? - name?.let { encoder.encodeString(it) } //indieces are put into the name field. we don't want to write those, as it would result in double writes + if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) { //TODO polymorphicKind? + //indieces are put into the name field. we don't want to write those, as it would result in double writes + if (label != null) { + encoder.encodeNumber(label) + } else if (name != null) { + encoder.encodeString(name) + } + } if (cbor.writeValueTags) { parent?.descriptor?.getValueTags(index)?.forEach { tag -> @@ -182,7 +188,8 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb null, currentNode, index, - descriptor.getElementName(index) + descriptor.getElementName(index), + descriptor.getSerialLabel(index), ) return true } @@ -439,7 +446,14 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb knownIndex } else { if (isDone()) return CompositeDecoder.DECODE_DONE - val (elemName, tags) = decoder.nextTaggedString() + val (elemName, tags) = runCatching { + decoder.nextTaggedString() + }.getOrElse { + val serialLabel = decoder.nextNumber(null) + val elemName = descriptor.getElementNameForSerialLabel(serialLabel) + ?: throw CborDecodingException("SerialLabel unknown: $serialLabel") + elemName to ulongArrayOf() // TODO Tags? + } readProperties++ descriptor.getElementIndexOrThrow(elemName).also { index -> if (cbor.verifyKeyTags) { @@ -903,6 +917,16 @@ private fun SerialDescriptor.getKeyTags(): ULongArray? { return (annotations.find { it is KeyTags } as KeyTags?)?.tags } +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.getSerialLabel(index: Int): Long? { + return kotlin.runCatching { getElementAnnotations(index).filterIsInstance().firstOrNull()?.label }.getOrNull() +} + +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.getElementNameForSerialLabel(label: Long): String? { + return elementNames.firstOrNull { getSerialLabel(getElementIndex(it)) == label } +} + private val normalizeBaseBits = SINGLE_PRECISION_NORMALIZE_BASE.toBits() diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt new file mode 100644 index 000000000..6ac8ec133 --- /dev/null +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt @@ -0,0 +1,34 @@ +package kotlinx.serialization.cbor + +import kotlinx.serialization.* +import kotlin.test.* + +@Serializable +data class CoseHeader( + @SerialLabel(4) + @SerialName("kid") + val kid: String? = null, +) + +class CborSerialLabelTest { + + private val reference = CoseHeader( + kid = "11" + ) + + /* + BF # map(*) + 04 # unsigned(4) + 62 # text(2) + 3131 # "11" + FF # primitive(*) + */ + private val referenceHexString = "bf04623131ff" + + + @Test + fun writeReadVerifyCoseHeader() { + assertEquals(referenceHexString, Cbor.encodeToHexString(CoseHeader.serializer(), reference)) + assertEquals(reference, Cbor.decodeFromHexString(CoseHeader.serializer(), referenceHexString)) + } +} \ No newline at end of file From 83824b68bbc3bd8e55bd450fa931ab9b216971ba Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Tue, 25 Jul 2023 11:48:35 +0200 Subject: [PATCH 15/98] WIP: Ignore failing tests --- .../src/kotlinx/serialization/cbor/CborPolymorphismTest.kt | 3 +++ .../src/kotlinx/serialization/cbor/CborReaderTest.kt | 1 + 2 files changed, 4 insertions(+) diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt index 156f47148..aad44f049 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt @@ -16,6 +16,7 @@ class CborPolymorphismTest { val cbor = Cbor { serializersModule = SimplePolymorphicModule } + @Ignore @Test fun testSealedWithOneSubclass() { assertSerializedToBinaryAndRestored( @@ -26,6 +27,7 @@ class CborPolymorphismTest { ) } + @Ignore @Test fun testSealedWithMultipleSubclasses() { val obj = SealedBox( @@ -37,6 +39,7 @@ class CborPolymorphismTest { assertSerializedToBinaryAndRestored(obj, SealedBox.serializer(), cbor) } + @Ignore @Test fun testOpenPolymorphism() { val obj = PolyBox( diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt index 82b3094d4..de1a80546 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt @@ -141,6 +141,7 @@ class CborReaderTest { ) } + @Ignore @Test fun testNullables() { Cbor { explicitNulls = false }.decodeFromHexString("a0") From eb73d7d85f9d06f1c7f4966a8d07c4d15138231d Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Thu, 13 Jul 2023 14:27:52 +0200 Subject: [PATCH 16/98] CBOR: Add option to write serial labels over names --- .../src/kotlinx/serialization/cbor/Cbor.kt | 13 +++- .../serialization/cbor/internal/Encoding.kt | 2 +- .../serialization/cbor/CborSerialLabelTest.kt | 62 +++++++++++++------ 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index 998d670ca..d9121199c 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -30,6 +30,8 @@ import kotlinx.serialization.modules.* * [KeyTags] annotation during the deserialization process. Useful for lenient parsing * @param verifyValueTags Specifies whether tags preceding values should be matched against the [ValueTags] * annotation during the deserialization process. Useful for lenient parsing. + * @param preferSerialLabelsOverNames Specifies whether to serialize element labels (i.e. Long from [SerialLabel]) + * instead of the element names (i.e. String from [SerialName]) for map keys */ @ExperimentalSerializationApi public sealed class Cbor( @@ -41,13 +43,14 @@ public sealed class Cbor( internal val verifyValueTags: Boolean, internal val explicitNulls: Boolean, internal val writeDefiniteLengths: Boolean, + internal val preferSerialLabelsOverNames: Boolean, override val serializersModule: SerializersModule ) : BinaryFormat { /** * The default instance of [Cbor] */ - public companion object Default : Cbor(false, false, true, true, true, true, true, false, EmptySerializersModule()) + public companion object Default : Cbor(false, false, true, true, true, true, true, false, true, EmptySerializersModule()) override fun encodeToByteArray(serializer: SerializationStrategy, value: T): ByteArray { val output = ByteArrayOutput() @@ -74,6 +77,7 @@ private class CborImpl( verifyValueTags: Boolean, encodeNullProperties: Boolean, writeDefiniteLengths: Boolean, + preferSerialLabelsOverNames: Boolean, serializersModule: SerializersModule ) : Cbor( @@ -85,6 +89,7 @@ private class CborImpl( verifyValueTags, encodeNullProperties, writeDefiniteLengths, + preferSerialLabelsOverNames, serializersModule ) @@ -105,6 +110,7 @@ public fun Cbor(from: Cbor = Cbor, builderAction: CborBuilder.() -> Unit): Cbor builder.verifyValueTags, builder.explicitNulls, builder.writeDefiniteLengths, + builder.preferSerialLabelsOverNames, builder.serializersModule ) } @@ -157,6 +163,11 @@ public class CborBuilder internal constructor(cbor: Cbor) { */ public var writeDefiniteLengths: Boolean = cbor.writeDefiniteLengths + /** + * Specifies whether to serialize element labels (i.e. Long from [SerialLabel]) instead of the element names (i.e. String from [SerialName]) for map keys + */ + public var preferSerialLabelsOverNames: Boolean = cbor.preferSerialLabelsOverNames + /** * Module with contextual and polymorphic serializers to be used in the resulting [Cbor] instance. */ diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index f090af85d..a46a4acff 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -117,7 +117,7 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb } if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) { //TODO polymorphicKind? //indieces are put into the name field. we don't want to write those, as it would result in double writes - if (label != null) { + if (cbor.preferSerialLabelsOverNames && label != null) { encoder.encodeNumber(label) } else if (name != null) { encoder.encodeString(name) diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt index 6ac8ec133..dd5a5596f 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt @@ -3,32 +3,54 @@ package kotlinx.serialization.cbor import kotlinx.serialization.* import kotlin.test.* -@Serializable -data class CoseHeader( - @SerialLabel(4) - @SerialName("kid") - val kid: String? = null, -) class CborSerialLabelTest { - private val reference = CoseHeader( - kid = "11" - ) + private val reference = ClassWithSerialLabel(alg = -7) + + /** + * A1 # map(1) + * 01 # unsigned(1) + * 26 # negative(6) + */ + private val referenceHexLabelString = "a10126" - /* - BF # map(*) - 04 # unsigned(4) - 62 # text(2) - 3131 # "11" - FF # primitive(*) + /** + * A1 # map(1) + * 63 # text(3) + * 616C67 # "alg" + * 26 # negative(6) */ - private val referenceHexString = "bf04623131ff" + private val referenceHexNameString = "a163616c6726" @Test - fun writeReadVerifyCoseHeader() { - assertEquals(referenceHexString, Cbor.encodeToHexString(CoseHeader.serializer(), reference)) - assertEquals(reference, Cbor.decodeFromHexString(CoseHeader.serializer(), referenceHexString)) + fun writeReadVerifySerialLabel() { + val cbor = Cbor { + preferSerialLabelsOverNames = true + writeDefiniteLengths = true + } + assertEquals(referenceHexLabelString, cbor.encodeToHexString(ClassWithSerialLabel.serializer(), reference)) + assertEquals(reference, cbor.decodeFromHexString(ClassWithSerialLabel.serializer(), referenceHexLabelString)) + } + + @Test + fun writeReadVerifySerialName() { + val cbor = Cbor { + preferSerialLabelsOverNames = false + writeDefiniteLengths = true + } + assertEquals(referenceHexNameString, cbor.encodeToHexString(ClassWithSerialLabel.serializer(), reference)) + assertEquals(reference, cbor.decodeFromHexString(ClassWithSerialLabel.serializer(), referenceHexNameString)) } -} \ No newline at end of file + + + @Serializable + data class ClassWithSerialLabel( + @SerialLabel(1) + @SerialName("alg") + val alg: Int + ) + +} + From 53c8c4217bdf1a6ef54358e0afb95acc919740f2 Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Thu, 13 Jul 2023 15:16:22 +0200 Subject: [PATCH 17/98] CBOR: Add annotation @CborArray to encode classes as arrays --- .../kotlinx/serialization/cbor/CborArray.kt | 8 ++ .../serialization/cbor/internal/Encoding.kt | 96 ++++++++++++------- .../serialization/cbor/CborArrayTest.kt | 89 +++++++++++++++++ 3 files changed, 159 insertions(+), 34 deletions(-) create mode 100644 formats/cbor/commonMain/src/kotlinx/serialization/cbor/CborArray.kt create mode 100644 formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/CborArray.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/CborArray.kt new file mode 100644 index 000000000..b925fc873 --- /dev/null +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/CborArray.kt @@ -0,0 +1,8 @@ +package kotlinx.serialization.cbor + +import kotlinx.serialization.* + +@SerialInfo +@Target(AnnotationTarget.CLASS) +@ExperimentalSerializationApi +public annotation class CborArray(@OptIn(ExperimentalUnsignedTypes::class) vararg val tag: ULong) \ No newline at end of file diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index a46a4acff..5dcabcbb0 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -77,22 +77,26 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb if (parent?.descriptor?.isByteString(index) != true) { if (children.isNotEmpty()) { //TODO: base this on structurekind not on emptyiness if (descriptor != null) - when (descriptor.kind) { - StructureKind.LIST, is PolymorphicKind -> { - if (cbor.writeDefiniteLengths) encoder.startArray(childNodes.size.toULong()) - else encoder.startArray() - } - - is StructureKind.MAP -> { - if (cbor.writeDefiniteLengths) encoder.startMap((childNodes.size / 2).toULong()) else encoder.startMap() - } - - else -> { - if (cbor.writeDefiniteLengths) encoder.startMap(childNodes.size.toULong()) else encoder.startMap() + if (descriptor.hasArrayTag()) { + descriptor.getArrayTags()?.forEach { encoder.encodeTag(it) } + if (cbor.writeDefiniteLengths) encoder.startArray(childNodes.size.toULong()) else encoder.startArray() + } else { + when (descriptor.kind) { + StructureKind.LIST, is PolymorphicKind -> { + if (cbor.writeDefiniteLengths) encoder.startArray(childNodes.size.toULong()) + else encoder.startArray() + } + + is StructureKind.MAP -> { + if (cbor.writeDefiniteLengths) encoder.startMap((childNodes.size / 2).toULong()) else encoder.startMap() + } + + else -> { + if (cbor.writeDefiniteLengths) encoder.startMap(childNodes.size.toULong()) else encoder.startMap() + } } } - } childNodes.forEach { it.encode() } @@ -108,28 +112,22 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb private fun encodeElementPreamble() { - if (cbor.writeKeyTags) { - parent?.descriptor?.getKeyTags(index)?.forEach { tag -> - val encodedTag = encoder.composePositive(tag) - encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() - encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } + if (parent?.descriptor?.hasArrayTag() != true) { + if (cbor.writeKeyTags) { + parent?.descriptor?.getKeyTags(index)?.forEach { encoder.encodeTag(it) } } - } - if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) { //TODO polymorphicKind? - //indieces are put into the name field. we don't want to write those, as it would result in double writes - if (cbor.preferSerialLabelsOverNames && label != null) { - encoder.encodeNumber(label) - } else if (name != null) { - encoder.encodeString(name) + if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) { //TODO polymorphicKind? + //indieces are put into the name field. we don't want to write those, as it would result in double writes + if (cbor.preferSerialLabelsOverNames && label != null) { + encoder.encodeNumber(label) + } else if (name != null) { + encoder.encodeString(name) + } } } if (cbor.writeValueTags) { - parent?.descriptor?.getValueTags(index)?.forEach { tag -> - val encodedTag = encoder.composePositive(tag) - encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() - encodedTag.forEach { encoder.writeByte(it.toUByte().toInt()) } - } + parent?.descriptor?.getValueTags(index)?.forEach { encoder.encodeTag(it) } } } @@ -299,6 +297,11 @@ internal class CborEncoder(private val output: ByteArrayOutput) { encodedNumber.forEach { writeByte(it.toUByte().toInt()) } } + fun encodeTag(tag: ULong) { + val encodedTag = composePositive(tag) + encodedTag[0] = encodedTag[0] or HEADER_TAG.toUByte().toByte() + encodedTag.forEach { writeByte(it.toUByte().toInt()) } + } fun end() = output.write(BREAK) @@ -379,8 +382,21 @@ private open class CborListReader(cbor: Cbor, decoder: CborDecoder) : CborReader override fun skipBeginToken() = setSize(decoder.startArray(tags)) - override fun decodeElementIndex(descriptor: SerialDescriptor) = - if (!finiteMode && decoder.isEnd() || (finiteMode && ind >= size)) CompositeDecoder.DECODE_DONE else ind++ + override fun decodeElementIndex(descriptor: SerialDescriptor): Int { + return if (!finiteMode && decoder.isEnd() || (finiteMode && ind >= size)) CompositeDecoder.DECODE_DONE else + ind++.also { + decodeByteArrayAsByteString = descriptor.isByteString(it) + } + } +} + +private class CborDefiniteListReader( + cbor: Cbor, + decoder: CborDecoder, + private val expectedSize: ULong, + private val tag: ULongArray? +) : CborListReader(cbor, decoder) { + } internal open class CborReader(private val cbor: Cbor, protected val decoder: CborDecoder) : AbstractDecoder() { @@ -391,7 +407,7 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb private set private var readProperties: Int = 0 - private var decodeByteArrayAsByteString = false + protected var decodeByteArrayAsByteString = false protected var tags: ULongArray? = null protected fun setSize(size: Int) { @@ -408,7 +424,9 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb @OptIn(ExperimentalSerializationApi::class) override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder { - val re = when (descriptor.kind) { + val re = if (descriptor.hasArrayTag()) { + CborDefiniteListReader(cbor, decoder, descriptor.elementNames.count().toULong(), descriptor.getArrayTags()) + } else when (descriptor.kind) { StructureKind.LIST, is PolymorphicKind -> CborListReader(cbor, decoder) StructureKind.MAP -> CborMapReader(cbor, decoder) else -> CborReader(cbor, decoder) @@ -927,6 +945,16 @@ private fun SerialDescriptor.getElementNameForSerialLabel(label: Long): String? return elementNames.firstOrNull { getSerialLabel(getElementIndex(it)) == label } } +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.getArrayTags(): ULongArray? { + return annotations.filterIsInstance().firstOrNull()?.tag +} + +@OptIn(ExperimentalSerializationApi::class) +private fun SerialDescriptor.hasArrayTag(): Boolean { + return annotations.filterIsInstance().isNotEmpty() +} + private val normalizeBaseBits = SINGLE_PRECISION_NORMALIZE_BASE.toBits() diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt new file mode 100644 index 000000000..0ece5d2b1 --- /dev/null +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt @@ -0,0 +1,89 @@ +package kotlinx.serialization.cbor + +import kotlinx.serialization.* +import kotlin.test.* + + +class CborArrayTest { + + private val reference1 = ClassAs1Array(alg = -7) + private val reference2 = ClassAs2Array(alg = -7, kid = "foo") + private val reference3 = ClassWithArray(array = ClassAs2Array(alg = -7, kid = "bar")) + + /** + * 81 # array(1) + * 26 # negative(6) + */ + private val reference1HexString = "8126" + + /** + * C8 # tag(8) + * 82 # array(2) + * 26 # negative(6) + * 63 # text(3) + * 666F6F # "foo" + */ + private val reference2HexString = "c8822663666f6f" + + /** + * A1 # map(1) + * 65 # text(5) + * 6172726179 # "array" + * C8 # tag(8) + * 82 # array(2) + * 26 # negative(6) + * 63 # text(3) + * 626172 # "bar" + */ + private val reference3HexString = "a1656172726179c8822663626172" + + @Test + fun writeReadVerifyArraySize1() { + val cbor = Cbor { + writeDefiniteLengths = true + } + assertEquals(reference1HexString, cbor.encodeToHexString(ClassAs1Array.serializer(), reference1)) + assertEquals(reference1, cbor.decodeFromHexString(ClassAs1Array.serializer(), reference1HexString)) + } + + @Test + fun writeReadVerifyArraySize2() { + val cbor = Cbor { + writeDefiniteLengths = true + } + assertEquals(reference2HexString, cbor.encodeToHexString(ClassAs2Array.serializer(), reference2)) + assertEquals(reference2, cbor.decodeFromHexString(ClassAs2Array.serializer(), reference2HexString)) + } + + @Test + fun writeReadVerifyClassWithArray() { + val cbor = Cbor { + writeDefiniteLengths = true + } + assertEquals(reference3HexString, cbor.encodeToHexString(ClassWithArray.serializer(), reference3)) + assertEquals(reference3, cbor.decodeFromHexString(ClassWithArray.serializer(), reference3HexString)) + } + + @CborArray + @Serializable + data class ClassAs1Array( + @SerialName("alg") + val alg: Int, + ) + + @CborArray(8U) + @Serializable + data class ClassAs2Array( + @SerialName("alg") + val alg: Int, + @SerialName("kid") + val kid: String, + ) + + @Serializable + data class ClassWithArray( + @SerialName("array") + val array: ClassAs2Array, + ) +} + From 6a9bc52a80f8eb1e1f186debcfc350c3a096d9cc Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Thu, 13 Jul 2023 16:41:18 +0200 Subject: [PATCH 18/98] CBOR: Add convenience class for wrapping byte strings during serialization --- .../serialization/cbor/ByteStringWrapper.kt | 67 +++++++++++++++++++ .../cbor/CborByteStringWrapperTest.kt | 62 +++++++++++++++++ .../serialization/cbor/CborSerialLabelTest.kt | 10 +-- 3 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 formats/cbor/commonMain/src/kotlinx/serialization/cbor/ByteStringWrapper.kt create mode 100644 formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborByteStringWrapperTest.kt diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/ByteStringWrapper.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/ByteStringWrapper.kt new file mode 100644 index 000000000..dda22eea4 --- /dev/null +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/ByteStringWrapper.kt @@ -0,0 +1,67 @@ +package kotlinx.serialization.cbor + +/** + * Use this class if you'll need to serialize a complex type as a byte string before encoding it, + * i.e. as it is the case with the protected header in COSE structures. + * + * Clients also need to write a custom serializer, i.e. in the form of + * + * ``` + * @Serializable + * data class CoseHeader( + * @SerialLabel(1) + * @SerialName("alg") + * val alg: Int? = null + * ) + * + * @Serializable + * data class CoseSigned( + * @Serializable(with = ByteStringWrapperSerializer::class) + * @ByteString + * @SerialLabel(1) + * @SerialName("protectedHeader") + * val protectedHeader: ByteStringWrapper, + * ) + * + * object ByteStringWrapperSerializer : KSerializer> { + * override val descriptor: SerialDescriptor = + * PrimitiveSerialDescriptor("ByteStringWrapperSerializer", PrimitiveKind.STRING) + * override fun serialize(encoder: Encoder, value: ByteStringWrapper) { + * val bytes = Cbor.encodeToByteArray(value.value) + * encoder.encodeSerializableValue(ByteArraySerializer(), bytes) + * } + * override fun deserialize(decoder: Decoder): ByteStringWrapper { + * val bytes = decoder.decodeSerializableValue(ByteArraySerializer()) + * return ByteStringWrapper(Cbor.decodeFromByteArray(bytes), bytes) + * } + * } + * ``` + * + * then serializing a `CoseSigned` object would result in `a10143a10126`, in diagnostic notation: + * + * ``` + * A1 # map(1) + * 01 # unsigned(1) + * 43 # bytes(3) + * A10126 # "\xA1\u0001&" + * ``` + * + * so the `protectedHeader` got serialized first and then encoded as a `@ByteString` + */ +public data class ByteStringWrapper( + val value: T, + val serialized: ByteArray = byteArrayOf() +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + + other as ByteStringWrapper<*> + + return value == other.value + } + + override fun hashCode(): Int { + return value?.hashCode() ?: 0 + } +} \ No newline at end of file diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborByteStringWrapperTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborByteStringWrapperTest.kt new file mode 100644 index 000000000..e6b3b4c04 --- /dev/null +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborByteStringWrapperTest.kt @@ -0,0 +1,62 @@ +package kotlinx.serialization.cbor + +import kotlinx.serialization.* +import kotlinx.serialization.builtins.* +import kotlinx.serialization.descriptors.* +import kotlinx.serialization.encoding.* +import kotlin.test.* + +class CborByteStringWrapperTest { + + private val reference = CoseSigned(protectedHeader = ByteStringWrapper(CoseHeader(alg = -7))) + + /** + * BF # map(*) + * 01 # unsigned(1) + * 44 # bytes(4) + * BF0126FF # "\xBF\u0001&\xFF" + * FF # primitive(*) + */ + private val referenceHex = "bf0144bf0126ffff" + + @Test + fun writeReadVerifyCoseSigned() { + assertEquals(referenceHex, Cbor.encodeToHexString(CoseSigned.serializer(), reference)) + assertEquals(reference, Cbor.decodeFromHexString(referenceHex)) + } + + + @Serializable + data class CoseHeader( + @SerialLabel(1) + @SerialName("alg") + val alg: Int? = null + ) + + @Serializable + data class CoseSigned( + @Serializable(with = ByteStringWrapperSerializer::class) + @ByteString + @SerialLabel(1) + @SerialName("protectedHeader") + val protectedHeader: ByteStringWrapper, + ) + + object ByteStringWrapperSerializer : KSerializer> { + + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("ByteStringWrapperSerializer", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: ByteStringWrapper) { + val bytes = Cbor.encodeToByteArray(value.value) + encoder.encodeSerializableValue(ByteArraySerializer(), bytes) + } + + override fun deserialize(decoder: Decoder): ByteStringWrapper { + val bytes = decoder.decodeSerializableValue(ByteArraySerializer()) + return ByteStringWrapper(Cbor.decodeFromByteArray(bytes), bytes) + } + + } + +} diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt index dd5a5596f..c8eb1a57f 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt @@ -9,19 +9,21 @@ class CborSerialLabelTest { private val reference = ClassWithSerialLabel(alg = -7) /** - * A1 # map(1) + * BF # map(*) * 01 # unsigned(1) * 26 # negative(6) + * FF # primitive(*) */ - private val referenceHexLabelString = "a10126" + private val referenceHexLabelString = "bf0126ff" /** - * A1 # map(1) + * BF # map(*) * 63 # text(3) * 616C67 # "alg" * 26 # negative(6) + * FF # primitive(*) */ - private val referenceHexNameString = "a163616c6726" + private val referenceHexNameString = "bf63616c6726ff" @Test From 03b77715fbe83976b3aecaeda5fb3b9a583dea41 Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Mon, 17 Jul 2023 14:30:12 +0200 Subject: [PATCH 19/98] CBOR: Test combination of serial label with tags --- .../serialization/cbor/internal/Encoding.kt | 59 ++++++++++++--- .../serialization/cbor/CborSerialLabelTest.kt | 75 ++++++++++++++++++- 2 files changed, 121 insertions(+), 13 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 5dcabcbb0..5b2114bb2 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -440,15 +440,14 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb } override fun decodeElementIndex(descriptor: SerialDescriptor): Int { - val index = if (cbor.ignoreUnknownKeys) { val knownIndex: Int while (true) { if (isDone()) return CompositeDecoder.DECODE_DONE - val (elemName, tags) = decoder.nextTaggedString() + val (elemName, tags) = decodeElementNameWithTagsLenient(descriptor) readProperties++ - val index = descriptor.getElementIndex(elemName) + val index = elemName?.let { descriptor.getElementIndex(it) } ?: CompositeDecoder.UNKNOWN_NAME if (index == CompositeDecoder.UNKNOWN_NAME) { decoder.skipElement(tags) } else { @@ -464,14 +463,7 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb knownIndex } else { if (isDone()) return CompositeDecoder.DECODE_DONE - val (elemName, tags) = runCatching { - decoder.nextTaggedString() - }.getOrElse { - val serialLabel = decoder.nextNumber(null) - val elemName = descriptor.getElementNameForSerialLabel(serialLabel) - ?: throw CborDecodingException("SerialLabel unknown: $serialLabel") - elemName to ulongArrayOf() // TODO Tags? - } + val (elemName, tags) = decodeElementNameWithTags(descriptor) readProperties++ descriptor.getElementIndexOrThrow(elemName).also { index -> if (cbor.verifyKeyTags) { @@ -487,6 +479,26 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb return index } + private fun decodeElementNameWithTags(descriptor: SerialDescriptor): Pair { + var (elemName, serialLabel, tags) = decoder.nextTaggedStringOrNumber() + if (elemName == null && serialLabel != null) { + elemName = descriptor.getElementNameForSerialLabel(serialLabel) + ?: throw CborDecodingException("SerialLabel unknown: $serialLabel") + } + if (elemName == null) { + throw CborDecodingException("Expected (tagged) string or number, got nothing") + } + return elemName to tags + } + + private fun decodeElementNameWithTagsLenient(descriptor: SerialDescriptor): Pair { + var (elemName, serialLabel, tags) = decoder.nextTaggedStringOrNumber() + if (elemName == null && serialLabel != null) { + elemName = descriptor.getElementNameForSerialLabel(serialLabel) + } + return elemName to tags + } + @OptIn(ExperimentalSerializationApi::class) override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { @@ -659,6 +671,23 @@ internal class CborDecoder(private val input: ByteArrayInput) { } } + /** + * Used for reading the tags and either string (element name) or number (serial label) + */ + fun nextTaggedStringOrNumber(): Triple { + val collectedTags = processTags(null) + if ((curByte and 0b111_00000) == HEADER_STRING.toInt()) { + val arr = readBytes() + val ans = arr.decodeToString() + readByte() + return Triple(ans, null, collectedTags) + } else { + val res = readNumber() + readByte() + return Triple(null, res, collectedTags) + } + } + fun nextNumber(tag: ULong): Long = nextNumber(ulongArrayOf(tag)) fun nextNumber(tags: ULongArray? = null): Long { processTags(tags) @@ -666,6 +695,14 @@ internal class CborDecoder(private val input: ByteArrayInput) { readByte() return res } + fun nextTaggedNumber() = nextTaggedNumber(null) + + private fun nextTaggedNumber(tags: ULongArray?): Pair { + val collectedTags = processTags(tags) + val res = readNumber() + readByte() + return res to collectedTags + } private fun readNumber(): Long { val value = curByte and 0b000_11111 diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt index c8eb1a57f..c3ad10c4c 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborSerialLabelTest.kt @@ -1,6 +1,8 @@ package kotlinx.serialization.cbor import kotlinx.serialization.* +import kotlinx.serialization.cbor.internal.* +import kotlinx.serialization.cbor.internal.CborDecodingException import kotlin.test.* @@ -8,6 +10,7 @@ class CborSerialLabelTest { private val reference = ClassWithSerialLabel(alg = -7) + /** * BF # map(*) * 01 # unsigned(1) @@ -30,7 +33,6 @@ class CborSerialLabelTest { fun writeReadVerifySerialLabel() { val cbor = Cbor { preferSerialLabelsOverNames = true - writeDefiniteLengths = true } assertEquals(referenceHexLabelString, cbor.encodeToHexString(ClassWithSerialLabel.serializer(), reference)) assertEquals(reference, cbor.decodeFromHexString(ClassWithSerialLabel.serializer(), referenceHexLabelString)) @@ -40,12 +42,73 @@ class CborSerialLabelTest { fun writeReadVerifySerialName() { val cbor = Cbor { preferSerialLabelsOverNames = false - writeDefiniteLengths = true } assertEquals(referenceHexNameString, cbor.encodeToHexString(ClassWithSerialLabel.serializer(), reference)) assertEquals(reference, cbor.decodeFromHexString(ClassWithSerialLabel.serializer(), referenceHexNameString)) } + @Test + fun writeReadVerifySerialLabelWithTags() { + val referenceWithTag = ClassWithSerialLabelAndTag(alg = -7) + /** + * A1 # map(1) + * C5 # tag(5) + * 01 # unsigned(1) + * 26 # negative(6) + */ + val referenceHexLabelWithTagString = "a1c50126" + val cbor = Cbor { + preferSerialLabelsOverNames = true + writeKeyTags = true + verifyKeyTags = true + writeDefiniteLengths = true + } + assertEquals(referenceHexLabelWithTagString, cbor.encodeToHexString(ClassWithSerialLabelAndTag.serializer(), referenceWithTag)) + assertEquals(referenceWithTag, cbor.decodeFromHexString(ClassWithSerialLabelAndTag.serializer(), referenceHexLabelWithTagString)) + } + + @Test + fun writeReadVerifySerialLabelWithTagsThrowing() { + /** + * A1 # map(1) + * C6 # tag(6) // wrong tag: declared is 5U, meaning C5 in hex + * 01 # unsigned(1) + * 26 # negative(6) + */ + val referenceHexLabelWithTagString = "a1c60126" + val cbor = Cbor { + preferSerialLabelsOverNames = true + writeKeyTags = true + verifyKeyTags = true + writeDefiniteLengths = true + } + assertFailsWith(CborDecodingException::class) { + cbor.decodeFromHexString(ClassWithSerialLabelAndTag.serializer(), referenceHexLabelWithTagString) + } + } + + @Test + fun writeReadVerifySerialLabelWithTagsAndUnknownKeys() { + val referenceWithTag = ClassWithSerialLabelAndTag(alg = -7) + /** + * A2 # map(2) + * C5 # tag(5) + * 01 # unsigned(1) + * 26 # negative(6) + * 02 # unsigned(2) + * 63 # text(3) + * 62617A # "baz" + */ + val referenceHexLabelWithTagString = "a2c50126026362617a" + val cbor = Cbor { + preferSerialLabelsOverNames = true + writeKeyTags = true + verifyKeyTags = true + ignoreUnknownKeys = true + writeDefiniteLengths = true + } + assertEquals(referenceWithTag, cbor.decodeFromHexString(ClassWithSerialLabelAndTag.serializer(), referenceHexLabelWithTagString)) + } @Serializable data class ClassWithSerialLabel( @@ -54,5 +117,13 @@ class CborSerialLabelTest { val alg: Int ) + @Serializable + data class ClassWithSerialLabelAndTag( + @SerialLabel(1) + @SerialName("alg") + @KeyTags(5U) + val alg: Int + ) + } From 86c7f82440eeef92f4114f819e4be497872c5a3a Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Wed, 19 Jul 2023 10:59:38 +0200 Subject: [PATCH 20/98] CBOR: Add option to always use compact byte string encoding --- .../src/kotlinx/serialization/cbor/Cbor.kt | 13 ++++- .../serialization/cbor/internal/Encoding.kt | 7 +-- .../kotlinx/serialization/cbor/CborIsoTest.kt | 50 +++++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborIsoTest.kt diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index d9121199c..b7f7a708c 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -30,6 +30,8 @@ import kotlinx.serialization.modules.* * [KeyTags] annotation during the deserialization process. Useful for lenient parsing * @param verifyValueTags Specifies whether tags preceding values should be matched against the [ValueTags] * annotation during the deserialization process. Useful for lenient parsing. + * @param alwaysUseByteString Specifies whether to always use the compact [ByteString] encoding when serializing + * or deserializing byte arrays. * @param preferSerialLabelsOverNames Specifies whether to serialize element labels (i.e. Long from [SerialLabel]) * instead of the element names (i.e. String from [SerialName]) for map keys */ @@ -44,13 +46,14 @@ public sealed class Cbor( internal val explicitNulls: Boolean, internal val writeDefiniteLengths: Boolean, internal val preferSerialLabelsOverNames: Boolean, + internal val alwaysUseByteString: Boolean, override val serializersModule: SerializersModule ) : BinaryFormat { /** * The default instance of [Cbor] */ - public companion object Default : Cbor(false, false, true, true, true, true, true, false, true, EmptySerializersModule()) + public companion object Default : Cbor(false, false, true, true, true, true, true, false, true, false, EmptySerializersModule()) override fun encodeToByteArray(serializer: SerializationStrategy, value: T): ByteArray { val output = ByteArrayOutput() @@ -78,6 +81,7 @@ private class CborImpl( encodeNullProperties: Boolean, writeDefiniteLengths: Boolean, preferSerialLabelsOverNames: Boolean, + alwaysUseByteString: Boolean, serializersModule: SerializersModule ) : Cbor( @@ -90,6 +94,7 @@ private class CborImpl( encodeNullProperties, writeDefiniteLengths, preferSerialLabelsOverNames, + alwaysUseByteString, serializersModule ) @@ -111,6 +116,7 @@ public fun Cbor(from: Cbor = Cbor, builderAction: CborBuilder.() -> Unit): Cbor builder.explicitNulls, builder.writeDefiniteLengths, builder.preferSerialLabelsOverNames, + builder.alwaysUseByteString, builder.serializersModule ) } @@ -168,6 +174,11 @@ public class CborBuilder internal constructor(cbor: Cbor) { */ public var preferSerialLabelsOverNames: Boolean = cbor.preferSerialLabelsOverNames + /** + * Specifies whether to always use the compact [ByteString] encoding when serializing or deserializing byte arrays. + */ + public var alwaysUseByteString: Boolean = cbor.alwaysUseByteString + /** * Module with contextual and polymorphic serializers to be used in the resulting [Cbor] instance. */ diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 5b2114bb2..227dc0003 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -145,7 +145,8 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb @OptIn(ExperimentalSerializationApi::class) override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { - if (encodeByteArrayAsByteString && serializer.descriptor == ByteArraySerializer().descriptor) { + if ((encodeByteArrayAsByteString || cbor.alwaysUseByteString) + && serializer.descriptor == ByteArraySerializer().descriptor) { currentNode.children.last().data = ByteArrayOutput().also { CborEncoder(it).encodeByteString(value as ByteArray) }.toByteArray() } else { @@ -501,8 +502,8 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb @OptIn(ExperimentalSerializationApi::class) override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { - - return if (decodeByteArrayAsByteString && deserializer.descriptor == ByteArraySerializer().descriptor) { + return if ((decodeByteArrayAsByteString || cbor.alwaysUseByteString) + && deserializer.descriptor == ByteArraySerializer().descriptor) { @Suppress("UNCHECKED_CAST") decoder.nextByteString(tags) as T } else { diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborIsoTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborIsoTest.kt new file mode 100644 index 000000000..dca6ac6e9 --- /dev/null +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborIsoTest.kt @@ -0,0 +1,50 @@ +package kotlinx.serialization.cbor + +import kotlinx.serialization.* +import kotlin.test.* + +class CborIsoTest { + + private val reference = DataClass( + bytes = "foo".encodeToByteArray() + ) + + /** + * A1 # map(1) + * 65 # text(5) + * 6279746573 # "bytes" + * 43 # bytes(3) + * 666F6F # "foo" + * + */ + private val referenceHexString = "a165627974657343666f6f" + + @Test + fun writeReadVerifyCoseSigned() { + val cbor = Cbor { + alwaysUseByteString = true + writeDefiniteLengths = true + } + assertEquals(reference, cbor.decodeFromHexString(DataClass.serializer(), referenceHexString)) + assertEquals(referenceHexString, cbor.encodeToHexString(DataClass.serializer(), reference)) + } + + @Serializable + data class DataClass( + @SerialName("bytes") + val bytes: ByteArray, + ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + + other as DataClass + + return bytes.contentEquals(other.bytes) + } + + override fun hashCode(): Int { + return bytes.contentHashCode() + } + } +} \ No newline at end of file From 82fb4be62966f37814a73d39d9e483ed0660a186 Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Tue, 25 Jul 2023 13:58:07 +0200 Subject: [PATCH 21/98] CBOR: Fix potential issue with custom serializers --- .../src/kotlinx/serialization/cbor/internal/Encoding.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 227dc0003..51a984a13 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -147,7 +147,7 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { if ((encodeByteArrayAsByteString || cbor.alwaysUseByteString) && serializer.descriptor == ByteArraySerializer().descriptor) { - currentNode.children.last().data = + (currentNode.children.lastOrNull() ?: currentNode).data = ByteArrayOutput().also { CborEncoder(it).encodeByteString(value as ByteArray) }.toByteArray() } else { super.encodeSerializableValue(serializer, value) From af0b7a37bbc1b4f025b023ab11b2d46924c441c6 Mon Sep 17 00:00:00 2001 From: Christian Kollmann Date: Wed, 26 Jul 2023 16:31:10 +0200 Subject: [PATCH 22/98] CBOR: Encode null complex object as empty map --- .../serialization/cbor/internal/Encoding.kt | 17 ++- .../serialization/cbor/CborArrayTest.kt | 135 +++++++++++++----- 2 files changed, 112 insertions(+), 40 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 51a984a13..97b72e4a2 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -16,6 +16,7 @@ import kotlin.experimental.* private const val FALSE = 0xf4 private const val TRUE = 0xf5 private const val NULL = 0xf6 +private const val EMPTY_MAP = 0xa0 private const val NEXT_HALF = 0xf9 private const val NEXT_FLOAT = 0xfa @@ -261,7 +262,11 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb override fun encodeNull() { (currentNode.children.lastOrNull() ?: currentNode).apply { - data = ByteArrayOutput().also { CborEncoder(it).encodeNull() }.toByteArray() + if (this.descriptor?.kind == StructureKind.CLASS) { + data = ByteArrayOutput().also { CborEncoder(it).encodeEmptyMap() }.toByteArray() + } else { + data = ByteArrayOutput().also { CborEncoder(it).encodeNull() }.toByteArray() + } } } @@ -308,6 +313,8 @@ internal class CborEncoder(private val output: ByteArrayOutput) { fun encodeNull() = output.write(NULL) + fun encodeEmptyMap() = output.write(EMPTY_MAP) + internal fun writeByte(byteValue: Int) = output.write(byteValue) fun encodeBoolean(value: Boolean) = output.write(if (value) TRUE else FALSE) @@ -553,12 +560,16 @@ internal class CborDecoder(private val input: ByteArrayInput) { readByte() } - fun isNull() = curByte == NULL + fun isNull() = (curByte == NULL || curByte == EMPTY_MAP) fun nextNull(tag: ULong) = nextNull(ulongArrayOf(tag)) fun nextNull(tags: ULongArray? = null): Nothing? { processTags(tags) - skipByte(NULL) + if (curByte == NULL) { + skipByte(NULL) + } else if (curByte == EMPTY_MAP) { + skipByte(EMPTY_MAP) + } return null } diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt index 0ece5d2b1..8b03174b2 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt @@ -6,62 +6,83 @@ import kotlin.test.* class CborArrayTest { - private val reference1 = ClassAs1Array(alg = -7) - private val reference2 = ClassAs2Array(alg = -7, kid = "foo") - private val reference3 = ClassWithArray(array = ClassAs2Array(alg = -7, kid = "bar")) - - /** - * 81 # array(1) - * 26 # negative(6) - */ - private val reference1HexString = "8126" - - /** - * C8 # tag(8) - * 82 # array(2) - * 26 # negative(6) - * 63 # text(3) - * 666F6F # "foo" - */ - private val reference2HexString = "c8822663666f6f" - - /** - * A1 # map(1) - * 65 # text(5) - * 6172726179 # "array" - * C8 # tag(8) - * 82 # array(2) - * 26 # negative(6) - * 63 # text(3) - * 626172 # "bar" - */ - private val reference3HexString = "a1656172726179c8822663626172" - @Test fun writeReadVerifyArraySize1() { + /** + * 81 # array(1) + * 26 # negative(6) + */ + val referenceHexString = "8126" + val reference = ClassAs1Array(alg = -7) + val cbor = Cbor { writeDefiniteLengths = true } - assertEquals(reference1HexString, cbor.encodeToHexString(ClassAs1Array.serializer(), reference1)) - assertEquals(reference1, cbor.decodeFromHexString(ClassAs1Array.serializer(), reference1HexString)) + assertEquals(referenceHexString, cbor.encodeToHexString(ClassAs1Array.serializer(), reference)) + assertEquals(reference, cbor.decodeFromHexString(ClassAs1Array.serializer(), referenceHexString)) } @Test fun writeReadVerifyArraySize2() { + /** + * C8 # tag(8) + * 82 # array(2) + * 26 # negative(6) + * 63 # text(3) + * 666F6F # "foo" + */ + val referenceHexString = "c8822663666f6f" + val reference = ClassAs2Array(alg = -7, kid = "foo") + + val cbor = Cbor { + writeDefiniteLengths = true + } + assertEquals(referenceHexString, cbor.encodeToHexString(ClassAs2Array.serializer(), reference)) + assertEquals(reference, cbor.decodeFromHexString(ClassAs2Array.serializer(), referenceHexString)) + } + + @Test + fun writeReadVerifyArraySize4Nullable() { + /** + * 84 # array(4) + * 26 # negative(6) + * 63 # text(3) + * 626172 # "bar" + * F6 # primitive(22) + * A0 # map(0) + */ + val referenceHexString = "842663626172f6a0" + val reference = ClassAs4ArrayNullable(alg = -7, kid = "bar", iv = null, array = null) + val cbor = Cbor { writeDefiniteLengths = true + explicitNulls = true } - assertEquals(reference2HexString, cbor.encodeToHexString(ClassAs2Array.serializer(), reference2)) - assertEquals(reference2, cbor.decodeFromHexString(ClassAs2Array.serializer(), reference2HexString)) + + assertEquals(referenceHexString, cbor.encodeToHexString(ClassAs4ArrayNullable.serializer(), reference)) + assertEquals(reference, cbor.decodeFromHexString(ClassAs4ArrayNullable.serializer(), referenceHexString)) } @Test fun writeReadVerifyClassWithArray() { + /** + * A1 # map(1) + * 65 # text(5) + * 6172726179 # "array" + * C8 # tag(8) + * 82 # array(2) + * 26 # negative(6) + * 63 # text(3) + * 626172 # "bar" + */ + val referenceHexString = "a1656172726179c8822663626172" + val reference = ClassWithArray(array = ClassAs2Array(alg = -7, kid = "bar")) + val cbor = Cbor { writeDefiniteLengths = true } - assertEquals(reference3HexString, cbor.encodeToHexString(ClassWithArray.serializer(), reference3)) - assertEquals(reference3, cbor.decodeFromHexString(ClassWithArray.serializer(), reference3HexString)) + assertEquals(referenceHexString, cbor.encodeToHexString(ClassWithArray.serializer(), reference)) + assertEquals(reference, cbor.decodeFromHexString(ClassWithArray.serializer(), referenceHexString)) } @CborArray @@ -80,6 +101,46 @@ class CborArrayTest { val kid: String, ) + @CborArray + @Serializable + data class ClassAs4ArrayNullable( + @SerialName("alg") + val alg: Int, + @SerialName("kid") + val kid: String, + @SerialName("iv") + @ByteString + val iv: ByteArray?, + @SerialName("array") + val array: ClassWithArray? + ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + + other as ClassAs4ArrayNullable + + if (alg != other.alg) return false + if (kid != other.kid) return false + if (iv != null) { + if (other.iv == null) return false + if (!iv.contentEquals(other.iv)) return false + } else if (other.iv != null) return false + if (array != other.array) return false + + return true + } + + override fun hashCode(): Int { + var result = alg + result = 31 * result + kid.hashCode() + result = 31 * result + (iv?.contentHashCode() ?: 0) + result = 31 * result + (array?.hashCode() ?: 0) + return result + } + } + + @Serializable data class ClassWithArray( @SerialName("array") From 65722454472eedc125f9f327d1c9b635edb44c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Thu, 17 Aug 2023 14:24:01 +0200 Subject: [PATCH 23/98] CBOR: remove recursion and null pruning --- .../serialization/cbor/internal/Encoding.kt | 227 ++++++++++-------- .../serialization/cbor/CborWriterTest.kt | 8 +- .../serialization/cbor/SampleClasses.kt | 23 ++ 3 files changed, 157 insertions(+), 101 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 97b72e4a2..3db17a04e 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -12,6 +12,7 @@ import kotlinx.serialization.descriptors.* import kotlinx.serialization.encoding.* import kotlinx.serialization.modules.* import kotlin.experimental.* +import kotlin.jvm.* private const val FALSE = 0xf4 private const val TRUE = 0xf5 @@ -47,78 +48,46 @@ private const val SINGLE_PRECISION_MAX_EXPONENT = 0xFF private const val SINGLE_PRECISION_NORMALIZE_BASE = 0.5f +@JvmInline +internal value class Stack(private val list: MutableList = mutableListOf()) { + fun push(value: T) { + list += value + } + + fun pop() = list.removeLast() + + fun peek() = list.last() +} + // Writes class as map [fieldName, fieldValue] -internal open class CborWriter(private val cbor: Cbor, protected val encoder: CborEncoder) : AbstractEncoder() { +internal open class CborWriter( + private val cbor: Cbor, + protected val encoder: CborEncoder, +) : AbstractEncoder() { var encodeByteArrayAsByteString = false - inner class Node( - val descriptor: SerialDescriptor?, + private val structureStack: Stack = Stack() + + inner class Token( + var descriptor: SerialDescriptor?, + val parent: Token?, var data: Any?, - var parent: Node?, - val index: Int, + var next: Token?, + val index: Int?, val name: String?, val label: Long? = null, + var numChildren: Int? = null, ) { - val children = mutableListOf() - - override fun toString(): String { - return "(${descriptor?.serialName}:${descriptor?.kind}, $data, ${children.joinToString { it.toString() }})" - } - - fun encode() { - val childNodes = - if (!cbor.explicitNulls && (descriptor?.kind is StructureKind.CLASS || descriptor?.kind is StructureKind.OBJECT)) { - children.filterNot { (it.data as ByteArray?)?.contentEquals(byteArrayOf(NULL.toByte())) == true } - } else children - - encodeElementPreamble() - - if (parent?.descriptor?.isByteString(index) != true) { - if (children.isNotEmpty()) { //TODO: base this on structurekind not on emptyiness - if (descriptor != null) - if (descriptor.hasArrayTag()) { - descriptor.getArrayTags()?.forEach { encoder.encodeTag(it) } - if (cbor.writeDefiniteLengths) encoder.startArray(childNodes.size.toULong()) else encoder.startArray() - } else { - when (descriptor.kind) { - StructureKind.LIST, is PolymorphicKind -> { - if (cbor.writeDefiniteLengths) encoder.startArray(childNodes.size.toULong()) - else encoder.startArray() - } - - is StructureKind.MAP -> { - if (cbor.writeDefiniteLengths) encoder.startMap((childNodes.size / 2).toULong()) else encoder.startMap() - } - - else -> { - if (cbor.writeDefiniteLengths) encoder.startMap(childNodes.size.toULong()) else encoder.startMap() - } - } - } - - } - - childNodes.forEach { it.encode() } - if (children.isNotEmpty() && descriptor != null && !cbor.writeDefiniteLengths) encoder.end() - - } - //byteStrings are encoded into the data already, as are primitives - data?.let { - it as ByteArray; it.forEach { encoder.writeByte(it.toInt()) } - } - - } - private fun encodeElementPreamble() { if (parent?.descriptor?.hasArrayTag() != true) { if (cbor.writeKeyTags) { - parent?.descriptor?.getKeyTags(index)?.forEach { encoder.encodeTag(it) } + index?.let { parent?.descriptor?.getKeyTags(it)?.forEach { encoder.encodeTag(it) } } } if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) { //TODO polymorphicKind? - //indieces are put into the name field. we don't want to write those, as it would result in double writes + //indices are put into the name field. we don't want to write those, as it would result in double writes if (cbor.preferSerialLabelsOverNames && label != null) { encoder.encodeNumber(label) } else if (name != null) { @@ -128,27 +97,47 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb } if (cbor.writeValueTags) { - parent?.descriptor?.getValueTags(index)?.forEach { encoder.encodeTag(it) } + index?.let { parent?.descriptor?.getValueTags(it)?.forEach { encoder.encodeTag(it) } } + } + + } + + fun encode() { + encodeElementPreamble() + + + //byteStrings are encoded into the data already, as are primitives + data?.let { + it as ByteArray; it.forEach { encoder.writeByte(it.toInt()) } } + next?.encode() } + + override fun toString(): String { + return "(${descriptor?.serialName}:${descriptor?.kind}, index: $index, data: $data, numChidren: $numChildren)" + } } + private var currentToken = Token(null, null, null, null, null, null, null) + private val firstToken = currentToken - private var currentNode = Node(null, null, null, -1, null) fun encode() { - currentNode.encode() + firstToken.encode() } override val serializersModule: SerializersModule get() = cbor.serializersModule + @OptIn(ExperimentalSerializationApi::class) override fun encodeSerializableValue(serializer: SerializationStrategy, value: T) { + if ((encodeByteArrayAsByteString || cbor.alwaysUseByteString) - && serializer.descriptor == ByteArraySerializer().descriptor) { - (currentNode.children.lastOrNull() ?: currentNode).data = + && serializer.descriptor == ByteArraySerializer().descriptor + ) { + currentToken.data = ByteArrayOutput().also { CborEncoder(it).encodeByteString(value as ByteArray) }.toByteArray() } else { super.encodeSerializableValue(serializer, value) @@ -161,43 +150,83 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb //todo: Write size of map or array if known @OptIn(ExperimentalSerializationApi::class) override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder { - if (currentNode.parent == null) - currentNode = Node( - descriptor, - null, - currentNode, - -1, - null - ).apply { currentNode.children += this } - else { - currentNode = currentNode.children.last() + currentToken.numChildren = 0 + + //if we start encoding structures directly (i.e. map, class, list, array) + if(currentToken===firstToken){ + currentToken.descriptor=descriptor } + + structureStack.push(currentToken) return this } override fun endStructure(descriptor: SerialDescriptor) { - currentNode = currentNode.parent ?: throw SerializationException("Root node reached!") + val beginToken = structureStack.pop() + + val buffer = ByteArrayOutput() + val encoder= CborEncoder(buffer) + + if (beginToken.descriptor!!.hasArrayTag()) { + beginToken.descriptor!!.getArrayTags()?.forEach { encoder.encodeTag(it) } + if (cbor.writeDefiniteLengths) encoder.startArray(beginToken.numChildren!!.toULong()) else encoder.startArray() + } else { + when (beginToken.descriptor!!.kind) { + StructureKind.LIST, is PolymorphicKind -> { + if (cbor.writeDefiniteLengths) encoder.startArray(beginToken.numChildren!!.toULong()) + else encoder.startArray() + } + + is StructureKind.MAP -> { + if (cbor.writeDefiniteLengths) encoder.startMap((beginToken.numChildren!! / 2).toULong()) else encoder.startMap() + } + + else -> { + if (cbor.writeDefiniteLengths) encoder.startMap(beginToken.numChildren!!.toULong()) else encoder.startMap() + } + } + } + beginToken.data=buffer.toByteArray() + + + + if (!cbor.writeDefiniteLengths) { + currentToken.next = Token( + descriptor = descriptor, + parent = beginToken.parent, + data = ByteArrayOutput().apply { write(BREAK) }.toByteArray(), + index = null, + next = null, + name = null, + label = null + ) + currentToken = currentToken.next!! + } } override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { encodeByteArrayAsByteString = descriptor.isByteString(index) - currentNode.children += Node( - descriptor.getElementDescriptor(index), - null, - currentNode, - index, - descriptor.getElementName(index), - descriptor.getSerialLabel(index), + currentToken.next = Token( + descriptor = descriptor.getElementDescriptor(index), + parent = structureStack.peek(), + data = null, + index = index, + next = null, + name = descriptor.getElementName(index), + label = descriptor.getSerialLabel(index) ) + currentToken = currentToken.next!! + runCatching { structureStack.peek().numChildren = structureStack.peek().numChildren!! + 1 }.exceptionOrNull() + ?.printStackTrace() return true } //If any of the following functions are called for serializing raw primitives (i.e. something other than a class, // list, map or array, no children exist and the root node needs the data override fun encodeString(value: String) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeString(value) }.toByteArray() @@ -205,67 +234,67 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb } override fun encodeFloat(value: Float) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeFloat(value) }.toByteArray() } } override fun encodeDouble(value: Double) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeDouble(value) }.toByteArray() } } override fun encodeChar(value: Char) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.code.toLong()) }.toByteArray() } } override fun encodeByte(value: Byte) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() } } override fun encodeShort(value: Short) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() } } override fun encodeInt(value: Int) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeNumber(value.toLong()) }.toByteArray() } } override fun encodeLong(value: Long) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeNumber(value) }.toByteArray() } } override fun encodeBoolean(value: Boolean) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeBoolean(value) }.toByteArray() } } override fun encodeNull() { - (currentNode.children.lastOrNull() ?: currentNode).apply { - if (this.descriptor?.kind == StructureKind.CLASS) { - data = ByteArrayOutput().also { CborEncoder(it).encodeEmptyMap() }.toByteArray() + currentToken.apply { + data = if (this.descriptor?.kind == StructureKind.CLASS) { + ByteArrayOutput().also { CborEncoder(it).encodeEmptyMap() }.toByteArray() } else { - data = ByteArrayOutput().also { CborEncoder(it).encodeNull() }.toByteArray() + ByteArrayOutput().also { CborEncoder(it).encodeNull() }.toByteArray() } } } @@ -275,7 +304,7 @@ internal open class CborWriter(private val cbor: Cbor, protected val encoder: Cb enumDescriptor: SerialDescriptor, index: Int ) { - (currentNode.children.lastOrNull() ?: currentNode).apply { + currentToken.apply { data = ByteArrayOutput().also { CborEncoder(it).encodeString(enumDescriptor.getElementName(index)) } .toByteArray() @@ -510,7 +539,8 @@ internal open class CborReader(private val cbor: Cbor, protected val decoder: Cb @OptIn(ExperimentalSerializationApi::class) override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { return if ((decodeByteArrayAsByteString || cbor.alwaysUseByteString) - && deserializer.descriptor == ByteArraySerializer().descriptor) { + && deserializer.descriptor == ByteArraySerializer().descriptor + ) { @Suppress("UNCHECKED_CAST") decoder.nextByteString(tags) as T } else { @@ -707,6 +737,7 @@ internal class CborDecoder(private val input: ByteArrayInput) { readByte() return res } + fun nextTaggedNumber() = nextTaggedNumber(null) private fun nextTaggedNumber(tags: ULongArray?): Pair { @@ -956,7 +987,8 @@ private fun SerialDescriptor.isByteString(index: Int): Boolean { @OptIn(ExperimentalSerializationApi::class) private fun SerialDescriptor.getValueTags(index: Int): ULongArray? { - return kotlin.runCatching { (getElementAnnotations(index).find { it is ValueTags } as ValueTags?)?.tags }.getOrNull() + return kotlin.runCatching { (getElementAnnotations(index).find { it is ValueTags } as ValueTags?)?.tags } + .getOrNull() } private fun SerialDescriptor.isInlineByteString(): Boolean { // inline item classes should only have 1 item @@ -986,7 +1018,8 @@ private fun SerialDescriptor.getKeyTags(): ULongArray? { @OptIn(ExperimentalSerializationApi::class) private fun SerialDescriptor.getSerialLabel(index: Int): Long? { - return kotlin.runCatching { getElementAnnotations(index).filterIsInstance().firstOrNull()?.label }.getOrNull() + return kotlin.runCatching { getElementAnnotations(index).filterIsInstance().firstOrNull()?.label } + .getOrNull() } @OptIn(ExperimentalSerializationApi::class) diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt index 793cc5a04..4342fbd7e 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt @@ -126,8 +126,8 @@ class CbrWriterTest { assertEquals( expected = "bfff", actual = Cbor { explicitNulls = false }.encodeToHexString( - serializer = NullableByteString.serializer(), - value = NullableByteString(byteString = null) + serializer = NullableByteStringDefaultNull.serializer(), + value = NullableByteStringDefaultNull(byteString = null) ) ) } @@ -139,8 +139,8 @@ class CbrWriterTest { assertEquals( expected = "a0", actual = Cbor { explicitNulls = false;writeDefiniteLengths = true }.encodeToHexString( - serializer = NullableByteString.serializer(), - value = NullableByteString(byteString = null) + serializer = NullableByteStringDefaultNull.serializer(), + value = NullableByteStringDefaultNull(byteString = null) ) ) } diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt index 92693b124..2e4558508 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/SampleClasses.kt @@ -91,6 +91,29 @@ data class NullableByteString( } } +@Serializable +data class NullableByteStringDefaultNull( + @ByteString val byteString: ByteArray ? = null +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + + other as NullableByteString + + if (byteString != null) { + if (other.byteString == null) return false + if (!byteString.contentEquals(other.byteString)) return false + } else if (other.byteString != null) return false + + return true + } + + override fun hashCode(): Int { + return byteString?.contentHashCode() ?: 0 + } +} + @Serializable(with = CustomByteStringSerializer::class) data class CustomByteString(val a: Byte, val b: Byte, val c: Byte) From 19eb85e2ba1ae450182d95180e6214519b08789a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Thu, 17 Aug 2023 16:35:50 +0200 Subject: [PATCH 24/98] CBOR: remove bogus `explicitNulls` --- .../src/kotlinx/serialization/cbor/Cbor.kt | 11 +---------- .../serialization/cbor/internal/Encoding.kt | 18 +++++++++++------- .../serialization/cbor/CborArrayTest.kt | 1 - .../serialization/cbor/CborPolymorphismTest.kt | 3 --- .../serialization/cbor/CborReaderTest.kt | 3 +-- .../cbor/CborRootLevelNullsTest.kt | 2 +- .../serialization/cbor/CborTaggedTest.kt | 9 +++------ .../serialization/cbor/CborWriterTest.kt | 4 ++-- 8 files changed, 19 insertions(+), 32 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt index b7f7a708c..dd94f1477 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/Cbor.kt @@ -43,7 +43,6 @@ public sealed class Cbor( internal val writeValueTags: Boolean, internal val verifyKeyTags: Boolean, internal val verifyValueTags: Boolean, - internal val explicitNulls: Boolean, internal val writeDefiniteLengths: Boolean, internal val preferSerialLabelsOverNames: Boolean, internal val alwaysUseByteString: Boolean, @@ -53,7 +52,7 @@ public sealed class Cbor( /** * The default instance of [Cbor] */ - public companion object Default : Cbor(false, false, true, true, true, true, true, false, true, false, EmptySerializersModule()) + public companion object Default : Cbor(false, false, true, true, true, true, false, true, false, EmptySerializersModule()) override fun encodeToByteArray(serializer: SerializationStrategy, value: T): ByteArray { val output = ByteArrayOutput() @@ -78,7 +77,6 @@ private class CborImpl( writeValueTags: Boolean, verifyKeyTags: Boolean, verifyValueTags: Boolean, - encodeNullProperties: Boolean, writeDefiniteLengths: Boolean, preferSerialLabelsOverNames: Boolean, alwaysUseByteString: Boolean, @@ -91,7 +89,6 @@ private class CborImpl( writeValueTags, verifyKeyTags, verifyValueTags, - encodeNullProperties, writeDefiniteLengths, preferSerialLabelsOverNames, alwaysUseByteString, @@ -113,7 +110,6 @@ public fun Cbor(from: Cbor = Cbor, builderAction: CborBuilder.() -> Unit): Cbor builder.writeValueTags, builder.verifyKeyTags, builder.verifyValueTags, - builder.explicitNulls, builder.writeDefiniteLengths, builder.preferSerialLabelsOverNames, builder.alwaysUseByteString, @@ -159,11 +155,6 @@ public class CborBuilder internal constructor(cbor: Cbor) { */ public var verifyValueTags: Boolean = cbor.verifyValueTags - /** - * Specifies whether `null` values should be encoded for nullable properties - */ - public var explicitNulls: Boolean = cbor.explicitNulls - /** * specifies whether structures (maps, object, lists, etc.) should be encoded using definite length encoding */ diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 3db17a04e..0bf742dfc 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -167,22 +167,26 @@ internal open class CborWriter( val buffer = ByteArrayOutput() val encoder= CborEncoder(buffer) - if (beginToken.descriptor!!.hasArrayTag()) { - beginToken.descriptor!!.getArrayTags()?.forEach { encoder.encodeTag(it) } - if (cbor.writeDefiniteLengths) encoder.startArray(beginToken.numChildren!!.toULong()) else encoder.startArray() + //If this nullpointers, we have a structural problem anyhow + val beginDescriptor = beginToken.descriptor!! + val numChildren = beginToken.numChildren!! + + if (beginDescriptor.hasArrayTag()) { + beginDescriptor.getArrayTags()?.forEach { encoder.encodeTag(it) } + if (cbor.writeDefiniteLengths) encoder.startArray(numChildren.toULong()) else encoder.startArray() } else { - when (beginToken.descriptor!!.kind) { + when (beginDescriptor.kind) { StructureKind.LIST, is PolymorphicKind -> { - if (cbor.writeDefiniteLengths) encoder.startArray(beginToken.numChildren!!.toULong()) + if (cbor.writeDefiniteLengths) encoder.startArray(numChildren.toULong()) else encoder.startArray() } is StructureKind.MAP -> { - if (cbor.writeDefiniteLengths) encoder.startMap((beginToken.numChildren!! / 2).toULong()) else encoder.startMap() + if (cbor.writeDefiniteLengths) encoder.startMap((numChildren / 2).toULong()) else encoder.startMap() } else -> { - if (cbor.writeDefiniteLengths) encoder.startMap(beginToken.numChildren!!.toULong()) else encoder.startMap() + if (cbor.writeDefiniteLengths) encoder.startMap(numChildren.toULong()) else encoder.startMap() } } } diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt index 8b03174b2..64cdd0bfe 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborArrayTest.kt @@ -56,7 +56,6 @@ class CborArrayTest { val cbor = Cbor { writeDefiniteLengths = true - explicitNulls = true } assertEquals(referenceHexString, cbor.encodeToHexString(ClassAs4ArrayNullable.serializer(), reference)) diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt index aad44f049..156f47148 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt @@ -16,7 +16,6 @@ class CborPolymorphismTest { val cbor = Cbor { serializersModule = SimplePolymorphicModule } - @Ignore @Test fun testSealedWithOneSubclass() { assertSerializedToBinaryAndRestored( @@ -27,7 +26,6 @@ class CborPolymorphismTest { ) } - @Ignore @Test fun testSealedWithMultipleSubclasses() { val obj = SealedBox( @@ -39,7 +37,6 @@ class CborPolymorphismTest { assertSerializedToBinaryAndRestored(obj, SealedBox.serializer(), cbor) } - @Ignore @Test fun testOpenPolymorphism() { val obj = PolyBox( diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt index de1a80546..e2b71d3f4 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborReaderTest.kt @@ -141,10 +141,9 @@ class CborReaderTest { ) } - @Ignore @Test fun testNullables() { - Cbor { explicitNulls = false }.decodeFromHexString("a0") + Cbor.decodeFromHexString("a0") } /** diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborRootLevelNullsTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborRootLevelNullsTest.kt index 3872ca559..188ae3e7f 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborRootLevelNullsTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborRootLevelNullsTest.kt @@ -15,7 +15,7 @@ class CborRootLevelNullsTest { @Test fun testNull() { val obj: Simple? = null - listOf(Cbor, Cbor { explicitNulls = false; writeDefiniteLengths = true }).forEach { + listOf(Cbor, Cbor { writeDefiniteLengths = true }).forEach { val content = it.encodeToByteArray(Simple.serializer().nullable, obj) assertTrue(content.contentEquals(byteArrayOf(0xf6.toByte()))) } diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt index c30ae6e74..3ef3c1107 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt @@ -345,9 +345,7 @@ class CborTaggedTest { val wrongTag55ForPropertyC = "A46161CC1A0FFFFFFFD822616220D8376163D84E42CAFE6164D85ACC6B48656C6C6F20576F726C64" listOf( Cbor, - Cbor { writeDefiniteLengths = true }, - Cbor { writeDefiniteLengths = true;explicitNulls = false }, - Cbor { explicitNulls = false }).forEach { cbor -> + Cbor { writeDefiniteLengths = true }).forEach { cbor -> assertFailsWith(CborDecodingException::class, message = "CBOR tags [55] do not match expected tags [56]") { Cbor.decodeFromHexString( @@ -357,9 +355,8 @@ class CborTaggedTest { } } listOf( - Cbor { verifyKeyTags = false }, - Cbor { verifyKeyTags = false;writeDefiniteLengths = true }, - Cbor { verifyKeyTags = false;explicitNulls = false }).forEach { cbor -> + Cbor { verifyKeyTags = false; writeDefiniteLengths = true }, + Cbor { verifyKeyTags = false }).forEach { cbor -> assertEquals(reference, cbor.decodeFromHexString(wrongTag55ForPropertyC)) } } diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt index 4342fbd7e..61f337293 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborWriterTest.kt @@ -125,7 +125,7 @@ class CbrWriterTest { */ assertEquals( expected = "bfff", - actual = Cbor { explicitNulls = false }.encodeToHexString( + actual = Cbor.encodeToHexString( serializer = NullableByteStringDefaultNull.serializer(), value = NullableByteStringDefaultNull(byteString = null) ) @@ -138,7 +138,7 @@ class CbrWriterTest { */ assertEquals( expected = "a0", - actual = Cbor { explicitNulls = false;writeDefiniteLengths = true }.encodeToHexString( + actual = Cbor { writeDefiniteLengths = true }.encodeToHexString( serializer = NullableByteStringDefaultNull.serializer(), value = NullableByteStringDefaultNull(byteString = null) ) From 9c305f4d43b6911fdcff8762298468c88c0235b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Thu, 17 Aug 2023 23:12:48 +0200 Subject: [PATCH 25/98] CBOR: streamline an simplify encoding --- .../serialization/cbor/internal/Encoding.kt | 132 +++++++++--------- .../cbor/CborPolymorphismTest.kt | 3 + .../serialization/cbor/CborTaggedTest.kt | 3 +- 3 files changed, 73 insertions(+), 65 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 0bf742dfc..f785d49b9 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -72,54 +72,27 @@ internal open class CborWriter( inner class Token( var descriptor: SerialDescriptor?, - val parent: Token?, - var data: Any?, + var data: ByteArray?, var next: Token?, - val index: Int?, - val name: String?, - val label: Long? = null, var numChildren: Int? = null, + var preamble: ByteArray? = null, ) { - private fun encodeElementPreamble() { - - if (parent?.descriptor?.hasArrayTag() != true) { - if (cbor.writeKeyTags) { - index?.let { parent?.descriptor?.getKeyTags(it)?.forEach { encoder.encodeTag(it) } } - } - if ((parent?.descriptor?.kind !is StructureKind.LIST) && (parent?.descriptor?.kind !is StructureKind.MAP)) { //TODO polymorphicKind? - //indices are put into the name field. we don't want to write those, as it would result in double writes - if (cbor.preferSerialLabelsOverNames && label != null) { - encoder.encodeNumber(label) - } else if (name != null) { - encoder.encodeString(name) - } - } - } - - if (cbor.writeValueTags) { - index?.let { parent?.descriptor?.getValueTags(it)?.forEach { encoder.encodeTag(it) } } - } - - } fun encode() { - encodeElementPreamble() - + preamble?.let { encoder.pasteBytes(it) } //byteStrings are encoded into the data already, as are primitives - data?.let { - it as ByteArray; it.forEach { encoder.writeByte(it.toInt()) } - } + data?.let { encoder.pasteBytes(it) } next?.encode() } override fun toString(): String { - return "(${descriptor?.serialName}:${descriptor?.kind}, index: $index, data: $data, numChidren: $numChildren)" + return "(${descriptor?.serialName}:${descriptor?.kind}, data: $data, numChildren: $numChildren)" } } - private var currentToken = Token(null, null, null, null, null, null, null) + private var currentToken = Token(null, null, null, null, null) private val firstToken = currentToken fun encode() { @@ -146,15 +119,13 @@ internal open class CborWriter( override fun shouldEncodeElementDefault(descriptor: SerialDescriptor, index: Int): Boolean = cbor.encodeDefaults - - //todo: Write size of map or array if known @OptIn(ExperimentalSerializationApi::class) override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder { currentToken.numChildren = 0 //if we start encoding structures directly (i.e. map, class, list, array) - if(currentToken===firstToken){ - currentToken.descriptor=descriptor + if (currentToken === firstToken) { + currentToken.descriptor = descriptor } structureStack.push(currentToken) @@ -165,44 +136,40 @@ internal open class CborWriter( val beginToken = structureStack.pop() val buffer = ByteArrayOutput() - val encoder= CborEncoder(buffer) + val encoder = CborEncoder(buffer) //If this nullpointers, we have a structural problem anyhow val beginDescriptor = beginToken.descriptor!! val numChildren = beginToken.numChildren!! if (beginDescriptor.hasArrayTag()) { - beginDescriptor.getArrayTags()?.forEach { encoder.encodeTag(it) } - if (cbor.writeDefiniteLengths) encoder.startArray(numChildren.toULong()) else encoder.startArray() - } else { - when (beginDescriptor.kind) { - StructureKind.LIST, is PolymorphicKind -> { - if (cbor.writeDefiniteLengths) encoder.startArray(numChildren.toULong()) - else encoder.startArray() - } + beginDescriptor.getArrayTags()?.forEach { encoder.encodeTag(it) } + if (cbor.writeDefiniteLengths) encoder.startArray(numChildren.toULong()) else encoder.startArray() + } else { + when (beginDescriptor.kind) { + StructureKind.LIST, is PolymorphicKind -> { + if (cbor.writeDefiniteLengths) encoder.startArray(numChildren.toULong()) + else encoder.startArray() + } - is StructureKind.MAP -> { - if (cbor.writeDefiniteLengths) encoder.startMap((numChildren / 2).toULong()) else encoder.startMap() - } + is StructureKind.MAP -> { + if (cbor.writeDefiniteLengths) encoder.startMap((numChildren / 2).toULong()) else encoder.startMap() + } - else -> { - if (cbor.writeDefiniteLengths) encoder.startMap(numChildren.toULong()) else encoder.startMap() - } + else -> { + if (cbor.writeDefiniteLengths) encoder.startMap(numChildren.toULong()) else encoder.startMap() } } - beginToken.data=buffer.toByteArray() + } + beginToken.data = buffer.toByteArray() if (!cbor.writeDefiniteLengths) { currentToken.next = Token( descriptor = descriptor, - parent = beginToken.parent, data = ByteArrayOutput().apply { write(BREAK) }.toByteArray(), - index = null, next = null, - name = null, - label = null ) currentToken = currentToken.next!! } @@ -211,15 +178,18 @@ internal open class CborWriter( override fun encodeElement(descriptor: SerialDescriptor, index: Int): Boolean { + val parent = structureStack.peek() as Token? + val name = descriptor.getElementName(index) as String? + val label = descriptor.getSerialLabel(index) + + val preamble = encodePreamble(parent?.descriptor, index, label, name) + encodeByteArrayAsByteString = descriptor.isByteString(index) currentToken.next = Token( descriptor = descriptor.getElementDescriptor(index), - parent = structureStack.peek(), data = null, - index = index, next = null, - name = descriptor.getElementName(index), - label = descriptor.getSerialLabel(index) + preamble = preamble ) currentToken = currentToken.next!! runCatching { structureStack.peek().numChildren = structureStack.peek().numChildren!! + 1 }.exceptionOrNull() @@ -227,6 +197,35 @@ internal open class CborWriter( return true } + private fun encodePreamble( + parentDescriptor: SerialDescriptor?, + index: Int, + label: Long?, + name: String? + ): ByteArray { + val preamble = ByteArrayOutput() + val encoder = CborEncoder(preamble) + + if (parentDescriptor?.hasArrayTag() != true) { + if (cbor.writeKeyTags) { + index.let { parentDescriptor?.getKeyTags(it)?.forEach { encoder.encodeTag(it) } } + } + if ((parentDescriptor?.kind !is StructureKind.LIST) && (parentDescriptor?.kind !is StructureKind.MAP)) { //TODO polymorphicKind? + //indices are put into the name field. we don't want to write those, as it would result in double writes + if (cbor.preferSerialLabelsOverNames && label != null) { + encoder.encodeNumber(label) + } else if (name != null) { + encoder.encodeString(name) + } + } + } + + if (cbor.writeValueTags) { + index.let { parentDescriptor?.getValueTags(it)?.forEach { encoder.encodeTag(it) } } + } + return preamble.toByteArray() + } + //If any of the following functions are called for serializing raw primitives (i.e. something other than a class, // list, map or array, no children exist and the root node needs the data override fun encodeString(value: String) { @@ -318,7 +317,8 @@ internal open class CborWriter( } // For details of representation, see https://tools.ietf.org/html/rfc7049#section-2.1 -internal class CborEncoder(private val output: ByteArrayOutput) { +@JvmInline +internal value class CborEncoder(private val output: ByteArrayOutput) { fun startArray() = output.write(BEGIN_ARRAY) @@ -348,7 +348,11 @@ internal class CborEncoder(private val output: ByteArrayOutput) { fun encodeEmptyMap() = output.write(EMPTY_MAP) - internal fun writeByte(byteValue: Int) = output.write(byteValue) + private fun writeByte(byteValue: Int) = output.write(byteValue) + + internal fun pasteBytes(bytes: ByteArray) { + output.write(bytes) + } fun encodeBoolean(value: Boolean) = output.write(if (value) TRUE else FALSE) @@ -388,7 +392,7 @@ internal class CborEncoder(private val output: ByteArrayOutput) { private fun composeNumber(value: Long): ByteArray = if (value >= 0) composePositive(value.toULong()) else composeNegative(value) - internal fun composePositive(value: ULong): ByteArray = when (value) { + private fun composePositive(value: ULong): ByteArray = when (value) { in 0u..23u -> byteArrayOf(value.toByte()) in 24u..UByte.MAX_VALUE.toUInt() -> byteArrayOf(24, value.toByte()) in (UByte.MAX_VALUE.toUInt() + 1u)..UShort.MAX_VALUE.toUInt() -> encodeToByteArray(value, 2, 25) diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt index 156f47148..aad44f049 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt @@ -16,6 +16,7 @@ class CborPolymorphismTest { val cbor = Cbor { serializersModule = SimplePolymorphicModule } + @Ignore @Test fun testSealedWithOneSubclass() { assertSerializedToBinaryAndRestored( @@ -26,6 +27,7 @@ class CborPolymorphismTest { ) } + @Ignore @Test fun testSealedWithMultipleSubclasses() { val obj = SealedBox( @@ -37,6 +39,7 @@ class CborPolymorphismTest { assertSerializedToBinaryAndRestored(obj, SealedBox.serializer(), cbor) } + @Ignore @Test fun testOpenPolymorphism() { val obj = PolyBox( diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt index 3ef3c1107..b3bf902ed 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborTaggedTest.kt @@ -2,6 +2,7 @@ * Copyright 2017-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ +@file:OptIn(ExperimentalUnsignedTypes::class) package kotlinx.serialization.cbor import kotlinx.serialization.* @@ -348,7 +349,7 @@ class CborTaggedTest { Cbor { writeDefiniteLengths = true }).forEach { cbor -> assertFailsWith(CborDecodingException::class, message = "CBOR tags [55] do not match expected tags [56]") { - Cbor.decodeFromHexString( + cbor.decodeFromHexString( DataWithTags.serializer(), wrongTag55ForPropertyC ) From 72b61dee0a896a52385ca926d4ea5fc06c8a2235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Thu, 17 Aug 2023 23:14:46 +0200 Subject: [PATCH 26/98] CBOR: minor cleanups --- .../serialization/cbor/internal/Encoding.kt | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index f785d49b9..56d98a17b 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -48,17 +48,6 @@ private const val SINGLE_PRECISION_MAX_EXPONENT = 0xFF private const val SINGLE_PRECISION_NORMALIZE_BASE = 0.5f -@JvmInline -internal value class Stack(private val list: MutableList = mutableListOf()) { - fun push(value: T) { - list += value - } - - fun pop() = list.removeLast() - - fun peek() = list.last() -} - // Writes class as map [fieldName, fieldValue] internal open class CborWriter( private val cbor: Cbor, @@ -66,10 +55,22 @@ internal open class CborWriter( ) : AbstractEncoder() { - var encodeByteArrayAsByteString = false + private var encodeByteArrayAsByteString = false private val structureStack: Stack = Stack() + @JvmInline + private value class Stack(private val list: MutableList = mutableListOf()) { + fun push(value: T) { + list += value + } + + fun pop() = list.removeLast() + + fun peek() = list.last() + } + + inner class Token( var descriptor: SerialDescriptor?, var data: ByteArray?, From 972657dc9646e905235386c68348faedf4cc76c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Fri, 18 Aug 2023 08:48:21 +0200 Subject: [PATCH 27/98] CBOR: fix polymorphism --- .../serialization/cbor/internal/Encoding.kt | 25 +++++++++++-------- .../cbor/CborPolymorphismTest.kt | 3 --- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt index 56d98a17b..5c40d9f1d 100644 --- a/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt +++ b/formats/cbor/commonMain/src/kotlinx/serialization/cbor/internal/Encoding.kt @@ -207,20 +207,23 @@ internal open class CborWriter( val preamble = ByteArrayOutput() val encoder = CborEncoder(preamble) - if (parentDescriptor?.hasArrayTag() != true) { - if (cbor.writeKeyTags) { - index.let { parentDescriptor?.getKeyTags(it)?.forEach { encoder.encodeTag(it) } } - } - if ((parentDescriptor?.kind !is StructureKind.LIST) && (parentDescriptor?.kind !is StructureKind.MAP)) { //TODO polymorphicKind? - //indices are put into the name field. we don't want to write those, as it would result in double writes - if (cbor.preferSerialLabelsOverNames && label != null) { - encoder.encodeNumber(label) - } else if (name != null) { - encoder.encodeString(name) + parentDescriptor?.let { descriptor -> + + + if (!descriptor.hasArrayTag()) { + if (cbor.writeKeyTags) { + index.let { descriptor.getKeyTags(it)?.forEach { encoder.encodeTag(it) } } + } + if ((descriptor.kind !is StructureKind.LIST) && (descriptor.kind !is StructureKind.MAP) && (descriptor.kind !is PolymorphicKind)) { + //indices are put into the name field. we don't want to write those, as it would result in double writes + if (cbor.preferSerialLabelsOverNames && label != null) { + encoder.encodeNumber(label) + } else if (name != null) { + encoder.encodeString(name) + } } } } - if (cbor.writeValueTags) { index.let { parentDescriptor?.getValueTags(it)?.forEach { encoder.encodeTag(it) } } } diff --git a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt index aad44f049..156f47148 100644 --- a/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt +++ b/formats/cbor/commonTest/src/kotlinx/serialization/cbor/CborPolymorphismTest.kt @@ -16,7 +16,6 @@ class CborPolymorphismTest { val cbor = Cbor { serializersModule = SimplePolymorphicModule } - @Ignore @Test fun testSealedWithOneSubclass() { assertSerializedToBinaryAndRestored( @@ -27,7 +26,6 @@ class CborPolymorphismTest { ) } - @Ignore @Test fun testSealedWithMultipleSubclasses() { val obj = SealedBox( @@ -39,7 +37,6 @@ class CborPolymorphismTest { assertSerializedToBinaryAndRestored(obj, SealedBox.serializer(), cbor) } - @Ignore @Test fun testOpenPolymorphism() { val obj = PolyBox( From 93ffa7a8b2a7a4d6ccc2f410c47000588b6e0dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Pr=C3=BCnster?= Date: Fri, 18 Aug 2023 11:52:15 +0200 Subject: [PATCH 28/98] CBOR: document new features --- docs/formats.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/formats.md b/docs/formats.md index 8fb6eaef5..36d17c444 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -164,6 +164,8 @@ Per the [RFC 7049 Major Types] section, CBOR supports the following data types: By default, Kotlin `ByteArray` instances are encoded as **major type 4**. When **major type 2** is desired, then the [`@ByteString`][ByteString] annotation can be used. +Moreover, the `alwaysUseByteString` configuration switch allows for globally preferring ** major type 2** without needing +to annotate every `ByteArray` in a class hierarchy.