feat: add bin format family support

**NOTES:** Changed MsgPack serializer `decodeFromByteArray` and `encodeToByteArray` to use decodeSerializableValue and encodeSerializableValue by default, to properly handle ByteArraySerializer This closes #6
esensar · Dec 24, 2020 · eb93fa0 · eb93fa0
1 parent 25c6dd8
commit eb93fa0
Show file tree

Hide file tree

Showing 8 changed files with 117 additions and 3 deletions.
diff --git a/...-msgpack/src/commonMain/kotlin/com.ensarsarajcic.kotlinx.serialization.msgpack/MsgPack.kt b/...-msgpack/src/commonMain/kotlin/com.ensarsarajcic.kotlinx.serialization.msgpack/MsgPack.kt
@@ -39,13 +39,13 @@ class MsgPack @JvmOverloads constructor(
 
     override fun <T> decodeFromByteArray(deserializer: DeserializationStrategy<T>, bytes: ByteArray): T {
         val decoder = MsgPackDecoder(configuration, serializersModule, bytes)
-        return deserializer.deserialize(decoder)
+        return decoder.decodeSerializableValue(deserializer)
     }
 
     override fun <T> encodeToByteArray(serializer: SerializationStrategy<T>, value: T): ByteArray {
         val encoder = MsgPackEncoder(configuration, serializersModule)
         kotlin.runCatching {
-            serializer.serialize(encoder, value)
+            encoder.encodeSerializableValue(serializer, value)
         }.fold(
             onSuccess = { return encoder.result.toByteArray() },
             onFailure = {

diff --git a/...k/src/commonMain/kotlin/com.ensarsarajcic.kotlinx.serialization.msgpack/MsgPackDecoder.kt b/...k/src/commonMain/kotlin/com.ensarsarajcic.kotlinx.serialization.msgpack/MsgPackDecoder.kt
@@ -1,6 +1,8 @@
 package com.ensarsarajcic.kotlinx.serialization.msgpack
 
 import com.ensarsarajcic.kotlinx.serialization.msgpack.types.MsgPackType
+import kotlinx.serialization.DeserializationStrategy
+import kotlinx.serialization.builtins.ByteArraySerializer
 import kotlinx.serialization.descriptors.SerialDescriptor
 import kotlinx.serialization.descriptors.StructureKind
 import kotlinx.serialization.encoding.AbstractDecoder
@@ -155,6 +157,23 @@ internal class MsgPackDecoder(
         return takeNext(length).decodeToString()
     }
 
+    fun decodeByteArray(): ByteArray {
+        val next = byteArray.getOrNull(index) ?: throw Exception("End of stream")
+        index++
+        val length = when (next) {
+            MsgPackType.Bin.BIN8 -> requireNextByte().toInt() and 0xff
+            MsgPackType.Bin.BIN16 -> takeNext(2).joinToNumber()
+            // TODO: this may have issues with long byte arrays, since size will overflow
+            MsgPackType.Bin.BIN32 -> takeNext(4).joinToNumber()
+            else -> {
+                index--
+                throw TODO("Add a more descriptive error when wrong type is found!")
+            }
+        }
+        if (length == 0) return byteArrayOf()
+        return takeNext(length)
+    }
+
     override fun decodeCollectionSize(descriptor: SerialDescriptor): Int {
         val next = byteArray.getOrNull(index) ?: throw Exception("End of stream")
         index++
@@ -191,6 +210,14 @@ internal class MsgPackDecoder(
         }
     }
 
+    override fun <T> decodeSerializableValue(deserializer: DeserializationStrategy<T>): T {
+        return if (deserializer == ByteArraySerializer()) {
+            decodeByteArray() as T
+        } else {
+            super.decodeSerializableValue(deserializer)
+        }
+    }
+
     override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder {
         if (descriptor.kind in arrayOf(StructureKind.CLASS, StructureKind.OBJECT)) {
             decodingClass = true

diff --git a/...k/src/commonMain/kotlin/com.ensarsarajcic.kotlinx.serialization.msgpack/MsgPackEncoder.kt b/...k/src/commonMain/kotlin/com.ensarsarajcic.kotlinx.serialization.msgpack/MsgPackEncoder.kt
@@ -3,6 +3,7 @@ package com.ensarsarajcic.kotlinx.serialization.msgpack
 import com.ensarsarajcic.kotlinx.serialization.msgpack.types.MsgPackType
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.SerializationStrategy
+import kotlinx.serialization.builtins.ByteArraySerializer
 import kotlinx.serialization.descriptors.SerialDescriptor
 import kotlinx.serialization.descriptors.StructureKind
 import kotlinx.serialization.encoding.AbstractEncoder
@@ -127,6 +128,25 @@ internal class MsgPackEncoder(
         result.addAll(bytes.toList())
     }
 
+    fun encodeByteArray(value: ByteArray) {
+        when {
+            value.size <= MsgPackType.Bin.MAX_BIN8_LENGTH -> {
+                result.add(MsgPackType.Bin.BIN8)
+                result.addAll(value.size.toByte().splitToByteArray().toList())
+            }
+            value.size <= MsgPackType.Bin.MAX_BIN16_LENGTH -> {
+                result.add(MsgPackType.Bin.BIN16)
+                result.addAll(value.size.toShort().splitToByteArray().toList())
+            }
+            value.size <= MsgPackType.Bin.MAX_BIN32_LENGTH -> {
+                result.add(MsgPackType.Bin.BIN32)
+                result.addAll(value.size.toInt().splitToByteArray().toList())
+            }
+            else -> TODO("TOO LONG STRING")
+        }
+        result.addAll(value.toList())
+    }
+
     override fun beginStructure(descriptor: SerialDescriptor): CompositeEncoder {
         return if (descriptor.kind in arrayOf(StructureKind.CLASS, StructureKind.OBJECT)) {
             beginCollection(descriptor, descriptor.elementsCount)
@@ -175,10 +195,27 @@ internal class MsgPackEncoder(
         return this
     }
 
+    override fun <T> encodeSerializableValue(serializer: SerializationStrategy<T>, value: T) {
+        if (serializer == ByteArraySerializer()) {
+            encodeByteArray(value as ByteArray)
+        } else {
+            super.encodeSerializableValue(serializer, value)
+        }
+    }
+
     override fun endStructure(descriptor: SerialDescriptor) {
         // no-op, everything is handled when starting structure/collection
     }
 
+    // TODO Refactor as a completely separate class
+    internal inner class ByteArrayEncoder : CompositeEncoder, AbstractEncoder() {
+        override val serializersModule: SerializersModule = this@MsgPackEncoder.serializersModule
+
+        override fun encodeByte(value: Byte) {
+            result.add(value)
+        }
+    }
+
     // TODO Refactor as a completely separate class
     internal inner class MsgPackClassEncoder : CompositeEncoder {
         override val serializersModule: SerializersModule = this@MsgPackEncoder.serializersModule

diff --git a/...rc/commonMain/kotlin/com.ensarsarajcic.kotlinx.serialization.msgpack/types/MsgPackType.kt b/...rc/commonMain/kotlin/com.ensarsarajcic.kotlinx.serialization.msgpack/types/MsgPackType.kt
@@ -70,6 +70,16 @@ internal object MsgPackType {
         const val MAX_STR32_LENGTH = Int.MAX_UINT
     }
 
+    internal object Bin {
+        const val BIN8 = 0xc4.toByte()
+        const val BIN16 = 0xc5.toByte()
+        const val BIN32 = 0xc6.toByte()
+
+        const val MAX_BIN8_LENGTH = Int.MAX_UBYTE
+        const val MAX_BIN16_LENGTH = Int.MAX_USHORT
+        const val MAX_BIN32_LENGTH = Int.MAX_UINT
+    }
+
     internal object Array {
         const val ARRAY16 = 0xdc.toByte()
         const val ARRAY32 = 0xdd.toByte()

diff --git a/...c/commonTest/kotlin/com/ensarsarajcic/kotlinx/serialization/msgpack/MsgPackDecoderTest.kt b/...c/commonTest/kotlin/com/ensarsarajcic/kotlinx/serialization/msgpack/MsgPackDecoderTest.kt
@@ -4,6 +4,7 @@ import kotlinx.serialization.builtins.ArraySerializer
 import kotlinx.serialization.builtins.MapSerializer
 import kotlinx.serialization.builtins.serializer
 import kotlinx.serialization.modules.SerializersModule
+import kotlinx.serialization.serializer
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertTrue
@@ -101,6 +102,15 @@ internal class MsgPackDecoderTest {
         )
     }
 
+    @Test
+    fun testByteArrayDecode() {
+        TestData.bin8TestPairs.forEach { (input, result) ->
+            MsgPackDecoder(MsgPackConfiguration.default, SerializersModule {}, input.hexStringToByteArray()).also {
+                assertTrue { result.contentEquals(it.decodeSerializableValue(serializer())) }
+            }
+        }
+    }
+
     @Test
     fun testArrayDecodeStringArrays() {
         TestData.stringArrayTestPairs.forEach { (input, result) ->
@@ -140,7 +150,6 @@ internal class MsgPackDecoderTest {
     private fun <RESULT> testPairs(decodeFunction: MsgPackDecoder.() -> RESULT, vararg pairs: Pair<String, RESULT>) {
         pairs.forEach { (input, result) ->
             MsgPackDecoder(MsgPackConfiguration.default, SerializersModule {}, input.hexStringToByteArray()).also {
-                println(input)
                 assertEquals(result, it.decodeFunction())
             }
         }

diff --git a/...c/commonTest/kotlin/com/ensarsarajcic/kotlinx/serialization/msgpack/MsgPackEncoderTest.kt b/...c/commonTest/kotlin/com/ensarsarajcic/kotlinx/serialization/msgpack/MsgPackEncoderTest.kt
@@ -4,6 +4,7 @@ import kotlinx.serialization.builtins.ArraySerializer
 import kotlinx.serialization.builtins.MapSerializer
 import kotlinx.serialization.builtins.serializer
 import kotlinx.serialization.modules.SerializersModule
+import kotlinx.serialization.serializer
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -93,6 +94,14 @@ internal class MsgPackEncoderTest {
         )
     }
 
+    @Test
+    fun testBinaryEncode() {
+        testPairs(
+            { this.encodeSerializableValue(serializer<ByteArray>(), it) },
+            *TestData.bin8TestPairs
+        )
+    }
+
     @Test
     fun testArrayEncodeStringArrays() {
         TestData.stringArrayTestPairs.forEach { (result, input) ->

diff --git a/...pack/src/commonTest/kotlin/com/ensarsarajcic/kotlinx/serialization/msgpack/MsgPackTest.kt b/...pack/src/commonTest/kotlin/com/ensarsarajcic/kotlinx/serialization/msgpack/MsgPackTest.kt
@@ -2,9 +2,11 @@ package com.ensarsarajcic.kotlinx.serialization.msgpack
 
 import kotlinx.serialization.KSerializer
 import kotlinx.serialization.builtins.ArraySerializer
+import kotlinx.serialization.builtins.ByteArraySerializer
 import kotlinx.serialization.builtins.MapSerializer
 import kotlinx.serialization.builtins.nullable
 import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.serializer
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertTrue
@@ -180,6 +182,21 @@ internal class MsgPackTest {
         )
     }
 
+    @Test
+    fun testBinaryEncode() {
+        testEncodePairs(
+            ByteArraySerializer(),
+            *TestData.bin8TestPairs
+        )
+    }
+
+    @Test
+    fun testBinaryDecode() {
+        TestData.bin8TestPairs.forEach { (value, expectedResult) ->
+            assertTrue { expectedResult.contentEquals(MsgPack.default.decodeFromByteArray(ByteArraySerializer(), value.hexStringToByteArray())) }
+        }
+    }
+
     @Test
     fun testArrayEncode() {
         testEncodePairs(

diff --git a/...msgpack/src/commonTest/kotlin/com/ensarsarajcic/kotlinx/serialization/msgpack/TestData.kt b/...msgpack/src/commonTest/kotlin/com/ensarsarajcic/kotlinx/serialization/msgpack/TestData.kt
@@ -86,6 +86,11 @@ object TestData {
         "da010074657374747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474" to "testtttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt", // Min Str16 size
         "da010a74657374747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474c48dc487c48dc487c2bc" to "testttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttčćčć¼", // UTF-8 support
     )
+    val bin8TestPairs = arrayOf(
+        "c409010203040506070809" to byteArrayOf(1, 2, 3, 4, 5, 6, 7, 8, 9),
+        "c400" to byteArrayOf(),
+        "c4ff746573747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474747474" to byteArrayOf(116, 101, 115, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116)
+    )
     val intArrayTestPairs = arrayOf(
         "93010203" to arrayOf(1, 2, 3),
         "94ffccffcd0145ce0009fbf4" to arrayOf(-1, 255, 325, 654324),