From 89015053b52d5d95c5e6f5fccd4dd3615baa658c Mon Sep 17 00:00:00 2001 From: "Sergey.Shanshin" Date: Mon, 7 Jun 2021 17:29:34 +0300 Subject: [PATCH 1/2] Explicit nulls flag for JSON format Resolves #195 --- .../benchmarks/json/ImplicitNullsBenchmark.kt | 118 ++++++++++++++ .../benchmarks/json/TwitterBenchmark.kt | 8 +- core/api/kotlinx-serialization-core.api | 14 +- .../serialization/encoding/AbstractEncoder.kt | 4 +- .../serialization/internal/ElementMarker.kt | 116 ++++++++++++++ .../kotlinx/serialization/internal/Tagged.kt | 4 +- .../serialization/ElementMarkerTest.kt | 97 +++++++++++ .../json/api/kotlinx-serialization-json.api | 3 + .../src/kotlinx/serialization/json/Json.kt | 15 +- .../serialization/json/JsonConfiguration.kt | 5 +- .../json/internal/JsonElementMarker.kt | 32 ++++ .../json/internal/StreamingJsonDecoder.kt | 19 ++- .../json/internal/StreamingJsonEncoder.kt | 11 ++ .../json/internal/TreeJsonDecoder.kt | 21 ++- .../json/internal/TreeJsonEncoder.kt | 11 ++ .../json/AbstractJsonImplicitNullsTest.kt | 151 ++++++++++++++++++ .../json/JsonImplicitNullsTest.kt | 13 ++ .../serialization/json/JsonTestBase.kt | 2 +- .../json/JsonTreeImplicitNullsTest.kt | 14 ++ .../json/internal/DynamicDecoders.kt | 19 ++- .../json/internal/DynamicEncoders.kt | 11 ++ .../serialization/json/EncodeToDynamicTest.kt | 21 --- .../json/JsonDynamicImplicitNullsTest.kt | 14 ++ .../protobuf/internal/ProtobufDecoding.kt | 129 +++------------ 24 files changed, 694 insertions(+), 158 deletions(-) create mode 100644 benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/ImplicitNullsBenchmark.kt create mode 100644 core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt create mode 100644 core/commonTest/src/kotlinx/serialization/ElementMarkerTest.kt create mode 100644 formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonElementMarker.kt create mode 100644 formats/json/commonTest/src/kotlinx/serialization/json/AbstractJsonImplicitNullsTest.kt create mode 100644 formats/json/commonTest/src/kotlinx/serialization/json/JsonImplicitNullsTest.kt create mode 100644 formats/json/commonTest/src/kotlinx/serialization/json/JsonTreeImplicitNullsTest.kt create mode 100644 formats/json/jsTest/src/kotlinx/serialization/json/JsonDynamicImplicitNullsTest.kt diff --git a/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/ImplicitNullsBenchmark.kt b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/ImplicitNullsBenchmark.kt new file mode 100644 index 000000000..11d1240dd --- /dev/null +++ b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/ImplicitNullsBenchmark.kt @@ -0,0 +1,118 @@ +package kotlinx.benchmarks.json + +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.openjdk.jmh.annotations.* +import java.util.concurrent.TimeUnit + +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Fork(2) +open class ImplicitNullsBenchmark { + + @Serializable + data class Values( + val field0: Int?, + val field1: Int?, + val field2: Int?, + val field3: Int?, + val field4: Int?, + val field5: Int?, + val field6: Int?, + val field7: Int?, + val field8: Int?, + val field9: Int?, + + val field10: Int?, + val field11: Int?, + val field12: Int?, + val field13: Int?, + val field14: Int?, + val field15: Int?, + val field16: Int?, + val field17: Int?, + val field18: Int?, + val field19: Int?, + + val field20: Int?, + val field21: Int?, + val field22: Int?, + val field23: Int?, + val field24: Int?, + val field25: Int?, + val field26: Int?, + val field27: Int?, + val field28: Int?, + val field29: Int?, + + val field30: Int?, + val field31: Int? + ) + + + private val jsonImplicitNulls = Json { explicitNulls = false } + + private val valueWithNulls = Values( + null, null, 2, null, null, null, null, null, null, null, + null, null, null, null, 14, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, + null, null + ) + + + private val jsonWithNulls = """{"field0":null,"field1":null,"field2":2,"field3":null,"field4":null,"field5":null, + |"field6":null,"field7":null,"field8":null,"field9":null,"field10":null,"field11":null,"field12":null, + |"field13":null,"field14":14,"field15":null,"field16":null,"field17":null,"field18":null,"field19":null, + |"field20":null,"field21":null,"field22":null,"field23":null,"field24":null,"field25":null,"field26":null, + |"field27":null,"field28":null,"field29":null,"field30":null,"field31":null}""".trimMargin() + + private val jsonNoNulls = """{"field0":0,"field1":1,"field2":2,"field3":3,"field4":4,"field5":5, + |"field6":6,"field7":7,"field8":8,"field9":9,"field10":10,"field11":11,"field12":12, + |"field13":13,"field14":14,"field15":15,"field16":16,"field17":17,"field18":18,"field19":19, + |"field20":20,"field21":21,"field22":22,"field23":23,"field24":24,"field25":25,"field26":26, + |"field27":27,"field28":28,"field29":29,"field30":30,"field31":31}""".trimMargin() + + private val jsonWithAbsence = """{"field2":2, "field14":14}""" + + private val serializer = Values.serializer() + + @Benchmark + fun decodeNoNulls() { + Json.decodeFromString(serializer, jsonNoNulls) + } + + @Benchmark + fun decodeNoNullsImplicit() { + jsonImplicitNulls.decodeFromString(serializer, jsonNoNulls) + } + + @Benchmark + fun decodeNulls() { + Json.decodeFromString(serializer, jsonWithNulls) + } + + @Benchmark + fun decodeNullsImplicit() { + jsonImplicitNulls.decodeFromString(serializer, jsonWithNulls) + } + + @Benchmark + fun decodeAbsenceImplicit() { + jsonImplicitNulls.decodeFromString(serializer, jsonWithAbsence) + } + + @Benchmark + fun encodeNulls() { + Json.encodeToString(serializer, valueWithNulls) + } + + @Benchmark + fun encodeNullsImplicit() { + jsonImplicitNulls.encodeToString(serializer, valueWithNulls) + } +} diff --git a/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterBenchmark.kt b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterBenchmark.kt index 15e9ea46b..5c930ec6a 100644 --- a/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterBenchmark.kt +++ b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterBenchmark.kt @@ -1,10 +1,7 @@ package kotlinx.benchmarks.json import kotlinx.benchmarks.model.* -import kotlinx.serialization.* import kotlinx.serialization.json.* -import kotlinx.serialization.json.Json.Default.decodeFromString -import kotlinx.serialization.json.Json.Default.encodeToString import org.openjdk.jmh.annotations.* import java.util.concurrent.* @@ -25,6 +22,8 @@ open class TwitterBenchmark { private val input = TwitterBenchmark::class.java.getResource("/twitter.json").readBytes().decodeToString() private val twitter = Json.decodeFromString(Twitter.serializer(), input) + private val jsonImplicitNulls = Json { explicitNulls = false } + @Setup fun init() { require(twitter == Json.decodeFromString(Twitter.serializer(), Json.encodeToString(Twitter.serializer(), twitter))) @@ -34,6 +33,9 @@ open class TwitterBenchmark { @Benchmark fun decodeTwitter() = Json.decodeFromString(Twitter.serializer(), input) + @Benchmark + fun decodeTwitterImplicitNulls() = jsonImplicitNulls.decodeFromString(Twitter.serializer(), input) + @Benchmark fun encodeTwitter() = Json.encodeToString(Twitter.serializer(), twitter) } diff --git a/core/api/kotlinx-serialization-core.api b/core/api/kotlinx-serialization-core.api index a1b80b320..5b8a6bf86 100644 --- a/core/api/kotlinx-serialization-core.api +++ b/core/api/kotlinx-serialization-core.api @@ -369,9 +369,9 @@ public abstract class kotlinx/serialization/encoding/AbstractEncoder : kotlinx/s public final fun encodeLongElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IJ)V public fun encodeNotNullMark ()V public fun encodeNull ()V - public final fun encodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V + public fun encodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V public fun encodeNullableSerializableValue (Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V - public final fun encodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V + public fun encodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V public fun encodeSerializableValue (Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V public fun encodeShort (S)V public final fun encodeShortElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IS)V @@ -629,6 +629,12 @@ public final class kotlinx/serialization/internal/DoubleSerializer : kotlinx/ser public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V } +public final class kotlinx/serialization/internal/ElementMarker { + public fun (Lkotlinx/serialization/descriptors/SerialDescriptor;Lkotlin/jvm/functions/Function2;)V + public final fun mark (I)V + public final fun nextUnmarkedIndex ()I +} + public final class kotlinx/serialization/internal/EnumDescriptor : kotlinx/serialization/internal/PluginGeneratedSerialDescriptor { public fun (Ljava/lang/String;I)V public fun equals (Ljava/lang/Object;)Z @@ -1062,9 +1068,9 @@ public abstract class kotlinx/serialization/internal/TaggedEncoder : kotlinx/ser public final fun encodeLongElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IJ)V public final fun encodeNotNullMark ()V public fun encodeNull ()V - public final fun encodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V + public fun encodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V public fun encodeNullableSerializableValue (Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V - public final fun encodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V + public fun encodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V public fun encodeSerializableValue (Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V public final fun encodeShort (S)V public final fun encodeShortElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IS)V diff --git a/core/commonMain/src/kotlinx/serialization/encoding/AbstractEncoder.kt b/core/commonMain/src/kotlinx/serialization/encoding/AbstractEncoder.kt index 616a759b2..f197c40f4 100644 --- a/core/commonMain/src/kotlinx/serialization/encoding/AbstractEncoder.kt +++ b/core/commonMain/src/kotlinx/serialization/encoding/AbstractEncoder.kt @@ -70,7 +70,7 @@ public abstract class AbstractEncoder : Encoder, CompositeEncoder { ): Encoder = if (encodeElement(descriptor, index)) encodeInline(descriptor.getElementDescriptor(index)) else NoOpEncoder - final override fun encodeSerializableElement( + override fun encodeSerializableElement( descriptor: SerialDescriptor, index: Int, serializer: SerializationStrategy, @@ -80,7 +80,7 @@ public abstract class AbstractEncoder : Encoder, CompositeEncoder { encodeSerializableValue(serializer, value) } - final override fun encodeNullableSerializableElement( + override fun encodeNullableSerializableElement( descriptor: SerialDescriptor, index: Int, serializer: SerializationStrategy, diff --git a/core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt b/core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt new file mode 100644 index 000000000..91b268f13 --- /dev/null +++ b/core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt @@ -0,0 +1,116 @@ +/* + * Copyright 2017-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.serialization.internal + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.CompositeDecoder +import kotlin.jvm.JvmStatic + +@OptIn(ExperimentalSerializationApi::class) +@PublishedApi +internal class ElementMarker( + private val descriptor: SerialDescriptor, + private val readIfAbsent: (SerialDescriptor, Int) -> Boolean +) { + /* + * Element decoding marks from given bytes. + * The element number is the same as the bit position. + * Marks for the lowest 64 elements are always stored in a single Long value, higher elements stores in long array. + */ + private var lowerMarks: Long + private val highMarksArray: LongArray + + private companion object { + private val EMPTY_HIGH_MARKS = LongArray(0) + } + + init { + val elementsCount = descriptor.elementsCount + if (elementsCount <= Long.SIZE_BITS) { + lowerMarks = if (elementsCount == Long.SIZE_BITS) { + // number of bits in the mark is equal to the number of fields + 0L + } else { + // (1 - elementsCount) bits are always 1 since there are no fields for them + -1L shl elementsCount + } + highMarksArray = EMPTY_HIGH_MARKS + } else { + lowerMarks = 0L + highMarksArray = prepareHighMarksArray(elementsCount) + } + } + + fun mark(index: Int) { + if (index < Long.SIZE_BITS) { + lowerMarks = lowerMarks or (1L shl index) + } else { + markHigh(index) + } + } + + fun nextUnmarkedIndex(): Int { + val elementsCount = descriptor.elementsCount + while (lowerMarks != -1L) { + val index = lowerMarks.inv().countTrailingZeroBits() + lowerMarks = lowerMarks or (1L shl index) + + if (readIfAbsent(descriptor, index)) { + return index + } + } + + if (elementsCount > Long.SIZE_BITS) { + return nextUnmarkedHighIndex() + } + return CompositeDecoder.DECODE_DONE + } + + private fun prepareHighMarksArray(elementsCount: Int): LongArray { + // (elementsCount - 1) / Long.SIZE_BITS + // (elementsCount - 1) because only one Long value is needed to store 64 fields etc + val slotsCount = (elementsCount - 1) ushr 6 + // elementsCount % Long.SIZE_BITS + val elementsInLastSlot = elementsCount and (Long.SIZE_BITS - 1) + val highMarks = LongArray(slotsCount) + // if (elementsCount % Long.SIZE_BITS) == 0 means that the fields occupy all bits in mark + if (elementsInLastSlot != 0) { + // all marks except the higher are always 0 + highMarks[highMarks.lastIndex] = -1L shl elementsCount + } + return highMarks + } + + private fun markHigh(index: Int) { + // (index / Long.SIZE_BITS) - 1 + val slot = (index ushr 6) - 1 + // index % Long.SIZE_BITS + val offsetInSlot = index and (Long.SIZE_BITS - 1) + highMarksArray[slot] = highMarksArray[slot] or (1L shl offsetInSlot) + } + + private fun nextUnmarkedHighIndex(): Int { + for (slot in highMarksArray.indices) { + // (slot + 1) because first element in high marks has index 64 + val slotOffset = (slot + 1) * Long.SIZE_BITS + // store in a variable so as not to frequently use the array + var slotMarks = highMarksArray[slot] + + while (slotMarks != -1L) { + val indexInSlot = slotMarks.inv().countTrailingZeroBits() + slotMarks = slotMarks or (1L shl indexInSlot) + + val index = slotOffset + indexInSlot + if (readIfAbsent(descriptor, index)) { + highMarksArray[slot] = slotMarks + return index + } + } + highMarksArray[slot] = slotMarks + } + return CompositeDecoder.DECODE_DONE + } +} diff --git a/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt b/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt index 8f00b4ab0..e2e8d8f96 100644 --- a/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt +++ b/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt @@ -126,7 +126,7 @@ public abstract class TaggedEncoder : Encoder, CompositeEncoder { return encodeTaggedInline(descriptor.getTag(index), descriptor.getElementDescriptor(index)) } - final override fun encodeSerializableElement( + override fun encodeSerializableElement( descriptor: SerialDescriptor, index: Int, serializer: SerializationStrategy, @@ -137,7 +137,7 @@ public abstract class TaggedEncoder : Encoder, CompositeEncoder { } @OptIn(ExperimentalSerializationApi::class) - final override fun encodeNullableSerializableElement( + override fun encodeNullableSerializableElement( descriptor: SerialDescriptor, index: Int, serializer: SerializationStrategy, diff --git a/core/commonTest/src/kotlinx/serialization/ElementMarkerTest.kt b/core/commonTest/src/kotlinx/serialization/ElementMarkerTest.kt new file mode 100644 index 000000000..a22be3ffb --- /dev/null +++ b/core/commonTest/src/kotlinx/serialization/ElementMarkerTest.kt @@ -0,0 +1,97 @@ +package kotlinx.serialization + +import kotlinx.serialization.descriptors.* +import kotlinx.serialization.encoding.CompositeDecoder +import kotlinx.serialization.internal.ElementMarker +import kotlin.test.Test +import kotlin.test.assertEquals + +class ElementMarkerTest { + @Test + fun testNothingWasRead() { + val size = 5 + val descriptor = createClassDescriptor(size) + val reader = ElementMarker(descriptor) { _, _ -> true } + + for (i in 0 until size) { + assertEquals(i, reader.nextUnmarkedIndex()) + } + assertEquals(CompositeDecoder.DECODE_DONE, reader.nextUnmarkedIndex()) + } + + @Test + fun testAllWasRead() { + val size = 5 + val descriptor = createClassDescriptor(size) + val reader = ElementMarker(descriptor) { _, _ -> true } + for (i in 0 until size) { + reader.mark(i) + } + + assertEquals(CompositeDecoder.DECODE_DONE, reader.nextUnmarkedIndex()) + } + + @Test + fun testFilteredRead() { + val size = 10 + val readIndex = 4 + + val predicate: (Any?, Int) -> Boolean = { _, i -> i % 2 == 0 } + val descriptor = createClassDescriptor(size) + val reader = ElementMarker(descriptor, predicate) + reader.mark(readIndex) + + for (i in 0 until size) { + if (predicate(descriptor, i) && i != readIndex) { + //`readIndex` already read and only filtered elements must be read + assertEquals(i, reader.nextUnmarkedIndex()) + } + } + assertEquals(CompositeDecoder.DECODE_DONE, reader.nextUnmarkedIndex()) + } + + @Test + fun testSmallPartiallyRead() { + testPartiallyRead(Long.SIZE_BITS / 3) + } + + @Test + fun test64PartiallyRead() { + testPartiallyRead(Long.SIZE_BITS) + } + + @Test + fun test128PartiallyRead() { + testPartiallyRead(Long.SIZE_BITS * 2) + } + + @Test + fun testLargePartiallyRead() { + testPartiallyRead(Long.SIZE_BITS * 2 + Long.SIZE_BITS / 3) + } + + private fun testPartiallyRead(size: Int) { + val descriptor = createClassDescriptor(size) + val reader = ElementMarker(descriptor) { _, _ -> true } + for (i in 0 until size) { + if (i % 2 == 0) { + reader.mark(i) + } + } + + for (i in 0 until size) { + if (i % 2 != 0) { + assertEquals(i, reader.nextUnmarkedIndex()) + } + } + assertEquals(CompositeDecoder.DECODE_DONE, reader.nextUnmarkedIndex()) + } + + private fun createClassDescriptor(size: Int): SerialDescriptor { + return buildClassSerialDescriptor("descriptor") { + for (i in 0 until size) { + element("element$i", buildSerialDescriptor("int", PrimitiveKind.INT)) + } + } + } +} diff --git a/formats/json/api/kotlinx-serialization-json.api b/formats/json/api/kotlinx-serialization-json.api index 1797d5974..dbee022e1 100644 --- a/formats/json/api/kotlinx-serialization-json.api +++ b/formats/json/api/kotlinx-serialization-json.api @@ -80,6 +80,7 @@ public final class kotlinx/serialization/json/JsonBuilder { public final fun getClassDiscriminator ()Ljava/lang/String; public final fun getCoerceInputValues ()Z public final fun getEncodeDefaults ()Z + public final fun getExplicitNulls ()Z public final fun getIgnoreUnknownKeys ()Z public final fun getPrettyPrint ()Z public final fun getPrettyPrintIndent ()Ljava/lang/String; @@ -92,6 +93,7 @@ public final class kotlinx/serialization/json/JsonBuilder { public final fun setClassDiscriminator (Ljava/lang/String;)V public final fun setCoerceInputValues (Z)V public final fun setEncodeDefaults (Z)V + public final fun setExplicitNulls (Z)V public final fun setIgnoreUnknownKeys (Z)V public final fun setLenient (Z)V public final fun setPrettyPrint (Z)V @@ -108,6 +110,7 @@ public final class kotlinx/serialization/json/JsonConfiguration { public final fun getClassDiscriminator ()Ljava/lang/String; public final fun getCoerceInputValues ()Z public final fun getEncodeDefaults ()Z + public final fun getExplicitNulls ()Z public final fun getIgnoreUnknownKeys ()Z public final fun getPrettyPrint ()Z public final fun getPrettyPrintIndent ()Ljava/lang/String; diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt b/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt index 7ef3f40ba..ba6427912 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt @@ -96,7 +96,7 @@ public sealed class Json( */ public final override fun decodeFromString(deserializer: DeserializationStrategy, string: String): T { val lexer = JsonLexer(string) - val input = StreamingJsonDecoder(this, WriteMode.OBJ, lexer) + val input = StreamingJsonDecoder(this, WriteMode.OBJ, lexer, deserializer.descriptor) val result = input.decodeSerializableValue(deserializer) lexer.expectEof() return result @@ -170,6 +170,17 @@ public class JsonBuilder internal constructor(json: Json) { */ public var encodeDefaults: Boolean = json.configuration.encodeDefaults + /** + * Specifies whether `null` values should be encoded for nullable properties and must be present in JSON object + * during decoding. + * + * When this flag is disabled properties with `null` values without default are not encoded; + * during decoding, the absence of a field value is treated as `null` for nullable properties without a default value. + * + * `true` by default. + */ + public var explicitNulls: Boolean = json.configuration.explicitNulls + /** * Specifies whether encounters of unknown properties in the input JSON * should be ignored instead of throwing [SerializationException]. @@ -275,7 +286,7 @@ public class JsonBuilder internal constructor(json: Json) { return JsonConfiguration( encodeDefaults, ignoreUnknownKeys, isLenient, - allowStructuredMapKeys, prettyPrint, prettyPrintIndent, + allowStructuredMapKeys, prettyPrint, explicitNulls, prettyPrintIndent, coerceInputValues, useArrayPolymorphism, classDiscriminator, allowSpecialFloatingPointValues, useAlternativeNames ) diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt b/formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt index c6a87ebea..a764d8f25 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt @@ -1,7 +1,6 @@ package kotlinx.serialization.json import kotlinx.serialization.* -import kotlinx.serialization.modules.* /** * Configuration of the current [Json] instance available through [Json.configuration] @@ -23,6 +22,8 @@ public class JsonConfiguration internal constructor( public val allowStructuredMapKeys: Boolean = false, public val prettyPrint: Boolean = false, @ExperimentalSerializationApi + public val explicitNulls: Boolean = true, + @ExperimentalSerializationApi public val prettyPrintIndent: String = " ", public val coerceInputValues: Boolean = false, public val useArrayPolymorphism: Boolean = false, @@ -33,6 +34,6 @@ public class JsonConfiguration internal constructor( /** @suppress Dokka **/ override fun toString(): String { - return "JsonConfiguration(encodeDefaults=$encodeDefaults, ignoreUnknownKeys=$ignoreUnknownKeys, isLenient=$isLenient, allowStructuredMapKeys=$allowStructuredMapKeys, prettyPrint=$prettyPrint, prettyPrintIndent='$prettyPrintIndent', coerceInputValues=$coerceInputValues, useArrayPolymorphism=$useArrayPolymorphism, classDiscriminator='$classDiscriminator', allowSpecialFloatingPointValues=$allowSpecialFloatingPointValues)" + return "JsonConfiguration(encodeDefaults=$encodeDefaults, ignoreUnknownKeys=$ignoreUnknownKeys, isLenient=$isLenient, allowStructuredMapKeys=$allowStructuredMapKeys, prettyPrint=$prettyPrint, explicitNulls=$explicitNulls, prettyPrintIndent='$prettyPrintIndent', coerceInputValues=$coerceInputValues, useArrayPolymorphism=$useArrayPolymorphism, classDiscriminator='$classDiscriminator', allowSpecialFloatingPointValues=$allowSpecialFloatingPointValues)" } } diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonElementMarker.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonElementMarker.kt new file mode 100644 index 000000000..2535739cf --- /dev/null +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonElementMarker.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2017-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") + +package kotlinx.serialization.json.internal + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.internal.ElementMarker + +@OptIn(ExperimentalSerializationApi::class) +internal class JsonElementMarker(descriptor: SerialDescriptor) { + private val origin: ElementMarker = ElementMarker(descriptor, ::readIfAbsent) + + internal var isUnmarkedNull: Boolean = false + private set + + internal fun mark(index: Int) { + origin.mark(index) + } + + internal fun nextUnmarkedIndex(): Int { + return origin.nextUnmarkedIndex() + } + + private fun readIfAbsent(descriptor: SerialDescriptor, index: Int): Boolean { + isUnmarkedNull = !descriptor.isElementOptional(index) && descriptor.getElementDescriptor(index).isNullable + return isUnmarkedNull + } +} diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt index cec067581..a0ec61ad6 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt @@ -19,13 +19,16 @@ import kotlin.jvm.* internal open class StreamingJsonDecoder( final override val json: Json, private val mode: WriteMode, - @JvmField internal val lexer: JsonLexer + @JvmField internal val lexer: JsonLexer, + descriptor: SerialDescriptor ) : JsonDecoder, AbstractDecoder() { override val serializersModule: SerializersModule = json.serializersModule private var currentIndex = -1 private val configuration = json.configuration + private val elementMarker: JsonElementMarker? = if (configuration.explicitNulls) null else JsonElementMarker(descriptor) + override fun decodeJsonElement(): JsonElement = JsonTreeReader(json.configuration, lexer).read() override fun decodeSerializableValue(deserializer: DeserializationStrategy): T { @@ -41,12 +44,13 @@ internal open class StreamingJsonDecoder( WriteMode.LIST, WriteMode.MAP, WriteMode.POLY_OBJ -> StreamingJsonDecoder( json, newMode, - lexer + lexer, + descriptor ) - else -> if (mode == newMode) { + else -> if (mode == newMode && json.configuration.explicitNulls) { this } else { - StreamingJsonDecoder(json, newMode, lexer) + StreamingJsonDecoder(json, newMode, lexer, descriptor) } } } @@ -56,7 +60,7 @@ internal open class StreamingJsonDecoder( } override fun decodeNotNullMark(): Boolean { - return lexer.tryConsumeNotNull() + return !(elementMarker?.isUnmarkedNull ?: false) && lexer.tryConsumeNotNull() } override fun decodeNull(): Nothing? { @@ -111,6 +115,7 @@ internal open class StreamingJsonDecoder( { lexer.consumeString() /* skip unknown enum string*/ } ) + @Suppress("INVISIBLE_MEMBER") private fun decodeObjectIndex(descriptor: SerialDescriptor): Int { // hasComma checks are required to properly react on trailing commas var hasComma = lexer.tryConsumeComma() @@ -124,6 +129,7 @@ internal open class StreamingJsonDecoder( hasComma = lexer.tryConsumeComma() false // Known element, but coerced } else { + elementMarker?.mark(index) return index // Known element without coercing, return it } } else { @@ -135,7 +141,8 @@ internal open class StreamingJsonDecoder( } } if (hasComma) lexer.fail("Unexpected trailing comma") - return CompositeDecoder.DECODE_DONE + + return elementMarker?.nextUnmarkedIndex() ?: CompositeDecoder.DECODE_DONE } private fun handleUnknown(key: String): Boolean { diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonEncoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonEncoder.kt index 5b68278c2..2693e8e29 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonEncoder.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonEncoder.kt @@ -147,6 +147,17 @@ internal class StreamingJsonEncoder( return true } + override fun encodeNullableSerializableElement( + descriptor: SerialDescriptor, + index: Int, + serializer: SerializationStrategy, + value: T? + ) { + if (value != null || configuration.explicitNulls) { + super.encodeNullableSerializableElement(descriptor, index, serializer, value) + } + } + override fun encodeInline(inlineDescriptor: SerialDescriptor): Encoder = if (inlineDescriptor.isUnsignedNumber) StreamingJsonEncoder( ComposerForUnsignedNumbers( diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt index 78017bb5f..0fde4c996 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt @@ -188,7 +188,7 @@ private open class JsonTreeDecoder( private val polyDescriptor: SerialDescriptor? = null ) : AbstractJsonTreeDecoder(json, value) { private var position = 0 - + private var forceNull: Boolean = false /* * Checks whether JSON has `null` value for non-null property or unknown enum value for enum property */ @@ -199,16 +199,31 @@ private open class JsonTreeDecoder( { (currentElement(tag) as? JsonPrimitive)?.contentOrNull } ) + @Suppress("INVISIBLE_MEMBER") override fun decodeElementIndex(descriptor: SerialDescriptor): Int { while (position < descriptor.elementsCount) { val name = descriptor.getTag(position++) - if (name in value && (!configuration.coerceInputValues || !coerceInputValue(descriptor, position - 1, name))) { - return position - 1 + val index = position - 1 + forceNull = false + if ((name in value || absenceIsNull(descriptor, index)) + && (!configuration.coerceInputValues || !coerceInputValue(descriptor, index, name)) + ) { + return index } } return CompositeDecoder.DECODE_DONE } + private fun absenceIsNull(descriptor: SerialDescriptor, index: Int): Boolean { + forceNull = !json.configuration.explicitNulls + && !descriptor.isElementOptional(index) && descriptor.getElementDescriptor(index).isNullable + return forceNull + } + + override fun decodeNotNullMark(): Boolean { + return !forceNull && super.decodeNotNullMark() + } + override fun elementName(desc: SerialDescriptor, index: Int): String { val mainName = desc.getElementName(index) if (!configuration.useAlternativeNames) return mainName diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt index 53d7e0174..a8df371fb 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt @@ -171,6 +171,17 @@ private open class JsonTreeEncoder( content[key] = element } + override fun encodeNullableSerializableElement( + descriptor: SerialDescriptor, + index: Int, + serializer: SerializationStrategy, + value: T? + ) { + if (value != null || configuration.explicitNulls) { + super.encodeNullableSerializableElement(descriptor, index, serializer, value) + } + } + override fun getCurrent(): JsonElement = JsonObject(content) } diff --git a/formats/json/commonTest/src/kotlinx/serialization/json/AbstractJsonImplicitNullsTest.kt b/formats/json/commonTest/src/kotlinx/serialization/json/AbstractJsonImplicitNullsTest.kt new file mode 100644 index 000000000..a2f4a9dfb --- /dev/null +++ b/formats/json/commonTest/src/kotlinx/serialization/json/AbstractJsonImplicitNullsTest.kt @@ -0,0 +1,151 @@ +package kotlinx.serialization.json + +import kotlinx.serialization.* +import kotlin.test.* + +/* + * Actual testing should be performed in subclasses. + * Subclasses implement serialization and deserialization for different types: serial Kotlin classes, JsonElement, dynamic etc + */ +@Ignore +abstract class AbstractJsonImplicitNullsTest { + @Serializable + data class Nullable( + val f0: Int?, + val f1: Int?, + val f2: Int?, + val f3: Int?, + ) + + @Serializable + data class WithNotNull( + val f0: Int?, + val f1: Int?, + val f2: Int, + ) + + @Serializable + data class WithOptional( + val f0: Int?, + val f1: Int? = 1, + val f2: Int = 2, + ) + + @Serializable + data class Outer(val i: Inner) + + @Serializable + data class Inner(val s1: String?, val s2: String?) + + @Serializable + data class ListWithNullable(val l: List) + + @Serializable + data class MapWithNullable(val m: Map) + + @Serializable + data class NullableList(val l: List?) + + @Serializable + data class NullableMap(val m: Map?) + + + private val format = Json { explicitNulls = false } + + protected abstract fun Json.encode(value: T, serializer: KSerializer): String + + protected abstract fun Json.decode(json: String, serializer: KSerializer): T + + @Test + fun testExplicit() { + val plain = Nullable(null, 10, null, null) + val json = """{"f0":null,"f1":10,"f2":null,"f3":null}""" + + assertEquals(json, Json.encode(plain, Nullable.serializer())) + assertEquals(plain, Json.decode(json, Nullable.serializer())) + } + + @Test + fun testNullable() { + val plain = Nullable(null, 10, null, null) + val json = """{"f1":10}""" + + assertEquals(json, format.encode(plain, Nullable.serializer())) + assertEquals(plain, format.decode(json, Nullable.serializer())) + } + + @Test + fun testMissingNotNull() { + val json = """{"f1":10}""" + + assertFailsWith(SerializationException::class) { + format.decode(json, WithNotNull.serializer()) + } + } + + @Test + fun testDecodeOptional() { + val json = """{}""" + + val decoded = format.decode(json, WithOptional.serializer()) + assertEquals(WithOptional(null), decoded) + } + + + @Test + fun testNestedJsonObject() { + val json = """{"i": {}}""" + + val decoded = format.decode(json, Outer.serializer()) + assertEquals(Outer(Inner(null, null)), decoded) + } + + @Test + fun testListWithNullable() { + val jsonWithNull = """{"l":[null]}""" + val jsonWithEmptyList = """{"l":[]}""" + + val encoded = format.encode(ListWithNullable(listOf(null)), ListWithNullable.serializer()) + assertEquals(jsonWithNull, encoded) + + val decoded = format.decode(jsonWithEmptyList, ListWithNullable.serializer()) + assertEquals(ListWithNullable(emptyList()), decoded) + } + + @Test + fun testMapWithNullable() { + val jsonWithNull = """{"m":{null:null}}""" + val jsonWithQuotedNull = """{"m":{"null":null}}""" + val jsonWithEmptyList = """{"m":{}}""" + + val encoded = format.encode(MapWithNullable(mapOf(null to null)), MapWithNullable.serializer()) + //Json encode map null key as `null:` but other external utilities may encode it as a String `"null":` + assertTrue { listOf(jsonWithNull, jsonWithQuotedNull).contains(encoded) } + + val decoded = format.decode(jsonWithEmptyList, MapWithNullable.serializer()) + assertEquals(MapWithNullable(emptyMap()), decoded) + } + + @Test + fun testNullableList() { + val json = """{}""" + + val encoded = format.encode(NullableList(null), NullableList.serializer()) + assertEquals(json, encoded) + + val decoded = format.decode(json, NullableList.serializer()) + assertEquals(NullableList(null), decoded) + } + + @Test + fun testNullableMap() { + val json = """{}""" + + val encoded = format.encode(NullableMap(null), NullableMap.serializer()) + assertEquals(json, encoded) + + val decoded = format.decode(json, NullableMap.serializer()) + assertEquals(NullableMap(null), decoded) + } + +} diff --git a/formats/json/commonTest/src/kotlinx/serialization/json/JsonImplicitNullsTest.kt b/formats/json/commonTest/src/kotlinx/serialization/json/JsonImplicitNullsTest.kt new file mode 100644 index 000000000..c06c058c6 --- /dev/null +++ b/formats/json/commonTest/src/kotlinx/serialization/json/JsonImplicitNullsTest.kt @@ -0,0 +1,13 @@ +package kotlinx.serialization.json + +import kotlinx.serialization.* + +class JsonImplicitNullsTest: AbstractJsonImplicitNullsTest() { + override fun Json.encode(value: T, serializer: KSerializer): String { + return encodeToString(serializer, value) + } + + override fun Json.decode(json: String, serializer: KSerializer): T { + return decodeFromString(serializer, json) + } +} diff --git a/formats/json/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt b/formats/json/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt index d0ab7821c..6adb71280 100644 --- a/formats/json/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt +++ b/formats/json/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt @@ -38,7 +38,7 @@ abstract class JsonTestBase { decodeFromString(deserializer, source) } else { val lexer = JsonLexer(source) - val input = StreamingJsonDecoder(this, WriteMode.OBJ, lexer) + val input = StreamingJsonDecoder(this, WriteMode.OBJ, lexer, deserializer.descriptor) val tree = input.decodeJsonElement() lexer.expectEof() readJson(tree, deserializer) diff --git a/formats/json/commonTest/src/kotlinx/serialization/json/JsonTreeImplicitNullsTest.kt b/formats/json/commonTest/src/kotlinx/serialization/json/JsonTreeImplicitNullsTest.kt new file mode 100644 index 000000000..995459e3c --- /dev/null +++ b/formats/json/commonTest/src/kotlinx/serialization/json/JsonTreeImplicitNullsTest.kt @@ -0,0 +1,14 @@ +package kotlinx.serialization.json + +import kotlinx.serialization.KSerializer + +class JsonTreeImplicitNullsTest: AbstractJsonImplicitNullsTest() { + override fun Json.encode(value: T, serializer: KSerializer): String { + return encodeToJsonElement(serializer, value).toString() + } + + override fun Json.decode(json: String, serializer: KSerializer): T { + val jsonElement = parseToJsonElement(json) + return decodeFromJsonElement(serializer, jsonElement) + } +} diff --git a/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicDecoders.kt b/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicDecoders.kt index 0e71fb879..a6658c7cb 100644 --- a/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicDecoders.kt +++ b/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicDecoders.kt @@ -41,6 +41,8 @@ private open class DynamicInput( protected val keys: dynamic = js("Object").keys(value ?: js("{}")) protected open val size: Int = keys.length as Int + private var forceNull: Boolean = false + override val serializersModule: SerializersModule get() = json.serializersModule @@ -81,14 +83,23 @@ private open class DynamicInput( override fun decodeElementIndex(descriptor: SerialDescriptor): Int { while (currentPosition < descriptor.elementsCount) { val name = descriptor.getTag(currentPosition++) - if (hasName(name) && (!json.configuration.coerceInputValues || !coerceInputValue(descriptor, currentPosition - 1, name))) - return currentPosition - 1 + val index = currentPosition - 1 + forceNull = false + if ((hasName(name) || absenceIsNull(descriptor, index)) && (!json.configuration.coerceInputValues || !coerceInputValue(descriptor, index, name))) { + return index + } } return CompositeDecoder.DECODE_DONE } private fun hasName(name: String) = value[name] !== undefined + private fun absenceIsNull(descriptor: SerialDescriptor, index: Int): Boolean { + forceNull = !json.configuration.explicitNulls + && !descriptor.isElementOptional(index) && descriptor.getElementDescriptor(index).isNullable + return forceNull + } + override fun elementName(desc: SerialDescriptor, index: Int): String { val mainName = desc.getElementName(index) if (!json.configuration.useAlternativeNames) return mainName @@ -141,6 +152,10 @@ private open class DynamicInput( } override fun decodeTaggedNotNullMark(tag: String): Boolean { + if (forceNull) { + return false + } + val o = getByTag(tag) if (o === undefined) throwMissingTag(tag) @Suppress("SENSELESS_COMPARISON") // null !== undefined ! diff --git a/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicEncoders.kt b/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicEncoders.kt index 633ab35fc..b97858e33 100644 --- a/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicEncoders.kt +++ b/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicEncoders.kt @@ -151,6 +151,17 @@ private class DynamicObjectEncoder( encodeValue(value) } + override fun encodeNullableSerializableElement( + descriptor: SerialDescriptor, + index: Int, + serializer: SerializationStrategy, + value: T? + ) { + if (value != null || json.configuration.explicitNulls) { + super.encodeNullableSerializableElement(descriptor, index, serializer, value) + } + } + override fun encodeJsonElement(element: JsonElement) { encodeSerializableValue(JsonElementSerializer, element) } diff --git a/formats/json/jsTest/src/kotlinx/serialization/json/EncodeToDynamicTest.kt b/formats/json/jsTest/src/kotlinx/serialization/json/EncodeToDynamicTest.kt index 5153f6ba3..1c3c24c70 100644 --- a/formats/json/jsTest/src/kotlinx/serialization/json/EncodeToDynamicTest.kt +++ b/formats/json/jsTest/src/kotlinx/serialization/json/EncodeToDynamicTest.kt @@ -136,27 +136,6 @@ class EncodeToDynamicTest { } } - // todo: this is a test for internal class. Rewrite it after implementing 'omitNulls' JSON flag. - @Test - @Ignore - fun nullsTest() { -// val data = DataWrapper("a string", null) -// -// val serialized = DynamicObjectSerializer( -// Json, -// encodeNullAsUndefined = true -// ).serialize(DataWrapper.serializer(), data) -// assertNull(serialized.d) -// assertFalse(js("""Object.keys(serialized).includes("d")"""), "should omit null properties") -// -// val serializedWithNull = DynamicObjectSerializer( -// Json, -// encodeNullAsUndefined = false -// ).serialize(DataWrapper.serializer(), data) -// assertNull(serializedWithNull.d) -// assertTrue(js("""Object.keys(serializedWithNull).includes("d")"""), "should contain null properties") - } - @Test fun listTest() { assertDynamicForm(listOf(1, 2, 3, 44), serializer = ListSerializer(Int.serializer())) { data, serialized -> diff --git a/formats/json/jsTest/src/kotlinx/serialization/json/JsonDynamicImplicitNullsTest.kt b/formats/json/jsTest/src/kotlinx/serialization/json/JsonDynamicImplicitNullsTest.kt new file mode 100644 index 000000000..1191e3c9b --- /dev/null +++ b/formats/json/jsTest/src/kotlinx/serialization/json/JsonDynamicImplicitNullsTest.kt @@ -0,0 +1,14 @@ +package kotlinx.serialization.json + +import kotlinx.serialization.KSerializer + +class JsonDynamicImplicitNullsTest : AbstractJsonImplicitNullsTest() { + override fun Json.encode(value: T, serializer: KSerializer): String { + return JSON.stringify(encodeToDynamic(serializer, value)) + } + + override fun Json.decode(json: String, serializer: KSerializer): T { + val x: dynamic = JSON.parse(json) + return decodeFromDynamic(serializer, x) + } +} diff --git a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt index 35b024e8e..f47b83dd9 100644 --- a/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt +++ b/formats/protobuf/commonMain/src/kotlinx/serialization/protobuf/internal/ProtobufDecoding.kt @@ -28,46 +28,13 @@ internal open class ProtobufDecoder( private var indexCache: IntArray? = null private var sparseIndexCache: MutableMap? = null - /* - Element decoding marks from given bytes. - The element number is the same as the bit position. - Marks for the lowest 64 elements are always stored in a single Long value, higher elements stores in long array. - */ - private var lowerReadMark: Long = 0 - private val highReadMarks: LongArray? - - private var valueIsNull: Boolean = false + private var nullValue: Boolean = false + private val elementMarker = ElementMarker(descriptor, ::readIfAbsent) init { - highReadMarks = prepareReadMarks(descriptor) populateCache(descriptor) } - private fun prepareReadMarks(descriptor: SerialDescriptor): LongArray? { - val elementsCount = descriptor.elementsCount - return if (elementsCount <= Long.SIZE_BITS) { - lowerReadMark = if (elementsCount == Long.SIZE_BITS) { - // number og bits in the mark is equal to the number of fields - 0 - } else { - // (1 - elementsCount) bits are always 1 since there are no fields for them - -1L shl elementsCount - } - null - } else { - // (elementsCount - 1) because only one Long value is needed to store 64 fields etc - val slotsCount = (elementsCount - 1) / Long.SIZE_BITS - val elementsInLastSlot = elementsCount % Long.SIZE_BITS - val highReadMarks = LongArray(slotsCount) - // (elementsCount % Long.SIZE_BITS) == 0 this means that the fields occupy all bits in mark - if (elementsInLastSlot != 0) { - // all marks except the higher are always 0 - highReadMarks[highReadMarks.lastIndex] = -1L shl elementsCount - } - highReadMarks - } - } - public fun populateCache(descriptor: SerialDescriptor) { val elements = descriptor.elementsCount if (elements < 32) { @@ -247,97 +214,39 @@ internal open class ProtobufDecoder( override fun SerialDescriptor.getTag(index: Int) = extractParameters(index) - private fun findUnreadElementIndex(): Int { - val elementsCount = descriptor.elementsCount - while (lowerReadMark != -1L) { - val index = lowerReadMark.inv().countTrailingZeroBits() - lowerReadMark = lowerReadMark or (1L shl index) - - if (!descriptor.isElementOptional(index)) { - val elementDescriptor = descriptor.getElementDescriptor(index) - val kind = elementDescriptor.kind - if (kind == StructureKind.MAP || kind == StructureKind.LIST) { - return index - } else if (elementDescriptor.isNullable) { - valueIsNull = true - return index - } - } - } - - if (elementsCount > Long.SIZE_BITS) { - val higherMarks = highReadMarks!! - - for (slot in higherMarks.indices) { - // (slot + 1) because first element in high marks has index 64 - val slotOffset = (slot + 1) * Long.SIZE_BITS - // store in a variable so as not to frequently use the array - var mark = higherMarks[slot] - - while (mark != -1L) { - val indexInSlot = mark.inv().countTrailingZeroBits() - mark = mark or (1L shl indexInSlot) - - val index = slotOffset + indexInSlot - if (!descriptor.isElementOptional(index)) { - val elementDescriptor = descriptor.getElementDescriptor(index) - val kind = elementDescriptor.kind - if (kind == StructureKind.MAP || kind == StructureKind.LIST) { - higherMarks[slot] = mark - return index - } else if (elementDescriptor.isNullable) { - higherMarks[slot] = mark - valueIsNull = true - return index - } - } - } - - higherMarks[slot] = mark - } - return -1 - } - return -1 - } - - private fun markElementAsRead(index: Int) { - if (index < Long.SIZE_BITS) { - lowerReadMark = lowerReadMark or (1L shl index) - } else { - val slot = (index / Long.SIZE_BITS) - 1 - val offsetInSlot = index % Long.SIZE_BITS - highReadMarks!![slot] = highReadMarks[slot] or (1L shl offsetInSlot) - } - } - override fun decodeElementIndex(descriptor: SerialDescriptor): Int { while (true) { val protoId = reader.readTag() if (protoId == -1) { // EOF - val absenceIndex = findUnreadElementIndex() - return if (absenceIndex == -1) { - CompositeDecoder.DECODE_DONE - } else { - absenceIndex - } + return elementMarker.nextUnmarkedIndex() } val index = getIndexByTag(protoId) if (index == -1) { // not found reader.skipElement() } else { - markElementAsRead(index) + elementMarker.mark(index) return index } } } override fun decodeNotNullMark(): Boolean { - return if (valueIsNull) { - valueIsNull = false - false - } else { - true + return !nullValue + } + + private fun readIfAbsent(descriptor: SerialDescriptor, index: Int): Boolean { + if (!descriptor.isElementOptional(index)) { + val elementDescriptor = descriptor.getElementDescriptor(index) + val kind = elementDescriptor.kind + if (kind == StructureKind.MAP || kind == StructureKind.LIST) { + nullValue = false + return true + } else if (elementDescriptor.isNullable) { + nullValue = true + return true + } } + return false } } From f1ef6ac77238608dc188c9e707fb6db6f13e52e8 Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Thu, 29 Jul 2021 16:08:06 +0300 Subject: [PATCH 2/2] ~minor comment --- .../src/kotlinx/serialization/internal/ElementMarker.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt b/core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt index 91b268f13..fca902628 100644 --- a/core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt +++ b/core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt @@ -7,17 +7,18 @@ package kotlinx.serialization.internal import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.encoding.CompositeDecoder -import kotlin.jvm.JvmStatic @OptIn(ExperimentalSerializationApi::class) @PublishedApi internal class ElementMarker( private val descriptor: SerialDescriptor, + // Instead of inheritance and virtual function in order to keep cross-module internal modifier via suppresses + // Can be reworked via public + internal api if necessary private val readIfAbsent: (SerialDescriptor, Int) -> Boolean ) { /* * Element decoding marks from given bytes. - * The element number is the same as the bit position. + * The element index is the same as the set bit position. * Marks for the lowest 64 elements are always stored in a single Long value, higher elements stores in long array. */ private var lowerMarks: Long