JSON omit null

benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/ImplicitNullsBenchmark.kt

@@ -0,0 +1,118 @@
+package kotlinx.benchmarks.json
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
+import org.openjdk.jmh.annotations.*
+import java.util.concurrent.TimeUnit
+
+@Warmup(iterations = 5, time = 1)
+@Measurement(iterations = 5, time = 1)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Fork(2)
+open class ImplicitNullsBenchmark {
+
+    @Serializable
+    data class Values(
+        val field0: Int?,
+        val field1: Int?,
+        val field2: Int?,
+        val field3: Int?,
+        val field4: Int?,
+        val field5: Int?,
+        val field6: Int?,
+        val field7: Int?,
+        val field8: Int?,
+        val field9: Int?,
+
+        val field10: Int?,
+        val field11: Int?,
+        val field12: Int?,
+        val field13: Int?,
+        val field14: Int?,
+        val field15: Int?,
+        val field16: Int?,
+        val field17: Int?,
+        val field18: Int?,
+        val field19: Int?,
+
+        val field20: Int?,
+        val field21: Int?,
+        val field22: Int?,
+        val field23: Int?,
+        val field24: Int?,
+        val field25: Int?,
+        val field26: Int?,
+        val field27: Int?,
+        val field28: Int?,
+        val field29: Int?,
+
+        val field30: Int?,
+        val field31: Int?
+    )
+
+
+    private val jsonImplicitNulls = Json { explicitNulls = false }
+
+    private val valueWithNulls = Values(
+        null, null, 2, null, null, null, null, null, null, null,
+        null, null, null, null, 14, null, null, null, null, null,
+        null, null, null, null, null, null, null, null, null, null,
+        null, null
+    )
+
+
+    private val jsonWithNulls = """{"field0":null,"field1":null,"field2":2,"field3":null,"field4":null,"field5":null,
+        |"field6":null,"field7":null,"field8":null,"field9":null,"field10":null,"field11":null,"field12":null,
+        |"field13":null,"field14":14,"field15":null,"field16":null,"field17":null,"field18":null,"field19":null,
+        |"field20":null,"field21":null,"field22":null,"field23":null,"field24":null,"field25":null,"field26":null,
+        |"field27":null,"field28":null,"field29":null,"field30":null,"field31":null}""".trimMargin()
+
+    private val jsonNoNulls = """{"field0":0,"field1":1,"field2":2,"field3":3,"field4":4,"field5":5,
+        |"field6":6,"field7":7,"field8":8,"field9":9,"field10":10,"field11":11,"field12":12,
+        |"field13":13,"field14":14,"field15":15,"field16":16,"field17":17,"field18":18,"field19":19,
+        |"field20":20,"field21":21,"field22":22,"field23":23,"field24":24,"field25":25,"field26":26,
+        |"field27":27,"field28":28,"field29":29,"field30":30,"field31":31}""".trimMargin()
+
+    private val jsonWithAbsence = """{"field2":2, "field14":14}"""
+
+    private val serializer = Values.serializer()
+
+    @Benchmark
+    fun decodeNoNulls() {
+        Json.decodeFromString(serializer, jsonNoNulls)
+    }
+
+    @Benchmark
+    fun decodeNoNullsImplicit() {
+        jsonImplicitNulls.decodeFromString(serializer, jsonNoNulls)
+    }
+
+    @Benchmark
+    fun decodeNulls() {
+        Json.decodeFromString(serializer, jsonWithNulls)
+    }
+
+    @Benchmark
+    fun decodeNullsImplicit() {
+        jsonImplicitNulls.decodeFromString(serializer, jsonWithNulls)
+    }
+
+    @Benchmark
+    fun decodeAbsenceImplicit() {
+        jsonImplicitNulls.decodeFromString(serializer, jsonWithAbsence)
+    }
+
+    @Benchmark
+    fun encodeNulls() {
+        Json.encodeToString(serializer, valueWithNulls)
+    }
+
+    @Benchmark
+    fun encodeNullsImplicit() {
+        jsonImplicitNulls.encodeToString(serializer, valueWithNulls)
+    }
+}

benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterBenchmark.kt

@@ -1,10 +1,7 @@
 package kotlinx.benchmarks.json
 
 import kotlinx.benchmarks.model.*
-import kotlinx.serialization.*
 import kotlinx.serialization.json.*
-import kotlinx.serialization.json.Json.Default.decodeFromString
-import kotlinx.serialization.json.Json.Default.encodeToString
 import org.openjdk.jmh.annotations.*
 import java.util.concurrent.*
 
@@ -25,6 +22,8 @@ open class TwitterBenchmark {
     private val input = TwitterBenchmark::class.java.getResource("/twitter.json").readBytes().decodeToString()
     private val twitter = Json.decodeFromString(Twitter.serializer(), input)
 
+    private val jsonImplicitNulls = Json { explicitNulls = false }
+
     @Setup
     fun init() {
         require(twitter == Json.decodeFromString(Twitter.serializer(), Json.encodeToString(Twitter.serializer(), twitter)))
@@ -34,6 +33,9 @@ open class TwitterBenchmark {
     @Benchmark
     fun decodeTwitter() = Json.decodeFromString(Twitter.serializer(), input)
 
+    @Benchmark
+    fun decodeTwitterImplicitNulls() = jsonImplicitNulls.decodeFromString(Twitter.serializer(), input)
+
     @Benchmark
     fun encodeTwitter() = Json.encodeToString(Twitter.serializer(), twitter)
 }

core/api/kotlinx-serialization-core.api

@@ -369,9 +369,9 @@ public abstract class kotlinx/serialization/encoding/AbstractEncoder : kotlinx/s
 	public final fun encodeLongElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IJ)V
 	public fun encodeNotNullMark ()V
 	public fun encodeNull ()V
-	public final fun encodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
+	public fun encodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
 	public fun encodeNullableSerializableValue (Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
-	public final fun encodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
+	public fun encodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
 	public fun encodeSerializableValue (Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
 	public fun encodeShort (S)V
 	public final fun encodeShortElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IS)V
@@ -629,6 +629,12 @@ public final class kotlinx/serialization/internal/DoubleSerializer : kotlinx/ser
 	public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V
 }
 
+public final class kotlinx/serialization/internal/ElementMarker {
+	public fun <init> (Lkotlinx/serialization/descriptors/SerialDescriptor;Lkotlin/jvm/functions/Function2;)V
+	public final fun mark (I)V
+	public final fun nextUnmarkedIndex ()I
+}
+
 public final class kotlinx/serialization/internal/EnumDescriptor : kotlinx/serialization/internal/PluginGeneratedSerialDescriptor {
 	public fun <init> (Ljava/lang/String;I)V
 	public fun equals (Ljava/lang/Object;)Z
@@ -1062,9 +1068,9 @@ public abstract class kotlinx/serialization/internal/TaggedEncoder : kotlinx/ser
 	public final fun encodeLongElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IJ)V
 	public final fun encodeNotNullMark ()V
 	public fun encodeNull ()V
-	public final fun encodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
+	public fun encodeNullableSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
 	public fun encodeNullableSerializableValue (Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
-	public final fun encodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
+	public fun encodeSerializableElement (Lkotlinx/serialization/descriptors/SerialDescriptor;ILkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
 	public fun encodeSerializableValue (Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;)V
 	public final fun encodeShort (S)V
 	public final fun encodeShortElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IS)V

core/commonMain/src/kotlinx/serialization/encoding/AbstractEncoder.kt

@@ -70,7 +70,7 @@ public abstract class AbstractEncoder : Encoder, CompositeEncoder {
     ): Encoder =
         if (encodeElement(descriptor, index)) encodeInline(descriptor.getElementDescriptor(index)) else NoOpEncoder
 
-    final override fun <T : Any?> encodeSerializableElement(
+    override fun <T : Any?> encodeSerializableElement(
         descriptor: SerialDescriptor,
         index: Int,
         serializer: SerializationStrategy<T>,
@@ -80,7 +80,7 @@ public abstract class AbstractEncoder : Encoder, CompositeEncoder {
             encodeSerializableValue(serializer, value)
     }
 
-    final override fun <T : Any> encodeNullableSerializableElement(
+    override fun <T : Any> encodeNullableSerializableElement(
         descriptor: SerialDescriptor,
         index: Int,
         serializer: SerializationStrategy<T>,

core/commonMain/src/kotlinx/serialization/internal/ElementMarker.kt

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2017-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.internal
+
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.CompositeDecoder
+
+@OptIn(ExperimentalSerializationApi::class)
+@PublishedApi
+internal class ElementMarker(
+    private val descriptor: SerialDescriptor,
+    // Instead of inheritance and virtual function in order to keep cross-module internal modifier via suppresses
+    // Can be reworked via public + internal api if necessary
+    private val readIfAbsent: (SerialDescriptor, Int) -> Boolean
+) {
+    /*
+     * Element decoding marks from given bytes.
+     * The element index is the same as the set bit position.
+     * Marks for the lowest 64 elements are always stored in a single Long value, higher elements stores in long array.
+     */
+    private var lowerMarks: Long
+    private val highMarksArray: LongArray
+
+    private companion object {
+        private val EMPTY_HIGH_MARKS = LongArray(0)
+    }
+
+    init {
+        val elementsCount = descriptor.elementsCount
+        if (elementsCount <= Long.SIZE_BITS) {
+            lowerMarks = if (elementsCount == Long.SIZE_BITS) {
+                // number of bits in the mark is equal to the number of fields
+                0L
+            } else {
+                // (1 - elementsCount) bits are always 1 since there are no fields for them
+                -1L shl elementsCount
+            }
+            highMarksArray = EMPTY_HIGH_MARKS
+        } else {
+            lowerMarks = 0L
+            highMarksArray = prepareHighMarksArray(elementsCount)
+        }
+    }
+
+    fun mark(index: Int) {
+        if (index < Long.SIZE_BITS) {
+            lowerMarks = lowerMarks or (1L shl index)
+        } else {
+            markHigh(index)
+        }
+    }
+
+    fun nextUnmarkedIndex(): Int {
+        val elementsCount = descriptor.elementsCount
+        while (lowerMarks != -1L) {
+            val index = lowerMarks.inv().countTrailingZeroBits()
+            lowerMarks = lowerMarks or (1L shl index)
+
+            if (readIfAbsent(descriptor, index)) {
+                return index
+            }
+        }
+
+        if (elementsCount > Long.SIZE_BITS) {
+            return nextUnmarkedHighIndex()
+        }
+        return CompositeDecoder.DECODE_DONE
+    }
+
+    private fun prepareHighMarksArray(elementsCount: Int): LongArray {
+        // (elementsCount - 1) / Long.SIZE_BITS
+        // (elementsCount - 1) because only one Long value is needed to store 64 fields etc
+        val slotsCount = (elementsCount - 1) ushr 6
+        // elementsCount % Long.SIZE_BITS
+        val elementsInLastSlot = elementsCount and (Long.SIZE_BITS - 1)
+        val highMarks = LongArray(slotsCount)
+        // if (elementsCount % Long.SIZE_BITS) == 0 means that the fields occupy all bits in mark
+        if (elementsInLastSlot != 0) {
+            // all marks except the higher are always 0
+            highMarks[highMarks.lastIndex] = -1L shl elementsCount
+        }
+        return highMarks
+    }
+
+    private fun markHigh(index: Int) {
+        // (index / Long.SIZE_BITS) - 1
+        val slot = (index ushr 6) - 1
+        // index % Long.SIZE_BITS
+        val offsetInSlot = index and (Long.SIZE_BITS - 1)
+        highMarksArray[slot] = highMarksArray[slot] or (1L shl offsetInSlot)
+    }
+
+    private fun nextUnmarkedHighIndex(): Int {
+        for (slot in highMarksArray.indices) {
+            // (slot + 1) because first element in high marks has index 64
+            val slotOffset = (slot + 1) * Long.SIZE_BITS
+            // store in a variable so as not to frequently use the array
+            var slotMarks = highMarksArray[slot]
+
+            while (slotMarks != -1L) {
+                val indexInSlot = slotMarks.inv().countTrailingZeroBits()
+                slotMarks = slotMarks or (1L shl indexInSlot)
+
+                val index = slotOffset + indexInSlot
+                if (readIfAbsent(descriptor, index)) {
+                    highMarksArray[slot] = slotMarks
+                    return index
+                }
+            }
+            highMarksArray[slot] = slotMarks
+        }
+        return CompositeDecoder.DECODE_DONE
+    }
+}

core/commonMain/src/kotlinx/serialization/internal/Tagged.kt

@@ -126,7 +126,7 @@ public abstract class TaggedEncoder<Tag : Any?> : Encoder, CompositeEncoder {
         return encodeTaggedInline(descriptor.getTag(index), descriptor.getElementDescriptor(index))
     }
 
-    final override fun <T : Any?> encodeSerializableElement(
+    override fun <T : Any?> encodeSerializableElement(
         descriptor: SerialDescriptor,
         index: Int,
         serializer: SerializationStrategy<T>,
@@ -137,7 +137,7 @@ public abstract class TaggedEncoder<Tag : Any?> : Encoder, CompositeEncoder {
     }
 
     @OptIn(ExperimentalSerializationApi::class)
-    final override fun <T : Any> encodeNullableSerializableElement(
+    override fun <T : Any> encodeNullableSerializableElement(
         descriptor: SerialDescriptor,
         index: Int,
         serializer: SerializationStrategy<T>,