diff --git a/benchmark/build.gradle.kts b/benchmark/build.gradle.kts index 7674ff294..5aeafcf03 100644 --- a/benchmark/build.gradle.kts +++ b/benchmark/build.gradle.kts @@ -30,6 +30,7 @@ tasks.processJmhResources { tasks.jmhJar { archiveBaseName.set("benchmarks") archiveVersion.set("") + archiveClassifier.set("") // benchmarks.jar, not benchmarks-jmh.jar destinationDirectory.set(file("$rootDir")) } @@ -57,8 +58,10 @@ dependencies { implementation(libs.jackson.databind) implementation(libs.jackson.module.kotlin) implementation(libs.okio) + implementation(libs.kotlinx.io) implementation(project(":kotlinx-serialization-core")) implementation(project(":kotlinx-serialization-json")) implementation(project(":kotlinx-serialization-json-okio")) + implementation(project(":kotlinx-serialization-json-io")) implementation(project(":kotlinx-serialization-protobuf")) } diff --git a/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedStreamBenchmark.kt b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedStreamBenchmark.kt index 7411c179f..c1104289e 100644 --- a/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedStreamBenchmark.kt +++ b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedStreamBenchmark.kt @@ -5,14 +5,17 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import kotlinx.benchmarks.model.MacroTwitterFeed import kotlinx.benchmarks.model.MicroTwitterFeed +import kotlinx.io.* import kotlinx.serialization.json.* +import kotlinx.serialization.json.io.* +import kotlinx.serialization.json.okio.* +import okio.* import org.openjdk.jmh.annotations.* import java.io.* -import java.nio.file.Files -import java.nio.file.Path import java.util.concurrent.TimeUnit -import kotlin.io.path.deleteIfExists -import kotlin.io.path.outputStream +import kotlin.io.use +import okio.Buffer as OkioBuffer +import okio.Sink as OkioSink @Warmup(iterations = 7, time = 1) @Measurement(iterations = 7, time = 1) @@ -67,6 +70,25 @@ open class TwitterFeedStreamBenchmark { } } + @Benchmark + fun encodeTwitterOkioStream(): OkioSink { + val b = OkioBuffer() + Json.encodeToBufferedSink(MacroTwitterFeed.serializer(), twitter, b) + return b + } + + @Benchmark + fun encodeTwitterKotlinxIoStream(): Sink { + val b = Buffer() + Json.encodeToSink(MacroTwitterFeed.serializer(), twitter, b) + return b + } + + /** + * While encode* benchmarks use MacroTwitterFeed model to output as many bytes as possible, + * decode* benchmarks use MicroTwitterFeed model to also factor for skipping over unnecessary data. + */ + // Difference with TwitterFeedBenchmark.decodeMicroTwitter shows how heavy Java's standard UTF-8 decoding actually is. @Benchmark fun decodeMicroTwitterReadText(): MicroTwitterFeed { @@ -88,4 +110,24 @@ open class TwitterFeedStreamBenchmark { objectMapper.readValue(it, MicroTwitterFeed::class.java) } } + + @Benchmark + fun decodeMicroTwitterOkioStream(): MicroTwitterFeed { + // It seems there is no optimal way to reuse `bytes` between benchmark, so we are forced + // to write them to buffer every time. + // Note that it makes comparison with Jackson and InputStream integration much less meaningful. + val b = OkioBuffer() + b.write(bytes) + return jsonIgnoreUnknwn.decodeFromBufferedSource(MicroTwitterFeed.serializer(), b) + } + + @Benchmark + fun decodeMicroTwitterKotlinxIoStream(): MicroTwitterFeed { + // It seems there is no way to reuse filled buffer between benchmark iterations, so we are forced + // to write bytes to buffer every time. + // Note that it makes comparison with Jackson and InputStream integration much less meaningful. + val b = Buffer() + b.write(bytes) + return jsonIgnoreUnknwn.decodeFromSource(MicroTwitterFeed.serializer(), b) + } } diff --git a/build.gradle.kts b/build.gradle.kts index 7b11cc652..b19a6f1f6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -184,9 +184,10 @@ val experimentalsInTestEnabled get() = listOf( val documentedSubprojects get() = setOf("kotlinx-serialization-core", "kotlinx-serialization-json", "kotlinx-serialization-json-okio", + "kotlinx-serialization-json-io", "kotlinx-serialization-cbor", "kotlinx-serialization-properties", "kotlinx-serialization-hocon", "kotlinx-serialization-protobuf") -val uncoveredProjects get() = setOf("kotlinx-serialization-bom", "benchmark", "guide", "kotlinx-serialization-json-okio") +val uncoveredProjects get() = setOf("kotlinx-serialization-bom", "benchmark", "guide", "kotlinx-serialization-json-okio", "kotlinx-serialization-json-io") diff --git a/buildSrc/src/main/kotlin/publishing-conventions.gradle.kts b/buildSrc/src/main/kotlin/publishing-conventions.gradle.kts index 0d535ef24..6cfe04117 100644 --- a/buildSrc/src/main/kotlin/publishing-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/publishing-conventions.gradle.kts @@ -16,9 +16,16 @@ plugins { signing } -val isMultiplatform = name in listOf("kotlinx-serialization-core", "kotlinx-serialization-json", "kotlinx-serialization-json-okio", - "kotlinx-serialization-json-tests", "kotlinx-serialization-protobuf", "kotlinx-serialization-cbor", - "kotlinx-serialization-properties") +val isMultiplatform = name in listOf( + "kotlinx-serialization-core", + "kotlinx-serialization-json", + "kotlinx-serialization-json-okio", + "kotlinx-serialization-json-io", + "kotlinx-serialization-json-tests", + "kotlinx-serialization-protobuf", + "kotlinx-serialization-cbor", + "kotlinx-serialization-properties" +) val isBom = name == "kotlinx-serialization-bom" @@ -221,4 +228,4 @@ fun mavenRepositoryUri(): URI { fun Project.getSensitiveProperty(name: String): String? { return project.findProperty(name) as? String ?: System.getenv(name) -} \ No newline at end of file +} diff --git a/dokka/moduledoc.md b/dokka/moduledoc.md index cd0462d4c..6e70b5f11 100644 --- a/dokka/moduledoc.md +++ b/dokka/moduledoc.md @@ -9,6 +9,10 @@ Stable and ready to use JSON format implementation, `JsonElement` API to operate Extensions for kotlinx.serialization.json.Json for integration with the popular [Okio](https://square.github.io/okio/) library. Currently experimental. +# Module kotlinx-serialization-json-io +Extensions for kotlinx.serialization.json.Json for integration with the [kotlinx-io](https://github.com/Kotlin/kotlinx-io) library. +Currently experimental. + # Module kotlinx-serialization-cbor Concise Binary Object Representation (CBOR) format implementation, as per [RFC 7049](https://tools.ietf.org/html/rfc7049). @@ -49,6 +53,9 @@ and JSON-specific serializers. # Package kotlinx.serialization.json.okio Extensions for kotlinx.serialization.json.Json for integration with the popular [Okio](https://square.github.io/okio/) library. +# Package kotlinx.serialization.json.io +Extensions for kotlinx.serialization.json.Json for integration with the [kotlinx-io](https://github.com/Kotlin/kotlinx-io) library. + # Package kotlinx.serialization.protobuf [Protocol buffers](https://protobuf.dev/) serialization format implementation. diff --git a/formats/json-io/api/kotlinx-serialization-json-io.api b/formats/json-io/api/kotlinx-serialization-json-io.api new file mode 100644 index 000000000..4d626a26a --- /dev/null +++ b/formats/json-io/api/kotlinx-serialization-json-io.api @@ -0,0 +1,7 @@ +public final class kotlinx/serialization/json/io/IoStreamsKt { + public static final fun decodeFromSource (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/io/Source;)Ljava/lang/Object; + public static final fun decodeSourceToSequence (Lkotlinx/serialization/json/Json;Lkotlinx/io/Source;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/DecodeSequenceMode;)Lkotlin/sequences/Sequence; + public static synthetic fun decodeSourceToSequence$default (Lkotlinx/serialization/json/Json;Lkotlinx/io/Source;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/DecodeSequenceMode;ILjava/lang/Object;)Lkotlin/sequences/Sequence; + public static final fun encodeToSink (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;Lkotlinx/io/Sink;)V +} + diff --git a/formats/json-io/api/kotlinx-serialization-json-io.klib.api b/formats/json-io/api/kotlinx-serialization-json-io.klib.api new file mode 100644 index 000000000..2a025754c --- /dev/null +++ b/formats/json-io/api/kotlinx-serialization-json-io.klib.api @@ -0,0 +1,14 @@ +// Klib ABI Dump +// Targets: [androidNativeArm32, androidNativeArm64, androidNativeX64, androidNativeX86, iosArm64, iosSimulatorArm64, iosX64, js, linuxArm32Hfp, linuxArm64, linuxX64, macosArm64, macosX64, mingwX64, tvosArm64, tvosSimulatorArm64, tvosX64, wasmJs, wasmWasi, watchosArm32, watchosArm64, watchosDeviceArm64, watchosSimulatorArm64, watchosX64] +// Rendering settings: +// - Signature version: 2 +// - Show manifest properties: true +// - Show declarations: true + +// Library unique name: +final fun <#A: kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/decodeFromSource(kotlinx.serialization/DeserializationStrategy<#A>, kotlinx.io/Source): #A // kotlinx.serialization.json.io/decodeFromSource|decodeFromSource@kotlinx.serialization.json.Json(kotlinx.serialization.DeserializationStrategy<0:0>;kotlinx.io.Source){0§}[0] +final fun <#A: kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/decodeSourceToSequence(kotlinx.io/Source, kotlinx.serialization/DeserializationStrategy<#A>, kotlinx.serialization.json/DecodeSequenceMode = ...): kotlin.sequences/Sequence<#A> // kotlinx.serialization.json.io/decodeSourceToSequence|decodeSourceToSequence@kotlinx.serialization.json.Json(kotlinx.io.Source;kotlinx.serialization.DeserializationStrategy<0:0>;kotlinx.serialization.json.DecodeSequenceMode){0§}[0] +final fun <#A: kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/encodeToSink(kotlinx.serialization/SerializationStrategy<#A>, #A, kotlinx.io/Sink) // kotlinx.serialization.json.io/encodeToSink|encodeToSink@kotlinx.serialization.json.Json(kotlinx.serialization.SerializationStrategy<0:0>;0:0;kotlinx.io.Sink){0§}[0] +final inline fun <#A: reified kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/decodeFromSource(kotlinx.io/Source): #A // kotlinx.serialization.json.io/decodeFromSource|decodeFromSource@kotlinx.serialization.json.Json(kotlinx.io.Source){0§}[0] +final inline fun <#A: reified kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/decodeSourceToSequence(kotlinx.io/Source, kotlinx.serialization.json/DecodeSequenceMode = ...): kotlin.sequences/Sequence<#A> // kotlinx.serialization.json.io/decodeSourceToSequence|decodeSourceToSequence@kotlinx.serialization.json.Json(kotlinx.io.Source;kotlinx.serialization.json.DecodeSequenceMode){0§}[0] +final inline fun <#A: reified kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/encodeToSink(#A, kotlinx.io/Sink) // kotlinx.serialization.json.io/encodeToSink|encodeToSink@kotlinx.serialization.json.Json(0:0;kotlinx.io.Sink){0§}[0] diff --git a/formats/json-io/build.gradle.kts b/formats/json-io/build.gradle.kts new file mode 100644 index 000000000..b7f39210b --- /dev/null +++ b/formats/json-io/build.gradle.kts @@ -0,0 +1,44 @@ +/* + * Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ +import Java9Modularity.configureJava9ModuleInfo +import org.jetbrains.dokka.gradle.* +import java.net.* + +plugins { + kotlin("multiplatform") + kotlin("plugin.serialization") + + id("native-targets-conventions") + id("source-sets-conventions") +} + +kotlin { + sourceSets { + configureEach { + languageSettings { + optIn("kotlinx.serialization.internal.CoreFriendModuleApi") + optIn("kotlinx.serialization.json.internal.JsonFriendModuleApi") + } + } + val commonMain by getting { + dependencies { + api(project(":kotlinx-serialization-core")) + api(project(":kotlinx-serialization-json")) + implementation(libs.kotlinx.io) + } + } + } +} + +project.configureJava9ModuleInfo() + +tasks.named("dokkaHtmlPartial") { + dokkaSourceSets { + configureEach { + externalDocumentationLink { + url.set(URL("https://kotlin.github.io/kotlinx-io/")) + } + } + } +} diff --git a/formats/json-io/commonMain/src/kotlinx/serialization/json/io/IoStreams.kt b/formats/json-io/commonMain/src/kotlinx/serialization/json/io/IoStreams.kt new file mode 100644 index 000000000..2b42db823 --- /dev/null +++ b/formats/json-io/commonMain/src/kotlinx/serialization/json/io/IoStreams.kt @@ -0,0 +1,124 @@ +/* + * Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.serialization.json.io + +import kotlinx.serialization.* +import kotlinx.serialization.json.DecodeSequenceMode +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.internal.* +import kotlinx.serialization.json.io.internal.JsonToIoStreamWriter +import kotlinx.serialization.json.internal.decodeToSequenceByReader +import kotlinx.serialization.json.io.internal.IoSerialReader +import kotlinx.io.* + +/** + * Serializes the [value] with [serializer] into a [sink] using JSON format and UTF-8 encoding. + * + * @throws [SerializationException] if the given value cannot be serialized to JSON. + * @throws [kotlinx.io.IOException] If an I/O error occurs and sink can't be written to. + */ +@ExperimentalSerializationApi +public fun Json.encodeToSink( + serializer: SerializationStrategy, + value: T, + sink: Sink +) { + val writer = JsonToIoStreamWriter(sink) + try { + encodeByWriter(this, writer, serializer, value) + } finally { + writer.release() + } +} + +/** + * Serializes given [value] to a [sink] using UTF-8 encoding and serializer retrieved from the reified type parameter. + * + * @throws [SerializationException] if the given value cannot be serialized to JSON. + * @throws [kotlinx.io.IOException] If an I/O error occurs and sink can't be written to. + */ +@ExperimentalSerializationApi +public inline fun Json.encodeToSink( + value: T, + sink: Sink +): Unit = encodeToSink(serializersModule.serializer(), value, sink) + + +/** + * Deserializes JSON from [source] using UTF-8 encoding to a value of type [T] using [deserializer]. + * + * Note that this functions expects that exactly one object would be present in the source + * and throws an exception if there are any dangling bytes after an object. + * + * @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T]. + * @throws [kotlinx.io.IOException] If an I/O error occurs and source can't be read from. + */ +@ExperimentalSerializationApi +public fun Json.decodeFromSource( + deserializer: DeserializationStrategy, + source: Source +): T { + return decodeByReader(this, deserializer, IoSerialReader(source)) +} + +/** + * Deserializes the contents of given [source] to the value of type [T] using UTF-8 encoding and + * deserializer retrieved from the reified type parameter. + * + * Note that this functions expects that exactly one object would be present in the stream + * and throws an exception if there are any dangling bytes after an object. + * + * @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T]. + * @throws [kotlinx.io.IOException] If an I/O error occurs and source can't be read from. + */ +@ExperimentalSerializationApi +public inline fun Json.decodeFromSource(source: Source): T = + decodeFromSource(serializersModule.serializer(), source) + + +/** + * Transforms the given [source] into lazily deserialized sequence of elements of type [T] using UTF-8 encoding and [deserializer]. + * Unlike [decodeFromSource], [source] is allowed to have more than one element, separated as [format] declares. + * + * Elements must all be of type [T]. + * Elements are parsed lazily when resulting [Sequence] is evaluated. + * Resulting sequence is tied to the stream and can be evaluated only once. + * + * **Resource caution:** this method neither closes the [source] when the parsing is finished nor provides a method to close it manually. + * It is a caller responsibility to hold a reference to a source and close it. Moreover, because source is parsed lazily, + * closing it before returned sequence is evaluated completely will result in [Exception] from decoder. + * + * @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T]. + * @throws [kotlinx.io.IOException] If an I/O error occurs and source can't be read from. + */ +@ExperimentalSerializationApi +public fun Json.decodeSourceToSequence( + source: Source, + deserializer: DeserializationStrategy, + format: DecodeSequenceMode = DecodeSequenceMode.AUTO_DETECT +): Sequence { + return decodeToSequenceByReader(this, IoSerialReader(source), deserializer, format) +} + +/** + * Transforms the given [source] into lazily deserialized sequence of elements of type [T] using UTF-8 encoding and deserializer retrieved from the reified type parameter. + * Unlike [decodeSourceToSequence], [source] is allowed to have more than one element, separated as [format] declares. + * + * Elements must all be of type [T]. + * Elements are parsed lazily when resulting [Sequence] is evaluated. + * Resulting sequence is tied to the stream and constrained to be evaluated only once. + * + * **Resource caution:** this method does not close [source] when the parsing is finished neither provides method to close it manually. + * It is a caller responsibility to hold a reference to a source and close it. Moreover, because source is parsed lazily, + * closing it before returned sequence is evaluated fully would result in [Exception] from decoder. + * + * @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T]. + * @throws [kotlinx.io.IOException] If an I/O error occurs and source can't be read from. + */ +@ExperimentalSerializationApi +public inline fun Json.decodeSourceToSequence( + source: Source, + format: DecodeSequenceMode = DecodeSequenceMode.AUTO_DETECT +): Sequence = decodeSourceToSequence(source, serializersModule.serializer(), format) diff --git a/formats/json-io/commonMain/src/kotlinx/serialization/json/io/internal/IoJsonStreams.kt b/formats/json-io/commonMain/src/kotlinx/serialization/json/io/internal/IoJsonStreams.kt new file mode 100644 index 000000000..fc39affe6 --- /dev/null +++ b/formats/json-io/commonMain/src/kotlinx/serialization/json/io/internal/IoJsonStreams.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.serialization.json.io.internal + +import kotlinx.io.* +import kotlinx.serialization.json.internal.* + +private const val QUOTE_CODE = '"'.code + +internal class JsonToIoStreamWriter(private val sink: Sink) : InternalJsonWriter { + + override fun writeLong(value: Long) { + write(value.toString()) + } + + override fun writeChar(char: Char) { + sink.writeCodePointValue(char.code) + } + + override fun write(text: String) { + sink.writeString(text) + } + + override fun writeQuoted(text: String) { + sink.writeCodePointValue(QUOTE_CODE) + InternalJsonWriter.doWriteEscaping(text) { s, start, end -> sink.writeString(s, start, end) } + sink.writeCodePointValue(QUOTE_CODE) + } + + override fun release() { + // no-op, see https://github.com/Kotlin/kotlinx.serialization/pull/1982#discussion_r915043700 + } +} + +internal class IoSerialReader(private val source: Source): InternalJsonReaderCodePointImpl() { + override fun exhausted(): Boolean = source.exhausted() + override fun nextCodePoint(): Int = source.readCodePointValue() +} diff --git a/formats/json-okio/build.gradle.kts b/formats/json-okio/build.gradle.kts index e0e0d445c..6e49940fb 100644 --- a/formats/json-okio/build.gradle.kts +++ b/formats/json-okio/build.gradle.kts @@ -29,11 +29,6 @@ kotlin { implementation(libs.okio) } } - val commonTest by getting { - dependencies { - implementation(libs.okio) - } - } } } diff --git a/formats/json-okio/commonMain/src/kotlinx/serialization/json/okio/internal/OkioJsonStreams.kt b/formats/json-okio/commonMain/src/kotlinx/serialization/json/okio/internal/OkioJsonStreams.kt index 1de897137..a4e433b35 100644 --- a/formats/json-okio/commonMain/src/kotlinx/serialization/json/okio/internal/OkioJsonStreams.kt +++ b/formats/json-okio/commonMain/src/kotlinx/serialization/json/okio/internal/OkioJsonStreams.kt @@ -7,32 +7,7 @@ package kotlinx.serialization.json.okio.internal import kotlinx.serialization.json.internal.* import okio.* -// Copied from kotlinx/serialization/json/internal/StringOps.kt -private fun toHexChar(i: Int) : Char { - val d = i and 0xf - return if (d < 10) (d + '0'.code).toChar() - else (d - 10 + 'a'.code).toChar() -} - -// Copied from kotlinx/serialization/json/internal/StringOps.kt -private val ESCAPE_STRINGS: Array = arrayOfNulls(93).apply { - for (c in 0..0x1f) { - val c1 = toHexChar(c shr 12) - val c2 = toHexChar(c shr 8) - val c3 = toHexChar(c shr 4) - val c4 = toHexChar(c) - this[c] = "\\u$c1$c2$c3$c4" - } - this['"'.code] = "\\\"" - this['\\'.code] = "\\\\" - this['\t'.code] = "\\t" - this['\b'.code] = "\\b" - this['\n'.code] = "\\n" - this['\r'.code] = "\\r" - this[0x0c] = "\\f" -} - - +private const val QUOTE_CODE = '"'.code internal class JsonToOkioStreamWriter(private val sink: BufferedSink) : InternalJsonWriter { override fun writeLong(value: Long) { @@ -48,20 +23,9 @@ internal class JsonToOkioStreamWriter(private val sink: BufferedSink) : Internal } override fun writeQuoted(text: String) { - sink.writeUtf8CodePoint('"'.code) - var lastPos = 0 - for (i in text.indices) { - val c = text[i].code - if (c < ESCAPE_STRINGS.size && ESCAPE_STRINGS[c] != null) { - sink.writeUtf8(text, lastPos, i) // flush prev - sink.writeUtf8(ESCAPE_STRINGS[c]!!) - lastPos = i + 1 - } - } - - if (lastPos != 0) sink.writeUtf8(text, lastPos, text.length) - else sink.writeUtf8(text) - sink.writeUtf8CodePoint('"'.code) + sink.writeUtf8CodePoint(QUOTE_CODE) + InternalJsonWriter.doWriteEscaping(text) { s, start, end -> sink.writeUtf8(s, start, end) } + sink.writeUtf8CodePoint(QUOTE_CODE) } override fun release() { @@ -69,56 +33,8 @@ internal class JsonToOkioStreamWriter(private val sink: BufferedSink) : Internal } } -// Max value for a code point placed in one Char -private const val SINGLE_CHAR_MAX_CODEPOINT = Char.MAX_VALUE.code -// Value added to the high UTF-16 surrogate after shifting -private const val HIGH_SURROGATE_HEADER = 0xd800 - (0x010000 ushr 10) -// Value added to the low UTF-16 surrogate after masking -private const val LOW_SURROGATE_HEADER = 0xdc00 - - -internal class OkioSerialReader(private val source: BufferedSource): InternalJsonReader { - /* - A sequence of code points is read from UTF-8, some of it can take 2 characters. - In case the last code point requires 2 characters, and the array is already full, we buffer the second character - */ - private var bufferedChar: Char? = null - - override fun read(buffer: CharArray, bufferOffset: Int, count: Int): Int { - var i = 0 - - if (bufferedChar != null) { - buffer[bufferOffset + i] = bufferedChar!! - i++ - bufferedChar = null - } - - while (i < count && !source.exhausted()) { - val codePoint = source.readUtf8CodePoint() - if (codePoint <= SINGLE_CHAR_MAX_CODEPOINT) { - buffer[bufferOffset + i] = codePoint.toChar() - i++ - } else { - // an example of working with surrogates is taken from okio library with minor changes, see https://github.com/square/okio - // UTF-16 high surrogate: 110110xxxxxxxxxx (10 bits) - // UTF-16 low surrogate: 110111yyyyyyyyyy (10 bits) - // Unicode code point: 00010000000000000000 + xxxxxxxxxxyyyyyyyyyy (21 bits) - val upChar = ((codePoint ushr 10) + HIGH_SURROGATE_HEADER).toChar() - val lowChar = ((codePoint and 0x03ff) + LOW_SURROGATE_HEADER).toChar() - - buffer[bufferOffset + i] = upChar - i++ - - if (i < count) { - buffer[bufferOffset + i] = lowChar - i++ - } else { - // if char array is full - buffer lower surrogate - bufferedChar = lowChar - } - } - } - return if (i > 0) i else -1 - } +internal class OkioSerialReader(private val source: BufferedSource): InternalJsonReaderCodePointImpl() { + override fun exhausted(): Boolean = source.exhausted() + override fun nextCodePoint(): Int = source.readUtf8CodePoint() } diff --git a/formats/json-tests/build.gradle.kts b/formats/json-tests/build.gradle.kts index 06fa2da20..53a57f003 100644 --- a/formats/json-tests/build.gradle.kts +++ b/formats/json-tests/build.gradle.kts @@ -34,6 +34,8 @@ kotlin { dependencies { api(project(":kotlinx-serialization-json")) api(project(":kotlinx-serialization-json-okio")) + api(project(":kotlinx-serialization-json-io")) + implementation(libs.kotlinx.io) implementation(libs.okio) } } diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt index 6f3b132ee..de8cfb38b 100644 --- a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt @@ -4,8 +4,10 @@ package kotlinx.serialization.json +import kotlinx.io.* import kotlinx.serialization.* import kotlinx.serialization.json.internal.* +import kotlinx.serialization.json.io.* import kotlinx.serialization.json.okio.decodeFromBufferedSource import kotlinx.serialization.json.okio.encodeToBufferedSink import kotlinx.serialization.modules.EmptySerializersModule @@ -14,13 +16,16 @@ import kotlinx.serialization.test.* import kotlin.test.assertEquals import okio.* import kotlin.test.assertTrue +import kotlinx.io.Buffer as KotlinxIoBuffer +import okio.Buffer as OkioBuffer enum class JsonTestingMode { STREAMING, TREE, OKIO_STREAMS, - JAVA_STREAMS; + JAVA_STREAMS, + KXIO_STREAMS; companion object { fun value(i: Int) = values()[i] @@ -53,10 +58,15 @@ abstract class JsonTestBase { encodeToString(tree) } JsonTestingMode.OKIO_STREAMS -> { - val buffer = Buffer() + val buffer = OkioBuffer() encodeToBufferedSink(serializer, value, buffer) buffer.readUtf8() } + JsonTestingMode.KXIO_STREAMS -> { + val buffer = KotlinxIoBuffer() + encodeToSink(serializer, value, buffer) + buffer.readString() + } } internal inline fun Json.decodeFromString(source: String, jsonTestingMode: JsonTestingMode): T { @@ -81,10 +91,15 @@ abstract class JsonTestBase { readJson(this, tree, deserializer) } JsonTestingMode.OKIO_STREAMS -> { - val buffer = Buffer() + val buffer = OkioBuffer() buffer.writeUtf8(source) decodeFromBufferedSource(deserializer, buffer) } + JsonTestingMode.KXIO_STREAMS -> { + val buffer = KotlinxIoBuffer() + buffer.writeString(source) + decodeFromSource(deserializer, buffer) + } } protected open fun parametrizedTest(test: (JsonTestingMode) -> Unit) { @@ -92,6 +107,7 @@ abstract class JsonTestBase { add(runCatching { test(JsonTestingMode.STREAMING) }) add(runCatching { test(JsonTestingMode.TREE) }) add(runCatching { test(JsonTestingMode.OKIO_STREAMS) }) + add(runCatching { test(JsonTestingMode.KXIO_STREAMS) }) if (isJvm()) { add(runCatching { test(JsonTestingMode.JAVA_STREAMS) }) @@ -117,7 +133,8 @@ abstract class JsonTestBase { val streamingResult = runCatching { SwitchableJson(json, JsonTestingMode.STREAMING).test() } val treeResult = runCatching { SwitchableJson(json, JsonTestingMode.TREE).test() } val okioResult = runCatching { SwitchableJson(json, JsonTestingMode.OKIO_STREAMS).test() } - processResults(listOf(streamingResult, treeResult, okioResult)) + val kxioResult = runCatching { SwitchableJson(json, JsonTestingMode.KXIO_STREAMS).test() } + processResults(listOf(streamingResult, treeResult, okioResult, kxioResult)) } protected fun processResults(results: List>) { diff --git a/formats/json/api/kotlinx-serialization-json.api b/formats/json/api/kotlinx-serialization-json.api index 8082ee36d..4602ad353 100644 --- a/formats/json/api/kotlinx-serialization-json.api +++ b/formats/json/api/kotlinx-serialization-json.api @@ -407,7 +407,15 @@ public abstract interface class kotlinx/serialization/json/internal/InternalJson public abstract fun read ([CII)I } +public abstract class kotlinx/serialization/json/internal/InternalJsonReaderCodePointImpl : kotlinx/serialization/json/internal/InternalJsonReader { + public fun ()V + public abstract fun exhausted ()Z + public abstract fun nextCodePoint ()I + public final fun read ([CII)I +} + public abstract interface class kotlinx/serialization/json/internal/InternalJsonWriter { + public static final field Companion Lkotlinx/serialization/json/internal/InternalJsonWriter$Companion; public abstract fun release ()V public abstract fun write (Ljava/lang/String;)V public abstract fun writeChar (C)V @@ -415,6 +423,10 @@ public abstract interface class kotlinx/serialization/json/internal/InternalJson public abstract fun writeQuoted (Ljava/lang/String;)V } +public final class kotlinx/serialization/json/internal/InternalJsonWriter$Companion { + public final fun doWriteEscaping (Ljava/lang/String;Lkotlin/jvm/functions/Function3;)V +} + public final class kotlinx/serialization/json/internal/JsonStreamsKt { public static final fun decodeByReader (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/internal/InternalJsonReader;)Ljava/lang/Object; public static final fun decodeToSequenceByReader (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/json/internal/InternalJsonReader;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/DecodeSequenceMode;)Lkotlin/sequences/Sequence; @@ -426,6 +438,10 @@ public final class kotlinx/serialization/json/internal/StreamingJsonDecoderKt { public static final fun decodeStringToJsonTree (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/DeserializationStrategy;Ljava/lang/String;)Lkotlinx/serialization/json/JsonElement; } +public final class kotlinx/serialization/json/internal/StringOpsKt { + public static final fun getESCAPE_STRINGS ()[Ljava/lang/String; +} + public final class kotlinx/serialization/json/internal/TreeJsonDecoderKt { public static final fun readJson (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/json/JsonElement;Lkotlinx/serialization/DeserializationStrategy;)Ljava/lang/Object; } diff --git a/formats/json/api/kotlinx-serialization-json.klib.api b/formats/json/api/kotlinx-serialization-json.klib.api index 0839b115a..6026f81f6 100644 --- a/formats/json/api/kotlinx-serialization-json.klib.api +++ b/formats/json/api/kotlinx-serialization-json.klib.api @@ -23,6 +23,12 @@ abstract class <#A: kotlin/Any> kotlinx.serialization.json/JsonTransformingSeria open val descriptor // kotlinx.serialization.json/JsonTransformingSerializer.descriptor|{}descriptor[0] open fun (): kotlinx.serialization.descriptors/SerialDescriptor // kotlinx.serialization.json/JsonTransformingSerializer.descriptor.|(){}[0] } +abstract class kotlinx.serialization.json.internal/InternalJsonReaderCodePointImpl : kotlinx.serialization.json.internal/InternalJsonReader { // kotlinx.serialization.json.internal/InternalJsonReaderCodePointImpl|null[0] + abstract fun exhausted(): kotlin/Boolean // kotlinx.serialization.json.internal/InternalJsonReaderCodePointImpl.exhausted|exhausted(){}[0] + abstract fun nextCodePoint(): kotlin/Int // kotlinx.serialization.json.internal/InternalJsonReaderCodePointImpl.nextCodePoint|nextCodePoint(){}[0] + constructor () // kotlinx.serialization.json.internal/InternalJsonReaderCodePointImpl.|(){}[0] + final fun read(kotlin/CharArray, kotlin/Int, kotlin/Int): kotlin/Int // kotlinx.serialization.json.internal/InternalJsonReaderCodePointImpl.read|read(kotlin.CharArray;kotlin.Int;kotlin.Int){}[0] +} abstract fun interface kotlinx.serialization.json/JsonNamingStrategy { // kotlinx.serialization.json/JsonNamingStrategy|null[0] abstract fun serialNameForJson(kotlinx.serialization.descriptors/SerialDescriptor, kotlin/Int, kotlin/String): kotlin/String // kotlinx.serialization.json/JsonNamingStrategy.serialNameForJson|serialNameForJson(kotlinx.serialization.descriptors.SerialDescriptor;kotlin.Int;kotlin.String){}[0] final object Builtins { // kotlinx.serialization.json/JsonNamingStrategy.Builtins|null[0] @@ -41,6 +47,9 @@ abstract interface kotlinx.serialization.json.internal/InternalJsonWriter { // k abstract fun writeChar(kotlin/Char) // kotlinx.serialization.json.internal/InternalJsonWriter.writeChar|writeChar(kotlin.Char){}[0] abstract fun writeLong(kotlin/Long) // kotlinx.serialization.json.internal/InternalJsonWriter.writeLong|writeLong(kotlin.Long){}[0] abstract fun writeQuoted(kotlin/String) // kotlinx.serialization.json.internal/InternalJsonWriter.writeQuoted|writeQuoted(kotlin.String){}[0] + final object Companion { // kotlinx.serialization.json.internal/InternalJsonWriter.Companion|null[0] + final inline fun doWriteEscaping(kotlin/String, kotlin/Function3) // kotlinx.serialization.json.internal/InternalJsonWriter.Companion.doWriteEscaping|doWriteEscaping(kotlin.String;kotlin.Function3){}[0] + } } abstract interface kotlinx.serialization.json/JsonDecoder : kotlinx.serialization.encoding/CompositeDecoder, kotlinx.serialization.encoding/Decoder { // kotlinx.serialization.json/JsonDecoder|null[0] abstract fun decodeJsonElement(): kotlinx.serialization.json/JsonElement // kotlinx.serialization.json/JsonDecoder.decodeJsonElement|decodeJsonElement(){}[0] @@ -296,6 +305,8 @@ final object kotlinx.serialization.json/JsonPrimitiveSerializer : kotlinx.serial final val descriptor // kotlinx.serialization.json/JsonPrimitiveSerializer.descriptor|{}descriptor[0] final fun (): kotlinx.serialization.descriptors/SerialDescriptor // kotlinx.serialization.json/JsonPrimitiveSerializer.descriptor.|(){}[0] } +final val kotlinx.serialization.json.internal/ESCAPE_STRINGS // kotlinx.serialization.json.internal/ESCAPE_STRINGS|{}ESCAPE_STRINGS[0] + final fun (): kotlin/Array // kotlinx.serialization.json.internal/ESCAPE_STRINGS.|(){}[0] final val kotlinx.serialization.json/boolean // kotlinx.serialization.json/boolean|@kotlinx.serialization.json.JsonPrimitive{}boolean[0] final fun (kotlinx.serialization.json/JsonPrimitive).(): kotlin/Boolean // kotlinx.serialization.json/boolean.|@kotlinx.serialization.json.JsonPrimitive(){}[0] final val kotlinx.serialization.json/booleanOrNull // kotlinx.serialization.json/booleanOrNull|@kotlinx.serialization.json.JsonPrimitive{}booleanOrNull[0] diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonStreams.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonStreams.kt index d2df6da2d..81dd4fa60 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonStreams.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonStreams.kt @@ -14,6 +14,22 @@ public interface InternalJsonWriter { public fun write(text: String) public fun writeQuoted(text: String) public fun release() + + public companion object { + public inline fun doWriteEscaping(text: String, writeImpl: (text: String, startIndex: Int, endIndex: Int) -> Unit) { + var lastPos = 0 + for (i in text.indices) { + val c = text[i].code + if (c < ESCAPE_STRINGS.size && ESCAPE_STRINGS[c] != null) { + writeImpl(text, lastPos, i) // flush prev + val escape = ESCAPE_STRINGS[c]!! + writeImpl(escape, 0, escape.length) + lastPos = i + 1 + } + } + writeImpl(text, lastPos, text.length) + } + } } @JsonFriendModuleApi @@ -21,6 +37,58 @@ public interface InternalJsonReader { public fun read(buffer: CharArray, bufferOffset: Int, count: Int): Int } +// Max value for a code point placed in one Char +private const val SINGLE_CHAR_MAX_CODEPOINT = Char.MAX_VALUE.code +// Value added to the high UTF-16 surrogate after shifting +private const val HIGH_SURROGATE_HEADER = 0xd800 - (0x010000 ushr 10) +// Value added to the low UTF-16 surrogate after masking +private const val LOW_SURROGATE_HEADER = 0xdc00 + +@JsonFriendModuleApi +public abstract class InternalJsonReaderCodePointImpl: InternalJsonReader { + public abstract fun exhausted(): Boolean + public abstract fun nextCodePoint(): Int + + private var bufferedChar: Char? = null + + final override fun read(buffer: CharArray, bufferOffset: Int, count: Int): Int { + var i = 0 + + if (bufferedChar != null) { + buffer[bufferOffset + i] = bufferedChar!! + i++ + bufferedChar = null + } + + while (i < count && !exhausted()) { + val codePoint = nextCodePoint() + if (codePoint <= SINGLE_CHAR_MAX_CODEPOINT) { + buffer[bufferOffset + i] = codePoint.toChar() + i++ + } else { + // an example of working with surrogates is taken from okio library with minor changes, see https://github.com/square/okio + // UTF-16 high surrogate: 110110xxxxxxxxxx (10 bits) + // UTF-16 low surrogate: 110111yyyyyyyyyy (10 bits) + // Unicode code point: 00010000000000000000 + xxxxxxxxxxyyyyyyyyyy (21 bits) + val upChar = ((codePoint ushr 10) + HIGH_SURROGATE_HEADER).toChar() + val lowChar = ((codePoint and 0x03ff) + LOW_SURROGATE_HEADER).toChar() + + buffer[bufferOffset + i] = upChar + i++ + + if (i < count) { + buffer[bufferOffset + i] = lowChar + i++ + } else { + // if char array is full - buffer lower surrogate + bufferedChar = lowChar + } + } + } + return if (i > 0) i else -1 + } +} + @JsonFriendModuleApi public fun encodeByWriter(json: Json, writer: InternalJsonWriter, serializer: SerializationStrategy, value: T) { val encoder = StreamingJsonEncoder( diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StringOps.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StringOps.kt index ed76ba04c..1f3671542 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StringOps.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StringOps.kt @@ -12,6 +12,7 @@ private fun toHexChar(i: Int) : Char { else (d - 10 + 'a'.code).toChar() } +@PublishedApi internal val ESCAPE_STRINGS: Array = arrayOfNulls(93).apply { for (c in 0..0x1f) { val c1 = toHexChar(c shr 12) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e3642e2fe..d6a6d39c2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -13,6 +13,7 @@ guava = "31.1-jre" guava24 = "24.1.1-jre" jackson = "2.13.3" okio = "3.9.0" +kotlinx-io="0.4.0" gson = "2.8.5" kotlintest = "2.0.7" coroutines = "1.6.4" @@ -38,6 +39,7 @@ jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", ver jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson"} jackson-cbor = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", version.ref = "jackson"} okio = { module = "com.squareup.okio:okio", version.ref = "okio"} +kotlinx-io = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io"} gson = { module = "com.google.code.gson:gson", version.ref = "gson"} kotlintest = { module = "io.kotlintest:kotlintest", version.ref = "kotlintest"} coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "coroutines"} diff --git a/settings.gradle.kts b/settings.gradle.kts index ca4e340a9..0d7c75ab0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -55,6 +55,9 @@ project(":kotlinx-serialization-json").projectDir = file("./formats/json") include(":kotlinx-serialization-json-okio") project(":kotlinx-serialization-json-okio").projectDir = file("./formats/json-okio") +include(":kotlinx-serialization-json-io") +project(":kotlinx-serialization-json-io").projectDir = file("./formats/json-io") + include(":kotlinx-serialization-json-tests") project(":kotlinx-serialization-json-tests").projectDir = file("./formats/json-tests") @@ -107,4 +110,4 @@ fun overriddenKotlinVersion(): String? { return trainVersion ?: throw IllegalArgumentException("\"kotlin_snapshot_version\" should be defined when building with snapshot compiler") } return null -} \ No newline at end of file +}