From dedb52e066cef16ccd5c2342c1ca7000e0172d8c Mon Sep 17 00:00:00 2001 From: Leonid Startsev Date: Fri, 25 Nov 2022 15:43:50 +0100 Subject: [PATCH 1/5] Provide support for JsonNamingStrategy to be used in Json for properties' names. Provide basic implementation of SnakeCase strategy Fixes #33 --- .../benchmarks/json/TwitterFeedBenchmark.kt | 25 ++- .../model/MacroTwitterUntailored.kt | 170 ++++++++++++++++ .../src/kotlinx/serialization/Annotations.kt | 3 + .../descriptors/SerialDescriptor.kt | 2 +- .../kotlinx/serialization/internal/Tagged.kt | 2 +- .../JsonNamingStrategyExclusionTest.kt | 60 ++++++ .../features/JsonNamingStrategyTest.kt | 182 ++++++++++++++++++ .../json/JsonNamingStrategyDynamicTest.kt | 39 ++++ .../json/api/kotlinx-serialization-json.api | 16 ++ .../src/kotlinx/serialization/json/Json.kt | 18 +- .../serialization/json/JsonAnnotations.kt | 8 +- .../serialization/json/JsonConfiguration.kt | 9 +- .../serialization/json/JsonNamingStrategy.kt | 107 ++++++++++ .../json/internal/JsonExceptions.kt | 2 +- .../json/internal/JsonNamesMap.kt | 50 +++-- .../json/internal/SchemaCache.kt | 6 +- .../json/internal/StreamingJsonEncoder.kt | 2 +- .../json/internal/TreeJsonDecoder.kt | 50 +++-- .../json/internal/TreeJsonEncoder.kt | 3 + .../json/internal/DynamicDecoders.kt | 35 ++-- .../json/internal/DynamicEncoders.kt | 2 +- 21 files changed, 730 insertions(+), 61 deletions(-) create mode 100644 benchmark/src/jmh/kotlin/kotlinx/benchmarks/model/MacroTwitterUntailored.kt create mode 100644 formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyExclusionTest.kt create mode 100644 formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyTest.kt create mode 100644 formats/json-tests/jsTest/src/kotlinx/serialization/json/JsonNamingStrategyDynamicTest.kt create mode 100644 formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt diff --git a/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedBenchmark.kt b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedBenchmark.kt index e015ad96a..837a8ba33 100644 --- a/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedBenchmark.kt +++ b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedBenchmark.kt @@ -3,8 +3,6 @@ package kotlinx.benchmarks.json import kotlinx.benchmarks.model.* import kotlinx.serialization.* import kotlinx.serialization.json.* -import kotlinx.serialization.json.Json.Default.decodeFromString -import kotlinx.serialization.json.Json.Default.encodeToString import org.openjdk.jmh.annotations.* import java.util.concurrent.* @@ -24,19 +22,25 @@ open class TwitterFeedBenchmark { */ private val input = TwitterFeedBenchmark::class.java.getResource("/twitter_macro.json").readBytes().decodeToString() private val twitter = Json.decodeFromString(MacroTwitterFeed.serializer(), input) + private val jsonNoAltNames = Json { useAlternativeNames = false } private val jsonIgnoreUnknwn = Json { ignoreUnknownKeys = true } - private val jsonIgnoreUnknwnNoAltNames = Json { ignoreUnknownKeys = true; useAlternativeNames = false} + private val jsonIgnoreUnknwnNoAltNames = Json { ignoreUnknownKeys = true; useAlternativeNames = false } + private val jsonNamingStrategy = Json { namingStrategy = JsonNamingStrategy.SnakeCase } + private val jsonNamingStrategyIgnoreUnknwn = Json(jsonNamingStrategy) { ignoreUnknownKeys = true } + + private val twitterKt = jsonNamingStrategy.decodeFromString(MacroTwitterFeedKt.serializer(), input) @Setup fun init() { require(twitter == Json.decodeFromString(MacroTwitterFeed.serializer(), Json.encodeToString(MacroTwitterFeed.serializer(), twitter))) } - // Order of magnitude: ~400 op/s + // Order of magnitude: ~500 op/s @Benchmark fun decodeTwitter() = Json.decodeFromString(MacroTwitterFeed.serializer(), input) + // Should be the same as decodeTwitter, since decodeTwitter never hit unknown name and therefore should never build deserializationNamesMap anyway @Benchmark fun decodeTwitterNoAltNames() = jsonNoAltNames.decodeFromString(MacroTwitterFeed.serializer(), input) @@ -46,7 +50,20 @@ open class TwitterFeedBenchmark { @Benchmark fun decodeMicroTwitter() = jsonIgnoreUnknwn.decodeFromString(MicroTwitterFeed.serializer(), input) + // Should be faster than decodeMicroTwitter, as we explicitly opt-out from deserializationNamesMap on unknown name @Benchmark fun decodeMicroTwitterNoAltNames() = jsonIgnoreUnknwnNoAltNames.decodeFromString(MicroTwitterFeed.serializer(), input) + // Should be just a bit slower than decodeMicroTwitter, because alternative names map is created in both cases + @Benchmark + fun decodeMicroTwitterWithNamingStrategy(): MicroTwitterFeedKt = jsonNamingStrategyIgnoreUnknwn.decodeFromString(MicroTwitterFeedKt.serializer(), input) + + // Can be slower than decodeTwitter, as we always build deserializationNamesMap when naming strategy is used + @Benchmark + fun decodeTwitterWithNamingStrategy(): MacroTwitterFeedKt = jsonNamingStrategy.decodeFromString(MacroTwitterFeedKt.serializer(), input) + + // 15-20% slower than without the strategy. Without serializationNamesMap (invoking strategy on every write), up to 50% slower + @Benchmark + fun encodeTwitterWithNamingStrategy(): String = jsonNamingStrategy.encodeToString(MacroTwitterFeedKt.serializer(), twitterKt) + } diff --git a/benchmark/src/jmh/kotlin/kotlinx/benchmarks/model/MacroTwitterUntailored.kt b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/model/MacroTwitterUntailored.kt new file mode 100644 index 000000000..fca69cc07 --- /dev/null +++ b/benchmark/src/jmh/kotlin/kotlinx/benchmarks/model/MacroTwitterUntailored.kt @@ -0,0 +1,170 @@ +package kotlinx.benchmarks.model + +import kotlinx.serialization.* +import kotlinx.serialization.json.* + +/** + * All model classes are the same as in MacroTwitter.kt but named accordingly to Kotlin naming policies to test JsonNamingStrategy performance. + * Only Size, SizeType and Urls are not copied + */ + +@Serializable +data class MacroTwitterFeedKt( + val statuses: List, + val searchMetadata: SearchMetadata +) + +@Serializable +data class MicroTwitterFeedKt( + val statuses: List +) + +@Serializable +data class TwitterTrimmedStatusKt( + val metadata: MetadataKt, + val createdAt: String, + val id: Long, + val idStr: String, + val text: String, + val source: String, + val truncated: Boolean, + val user: TwitterTrimmedUserKt, + val retweetedStatus: TwitterTrimmedStatusKt? = null, +) + +@Serializable +data class TwitterStatusKt( + val metadata: MetadataKt, + val createdAt: String, + val id: Long, + val idStr: String, + val text: String, + val source: String, + val truncated: Boolean, + val inReplyToStatusId: Long?, + val inReplyToStatusIdStr: String?, + val inReplyToUserId: Long?, + val inReplyToUserIdStr: String?, + val inReplyToScreenName: String?, + val user: TwitterUserKt, + val geo: String?, + val coordinates: String?, + val place: String?, + val contributors: List?, + val retweetedStatus: TwitterStatusKt? = null, + val retweetCount: Int, + val favoriteCount: Int, + val entities: StatusEntitiesKt, + val favorited: Boolean, + val retweeted: Boolean, + val lang: String, + val possiblySensitive: Boolean? = null +) + +@Serializable +data class StatusEntitiesKt( + val hashtags: List, + val symbols: List, + val urls: List, + val userMentions: List, + val media: List? = null +) + +@Serializable +data class TwitterMediaKt( + val id: Long, + val idStr: String, + val url: String, + val mediaUrl: String, + val mediaUrlHttps: String, + val expandedUrl: String, + val displayUrl: String, + val indices: List, + val type: String, + val sizes: SizeType, + val sourceStatusId: Long? = null, + val sourceStatusIdStr: String? = null +) + +@Serializable +data class TwitterUserMentionKt( + val screenName: String, + val name: String, + val id: Long, + val idStr: String, + val indices: List +) + +@Serializable +data class MetadataKt( + val resultType: String, + val isoLanguageCode: String +) + +@Serializable +data class TwitterTrimmedUserKt( + val id: Long, + val idStr: String, + val name: String, + val screenName: String, + val location: String, + val description: String, + val url: String?, + val entities: UserEntitiesKt, + val protected: Boolean, + val followersCount: Int, + val friendsCount: Int, + val listedCount: Int, + val createdAt: String, + val favouritesCount: Int, +) + +@Serializable +data class TwitterUserKt( + val id: Long, + val idStr: String, + val name: String, + val screenName: String, + val location: String, + val description: String, + val url: String?, + val entities: UserEntitiesKt, + val protected: Boolean, + val followersCount: Int, + val friendsCount: Int, + val listedCount: Int, + val createdAt: String, + val favouritesCount: Int, + val utcOffset: Int?, + val timeZone: String?, + val geoEnabled: Boolean, + val verified: Boolean, + val statusesCount: Int, + val lang: String, + val contributorsEnabled: Boolean, + val isTranslator: Boolean, + val isTranslationEnabled: Boolean, + val profileBackgroundColor: String, + val profileBackgroundImageUrl: String, + val profileBackgroundImageUrlHttps: String, + val profileBackgroundTile: Boolean, + val profileImageUrl: String, + val profileImageUrlHttps: String, + val profileBannerUrl: String? = null, + val profileLinkColor: String, + val profileSidebarBorderColor: String, + val profileSidebarFillColor: String, + val profileTextColor: String, + val profileUseBackgroundImage: Boolean, + val defaultProfile: Boolean, + val defaultProfileImage: Boolean, + val following: Boolean, + val followRequestSent: Boolean, + val notifications: Boolean +) + +@Serializable +data class UserEntitiesKt( + val url: Urls? = null, + val description: Urls +) diff --git a/core/commonMain/src/kotlinx/serialization/Annotations.kt b/core/commonMain/src/kotlinx/serialization/Annotations.kt index 146a1ab2b..db8b11aab 100644 --- a/core/commonMain/src/kotlinx/serialization/Annotations.kt +++ b/core/commonMain/src/kotlinx/serialization/Annotations.kt @@ -145,6 +145,9 @@ public annotation class Serializer( * // Prints "{"int":42}" * println(Json.encodeToString(CustomName(42))) * ``` + * + * If a name of class or property is overridden with this annotation, original source code name is not available for the library. + * Tools like `JsonNamingStrategy` and `ProtoBufSchemaGenerator` would see and transform [value] from [SerialName] annotation. */ @Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS) // @Retention(AnnotationRetention.RUNTIME) still runtime, but KT-41082 diff --git a/core/commonMain/src/kotlinx/serialization/descriptors/SerialDescriptor.kt b/core/commonMain/src/kotlinx/serialization/descriptors/SerialDescriptor.kt index e2d0e4c4f..17fdbfe0f 100644 --- a/core/commonMain/src/kotlinx/serialization/descriptors/SerialDescriptor.kt +++ b/core/commonMain/src/kotlinx/serialization/descriptors/SerialDescriptor.kt @@ -259,7 +259,7 @@ public interface SerialDescriptor { public fun getElementDescriptor(index: Int): SerialDescriptor /** - * Whether the element at the given [index] is optional (can be absent is serialized form). + * Whether the element at the given [index] is optional (can be absent in serialized form). * For generated descriptors, all elements that have a corresponding default parameter value are * marked as optional. Custom serializers can treat optional values in a serialization-specific manner * without default parameters constraint. diff --git a/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt b/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt index 9d7dd09f8..8fc5f7cb1 100644 --- a/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt +++ b/core/commonMain/src/kotlinx/serialization/internal/Tagged.kt @@ -331,7 +331,7 @@ public abstract class NamedValueDecoder : TaggedDecoder() { final override fun SerialDescriptor.getTag(index: Int): String = nested(elementName(this, index)) protected fun nested(nestedName: String): String = composeName(currentTagOrNull ?: "", nestedName) - protected open fun elementName(desc: SerialDescriptor, index: Int): String = desc.getElementName(index) + protected open fun elementName(descriptor: SerialDescriptor, index: Int): String = descriptor.getElementName(index) protected open fun composeName(parentName: String, childName: String): String = if (parentName.isEmpty()) childName else "$parentName.$childName" } diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyExclusionTest.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyExclusionTest.kt new file mode 100644 index 000000000..b99745434 --- /dev/null +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyExclusionTest.kt @@ -0,0 +1,60 @@ +package kotlinx.serialization.features + +import kotlinx.serialization.* +import kotlinx.serialization.json.* +import kotlin.test.* + +class JsonNamingStrategyExclusionTest : JsonTestBase() { + @SerialInfo + @Target(AnnotationTarget.CLASS, AnnotationTarget.PROPERTY) + annotation class OriginalSerialName + + private fun List.hasOriginal() = filterIsInstance().isNotEmpty() + + private val myStrategy = JsonNamingStrategy { descriptor, index, serialName -> + if (descriptor.annotations.hasOriginal() || descriptor.getElementAnnotations(index).hasOriginal()) serialName + else JsonNamingStrategy.SnakeCase.serialNameForJson(descriptor, index, serialName) + } + + @Serializable + @OriginalSerialName + data class Foo(val firstArg: String = "a", val secondArg: String = "b") + + enum class E { + @OriginalSerialName + FIRST_E, + SECOND_E + } + + @Serializable + data class Bar( + val firstBar: String = "a", + @OriginalSerialName val secondBar: String = "b", + val fooBar: Foo = Foo(), + val enumBarOne: E = E.FIRST_E, + val enumBarTwo: E = E.SECOND_E + ) + + private fun doTest(json: Json) { + val j = Json(json) { + namingStrategy = myStrategy + } + val bar = Bar() + assertJsonFormAndRestored( + Bar.serializer(), + bar, + """{"first_bar":"a","secondBar":"b","foo_bar":{"firstArg":"a","secondArg":"b"},"enum_bar_one":"FIRST_E","enum_bar_two":"SECOND_E"}""", + j + ) + } + + @Test + fun testJsonNamingStrategyWithAlternativeNames() = doTest(Json(default) { + useAlternativeNames = true + }) + + @Test + fun testJsonNamingStrategyWithoutAlternativeNames() = doTest(Json(default) { + useAlternativeNames = false + }) +} diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyTest.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyTest.kt new file mode 100644 index 000000000..1b9da667e --- /dev/null +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyTest.kt @@ -0,0 +1,182 @@ +package kotlinx.serialization.features + +import kotlinx.serialization.* +import kotlinx.serialization.builtins.* +import kotlinx.serialization.json.* +import kotlinx.serialization.test.* +import kotlin.test.* + + +class JsonNamingStrategyTest : JsonTestBase() { + @Serializable + data class Foo( + val simple: String = "a", + val oneWord: String = "b", + val already_in_snake: String = "c", + val aLotOfWords: String = "d", + val FirstCapitalized: String = "e", + val hasAcronymURL: Bar = Bar.BAZ, + val hasDigit123AndPostfix: Bar = Bar.QUX, + val coercionTest: Bar = Bar.QUX + ) + + enum class Bar { BAZ, QUX } + + val jsonWithNaming = Json(default) { + namingStrategy = JsonNamingStrategy.SnakeCase + } + + @Test + fun testJsonNamingStrategyWithAlternativeNames() = doTest(Json(jsonWithNaming) { + useAlternativeNames = true + }) + + @Test + fun testJsonNamingStrategyWithoutAlternativeNames() = doTest(Json(jsonWithNaming) { + useAlternativeNames = false + }) + + private fun doTest(json: Json) { + val foo = Foo() + assertJsonFormAndRestored( + Foo.serializer(), + foo, + """{"simple":"a","one_word":"b","already_in_snake":"c","a_lot_of_words":"d","first_capitalized":"e","has_acronym_url":"BAZ","has_digit123_and_postfix":"QUX","coercion_test":"QUX"}""", + json + ) + } + + @Test + fun testNamingStrategyWorksWithCoercing() { + val j = Json(jsonWithNaming) { + coerceInputValues = true + useAlternativeNames = false + } + assertEquals( + Foo(), + j.decodeFromString("""{"simple":"a","one_word":"b","already_in_snake":"c","a_lot_of_words":"d","first_capitalized":"e","has_acronym_url":"baz","has_digit123_and_postfix":"qux","coercion_test":"invalid"}""") + ) + } + + @Test + fun testSnakeCaseStrategy() { + fun apply(name: String) = + JsonNamingStrategy.SnakeCase.serialNameForJson(String.serializer().descriptor, 0, name) + + val cases = mapOf( + "" to "", + "_" to "_", + "___" to "___", + "a" to "a", + "A" to "a", + "_1" to "_1", + "_a" to "_a", + "_A" to "_a", + "property" to "property", + "twoWords" to "two_words", + "Oneword" to "oneword", + "camel_Case_Underscores" to "camel_case_underscores", + "_many____underscores__" to "_many____underscores__", + "URLmapping" to "urlmapping", // can be fixed, maybe later + "InURLBetween" to "in_urlbetween", // can be fixed, maybe later + "theWWW" to "the_www", + "theWWW_URL_xxx" to "the_www_url_xxx", + "hasDigit123AndPostfix" to "has_digit123_and_postfix" + ) + + cases.forEach { (input, expected) -> + assertEquals(expected, apply(input)) + } + } + + @Serializable + data class DontUseOriginal(val testCase: String) + + @Test + fun testNamingStrategyOverridesOriginal() { + val json = Json(jsonWithNaming) { + ignoreUnknownKeys = true + } + parametrizedTest { mode -> + assertEquals(DontUseOriginal("a"), json.decodeFromString("""{"test_case":"a","testCase":"b"}""", mode)) + } + + val jsonThrows = Json(jsonWithNaming) { + ignoreUnknownKeys = false + } + parametrizedTest { mode -> + assertFailsWithMessage("Encountered an unknown key 'testCase'") { + jsonThrows.decodeFromString("""{"test_case":"a","testCase":"b"}""", mode) + } + } + } + + @Serializable + data class CollisionCheckPrimary(val testCase: String, val test_case: String) + + @Serializable + data class CollisionCheckAlternate(val testCase: String, @JsonNames("test_case") val testCase2: String) + + @Test + fun testNamingStrategyPrioritizesOverAlternative() = noLegacyJs { // @JsonNames not supported on legacy + val json = Json(jsonWithNaming) { + ignoreUnknownKeys = true + } + parametrizedTest { mode -> + assertFailsWithMessage("The suggested name 'test_case' for property test_case is already one of the names for property testCase") { + json.decodeFromString("""{"test_case":"a"}""", mode) + } + } + parametrizedTest { mode -> + assertFailsWithMessage("The suggested name 'test_case' for property testCase2 is already one of the names for property testCase") { + json.decodeFromString("""{"test_case":"a"}""", mode) + } + } + } + + + @Serializable + data class OriginalAsFallback(@JsonNames("testCase") val testCase: String) + + @Test + fun testCanUseOriginalNameAsAlternative() = noLegacyJs { // @JsonNames not supported on legacy + val json = Json(jsonWithNaming) { + ignoreUnknownKeys = true + } + parametrizedTest { mode -> + assertEquals(OriginalAsFallback("b"), json.decodeFromString("""{"testCase":"b"}""", mode)) + } + } + + @Serializable + sealed interface SealedBase { + @Serializable + @JsonClassDiscriminator("typeSub") + sealed class SealedMid : SealedBase { + @Serializable + @SerialName("SealedSub1") + object SealedSub1 : SealedMid() + } + + @Serializable + @SerialName("SealedSub2") + data class SealedSub2(val testCase: Int = 0) : SealedBase + } + + @Serializable + data class Holder(val testBase: SealedBase, val testMid: SealedBase.SealedMid) + + @Test + fun testNamingStrategyDoesNotAffectPolymorphism() = noLegacyJs { // @JsonClassDiscriminator + val json = Json(jsonWithNaming) { + classDiscriminator = "typeBase" + } + val holder = Holder(SealedBase.SealedSub2(), SealedBase.SealedMid.SealedSub1) + assertJsonFormAndRestored( + Holder.serializer(), + holder, + """{"test_base":{"typeBase":"SealedSub2","test_case":0},"test_mid":{"typeSub":"SealedSub1"}}""", + json + ) + } +} diff --git a/formats/json-tests/jsTest/src/kotlinx/serialization/json/JsonNamingStrategyDynamicTest.kt b/formats/json-tests/jsTest/src/kotlinx/serialization/json/JsonNamingStrategyDynamicTest.kt new file mode 100644 index 000000000..a1f7b0e65 --- /dev/null +++ b/formats/json-tests/jsTest/src/kotlinx/serialization/json/JsonNamingStrategyDynamicTest.kt @@ -0,0 +1,39 @@ +package kotlinx.serialization.json + +import kotlinx.serialization.* +import kotlinx.serialization.features.* +import kotlin.test.* + +class JsonNamingStrategyDynamicTest: JsonTestBase() { + private val jsForm = js("""{"simple":"a","one_word":"b","already_in_snake":"c","a_lot_of_words":"d","first_capitalized":"e","has_acronym_url":"BAZ","has_digit123_and_postfix":"QUX","coercion_test":"QUX"}""") + private val jsFormNeedsCoercing = js("""{"simple":"a","one_word":"b","already_in_snake":"c","a_lot_of_words":"d","first_capitalized":"e","has_acronym_url":"BAZ","has_digit123_and_postfix":"QUX","coercion_test":"invalid"}""") + + private fun doTest(json: Json) { + val j = Json(json) { + namingStrategy = JsonNamingStrategy.SnakeCase + } + val foo = JsonNamingStrategyTest.Foo() + assertDynamicForm(foo) + assertEquals(foo, j.decodeFromDynamic(jsForm)) + } + + @Test + fun testNamingStrategyWorksWithCoercing() { + val j = Json(default) { + coerceInputValues = true + useAlternativeNames = false + namingStrategy = JsonNamingStrategy.SnakeCase + } + assertEquals(JsonNamingStrategyTest.Foo(), j.decodeFromDynamic(jsFormNeedsCoercing)) + } + + @Test + fun testJsonNamingStrategyWithAlternativeNames() = doTest(Json(default) { + useAlternativeNames = true + }) + + @Test + fun testJsonNamingStrategyWithoutAlternativeNames() = doTest(Json(default) { + useAlternativeNames = false + }) +} diff --git a/formats/json/api/kotlinx-serialization-json.api b/formats/json/api/kotlinx-serialization-json.api index 24aaf10f8..cf088c5b2 100644 --- a/formats/json/api/kotlinx-serialization-json.api +++ b/formats/json/api/kotlinx-serialization-json.api @@ -90,6 +90,8 @@ public final class kotlinx/serialization/json/JsonBuilder { public final fun getEncodeDefaults ()Z public final fun getExplicitNulls ()Z public final fun getIgnoreUnknownKeys ()Z + public final fun getNamingStrategyForEnums ()Lkotlinx/serialization/json/JsonNamingStrategy; + public final fun getNamingStrategyForProperties ()Lkotlinx/serialization/json/JsonNamingStrategy; public final fun getPrettyPrint ()Z public final fun getPrettyPrintIndent ()Ljava/lang/String; public final fun getSerializersModule ()Lkotlinx/serialization/modules/SerializersModule; @@ -104,6 +106,8 @@ public final class kotlinx/serialization/json/JsonBuilder { public final fun setExplicitNulls (Z)V public final fun setIgnoreUnknownKeys (Z)V public final fun setLenient (Z)V + public final fun setNamingStrategyForEnums (Lkotlinx/serialization/json/JsonNamingStrategy;)V + public final fun setNamingStrategyForProperties (Lkotlinx/serialization/json/JsonNamingStrategy;)V public final fun setPrettyPrint (Z)V public final fun setPrettyPrintIndent (Ljava/lang/String;)V public final fun setSerializersModule (Lkotlinx/serialization/modules/SerializersModule;)V @@ -129,6 +133,8 @@ public final class kotlinx/serialization/json/JsonConfiguration { public final fun getEncodeDefaults ()Z public final fun getExplicitNulls ()Z public final fun getIgnoreUnknownKeys ()Z + public final fun getNamingStrategyForEnums ()Lkotlinx/serialization/json/JsonNamingStrategy; + public final fun getNamingStrategyForProperties ()Lkotlinx/serialization/json/JsonNamingStrategy; public final fun getPrettyPrint ()Z public final fun getPrettyPrintIndent ()Ljava/lang/String; public final fun getUseAlternativeNames ()Z @@ -242,6 +248,16 @@ public final class kotlinx/serialization/json/JsonNames$Impl : kotlinx/serializa public final synthetic fun names ()[Ljava/lang/String; } +public abstract interface class kotlinx/serialization/json/JsonNamingStrategy { + public static final field Builtins Lkotlinx/serialization/json/JsonNamingStrategy$Builtins; + public abstract fun serialNameForJson (Lkotlinx/serialization/descriptors/SerialDescriptor;ILjava/lang/String;)Ljava/lang/String; +} + +public final class kotlinx/serialization/json/JsonNamingStrategy$Builtins { + public final fun getAllLowercase ()Lkotlinx/serialization/json/JsonNamingStrategy; + public final fun getSnakeCase ()Lkotlinx/serialization/json/JsonNamingStrategy; +} + public final class kotlinx/serialization/json/JsonNull : kotlinx/serialization/json/JsonPrimitive { public static final field INSTANCE Lkotlinx/serialization/json/JsonNull; public fun getContent ()Ljava/lang/String; diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt b/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt index 339209392..a074b1520 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/Json.kt @@ -7,6 +7,7 @@ package kotlinx.serialization.json import kotlinx.serialization.* import kotlinx.serialization.json.internal.* import kotlinx.serialization.modules.* +import kotlinx.serialization.descriptors.* import kotlin.native.concurrent.* /** @@ -313,8 +314,22 @@ public class JsonBuilder internal constructor(json: Json) { */ public var useAlternativeNames: Boolean = json.configuration.useAlternativeNames + /** + * Specifies [JsonNamingStrategy] that should be used for all properties in classes for serialization and deserialization. + * + * `null` by default. + * + * This strategy is applied for all entities that have [StructureKind.CLASS]. + */ + @ExperimentalSerializationApi + public var namingStrategy: JsonNamingStrategy? = json.configuration.namingStrategy + /** * Module with contextual and polymorphic serializers to be used in the resulting [Json] instance. + * + * @see SerializersModule + * @see Contextual + * @see Polymorphic */ public var serializersModule: SerializersModule = json.serializersModule @@ -340,7 +355,8 @@ public class JsonBuilder internal constructor(json: Json) { encodeDefaults, ignoreUnknownKeys, isLenient, allowStructuredMapKeys, prettyPrint, explicitNulls, prettyPrintIndent, coerceInputValues, useArrayPolymorphism, - classDiscriminator, allowSpecialFloatingPointValues, useAlternativeNames + classDiscriminator, allowSpecialFloatingPointValues, useAlternativeNames, + namingStrategy ) } } diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/JsonAnnotations.kt b/formats/json/commonMain/src/kotlinx/serialization/json/JsonAnnotations.kt index aae69889a..4ec5a2b5c 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/JsonAnnotations.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/JsonAnnotations.kt @@ -24,12 +24,16 @@ import kotlin.native.concurrent.* * data class Project(@JsonNames("title") val name: String) * * val project = Json.decodeFromString("""{"name":"kotlinx.serialization"}""") - * println(project) + * println(project) // OK * val oldProject = Json.decodeFromString("""{"title":"kotlinx.coroutines"}""") - * println(oldProject) + * println(oldProject) // Also OK * ``` * * This annotation has lesser priority than [SerialName]. + * In practice, this means that if property A has `@SerialName("foo")` annotation, and property B has `@JsonNames("foo")` annotation, + * Json key `foo` will be deserialized into property A. + * + * Using the same alternative name for different properties across one class is prohibited and leads to a deserialization exception. * * @see JsonBuilder.useAlternativeNames */ diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt b/formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt index 612cfc7c4..d17d0fcc6 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/JsonConfiguration.kt @@ -14,7 +14,7 @@ import kotlinx.serialization.* * * Detailed description of each property is available in [JsonBuilder] class. */ -public class JsonConfiguration internal constructor( +public class JsonConfiguration @OptIn(ExperimentalSerializationApi::class) internal constructor( public val encodeDefaults: Boolean = false, public val ignoreUnknownKeys: Boolean = false, public val isLenient: Boolean = false, @@ -28,7 +28,9 @@ public class JsonConfiguration internal constructor( public val useArrayPolymorphism: Boolean = false, public val classDiscriminator: String = "type", public val allowSpecialFloatingPointValues: Boolean = false, - public val useAlternativeNames: Boolean = true + public val useAlternativeNames: Boolean = true, + @ExperimentalSerializationApi + public val namingStrategy: JsonNamingStrategy? = null, ) { /** @suppress Dokka **/ @@ -37,6 +39,7 @@ public class JsonConfiguration internal constructor( return "JsonConfiguration(encodeDefaults=$encodeDefaults, ignoreUnknownKeys=$ignoreUnknownKeys, isLenient=$isLenient, " + "allowStructuredMapKeys=$allowStructuredMapKeys, prettyPrint=$prettyPrint, explicitNulls=$explicitNulls, " + "prettyPrintIndent='$prettyPrintIndent', coerceInputValues=$coerceInputValues, useArrayPolymorphism=$useArrayPolymorphism, " + - "classDiscriminator='$classDiscriminator', allowSpecialFloatingPointValues=$allowSpecialFloatingPointValues)" + "classDiscriminator='$classDiscriminator', allowSpecialFloatingPointValues=$allowSpecialFloatingPointValues, useAlternativeNames=$useAlternativeNames, " + + "namingStrategy=$namingStrategy)" } } diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt b/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt new file mode 100644 index 000000000..550e35658 --- /dev/null +++ b/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt @@ -0,0 +1,107 @@ +package kotlinx.serialization.json + +import kotlinx.serialization.* +import kotlinx.serialization.descriptors.* + + +/** + * Represents naming strategy — a transformer for serial names in a [Json] format. + * Transformed serial names are used for both serialization and deserialization. + * Actual transformation happens in the [serialNameForJson] function. + * A naming strategy is always applied globally in the Json configuration builder + * (see [JsonBuilder.namingStrategy]). + * However, it is possible to apply additional filtering inside the transformer using the `descriptor` parameter in [serialNameForJson]. + * + * Original serial names are never used after transformation, so they are ignored in a Json input. + * If the original serial name is present in the Json input but transformed is not, + * [MissingFieldException] still would be thrown. If one wants to preserve the original serial name for deserialization, + * one should use the [JsonNames] annotation. + * + * ### Common pitfalls in conjunction with other Json features + * + * * Due to the nature of kotlinx.serialization framework, naming strategy transformation is applied to all properties regardless + * of whether their serial name was taken from the property name or provided by @[SerialName] annotation. + * Effectively it means one cannot avoid transformation by explicitly specifying the serial name. + * + * * Collision of the transformed name with any other (transformed) properties serial names or any alternative names + * specified with [JsonNames] will lead to a deserialization exception. + * + * * Naming strategies do not transform serial names of the types used for the polymorphism, as they always should be specified explicitly. + * Values from [JsonClassDiscriminator] or global [JsonBuilder.classDiscriminator] also are not altered. + * + * ### Controversy about using global naming strategies + * + * Global naming strategies have one key trait that makes them a debatable and controversial topic: + * They are very implicit. It means that by looking only at the definition of the class, + * it is impossible to say which names it will have in the serialized form. + * As a consequence, naming strategies are not friendly to refactorings. Programmer renaming `myId` to `userId` may forget + * to rename `my_id`, and vice versa. Generally, any tools one can imagine work poorly with global naming strategies: + * Find Usages/Rename in IDE, full-text search by grep, etc. For them, the original name and the transformed are two different things; + * changing one without the other may introduce bugs in many unexpected ways. + * The lack of a single place of definition, the inability to use automated tools, and more error-prone code lead + * to greater maintenance efforts for code with global naming strategies. + * However, there are cases where usage of naming strategies is inevitable, such as interop with existing API or migrating a large codebase. + * Therefore, one should carefully weigh the pros and cons before considering adding global naming strategies to an application. + */ +@ExperimentalSerializationApi +public fun interface JsonNamingStrategy { + /** + * Accepts an original [serialName] (defined by property name in the class or [SerialName] annotation) and returns + * a transformed serial name which should be used for serialization and deserialization. + * + * Besides string manipulation operations, it is also possible to implement transformations that depend on the [descriptor] + * and its element (defined by [elementIndex]) currently being serialized. + * It is guaranteed that `descriptor.getElementName(elementIndex) == serialName`. + * For example, one can choose different transformations depending on [SerialInfo] + * annotations (see [SerialDescriptor.getElementAnnotations]) or element optionality (see [SerialDescriptor.isElementOptional]). + * + * Note that invocations of this function are cached for performance reasons. + * Caching strategy is an implementation detail and shouldn't be assumed as a part of the public API contract, as it may be changed in future releases. + * Therefore, it is essential for this function to be pure: it should not have any side effects, and it should + * return the same String for a given [descriptor], [elementIndex], and [serialName], regardless of the number of invocations. + */ + public fun serialNameForJson(descriptor: SerialDescriptor, elementIndex: Int, serialName: String): String + + /** + * Contains basic, ready to use naming strategies. + */ + @ExperimentalSerializationApi + public companion object Builtins { + + /** + * A strategy that transforms serial names from camel case to snake case — lowercase characters with words separated by underscores. + * The descriptor parameter is not used. + * + * It applies to every character following transformation rules: + * + * 1. If character `C` is in upper case, and the previous character exists, was not uppercase, and was not underscore, character `C` is transformed into underscore + c.lowercaseChar(): `_c`. + * 2. If character `C` is in upper case but does not match other conditions from 1., it is transformed into lowercase: `c`. Thus, upper case acronyms like URL are transformed correctly. + * 3. Otherwise, the character remains intact. + * + * **Note on cases:** Whether a character is in upper case is determined by the result of [Char.isUpperCase] function. + * Lowercase transformation is performed by [Char.lowercaseChar], not by [Char.lowercase], + * and therefore does not support one-to-many and many-to-one character mappings. + * See the documentation of these functions for details. + */ + @ExperimentalSerializationApi + public val SnakeCase: JsonNamingStrategy = object : JsonNamingStrategy { + override fun serialNameForJson(descriptor: SerialDescriptor, elementIndex: Int, serialName: String): String = + buildString(serialName.length * 2) { + var previousWasUppercase = false + serialName.forEach { c -> + if (c.isUpperCase()) { + if (!previousWasUppercase && isNotEmpty() && last() != '_') + append('_') + previousWasUppercase = true + append(c.lowercaseChar()) + } else { + previousWasUppercase = false + append(c) + } + } + } + + override fun toString(): String = "kotlinx.serialization.json.JsonNamingStrategy.SnakeCase" + } + } +} diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonExceptions.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonExceptions.kt index d1698db2f..ad5011a93 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonExceptions.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonExceptions.kt @@ -70,7 +70,7 @@ private fun unexpectedFpErrorMessage(value: Number, key: String, output: String) internal fun UnknownKeyException(key: String, input: String) = JsonDecodingException( -1, - "Encountered unknown key '$key'.\n" + + "Encountered an unknown key '$key'.\n" + "$ignoreUnknownKeysHint\n" + "Current input: ${input.minify()}" ) diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt index 93e604a72..66a136d03 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt @@ -1,6 +1,7 @@ /* * Copyright 2017-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ +@file:OptIn(ExperimentalSerializationApi::class) package kotlinx.serialization.json.internal @@ -11,36 +12,63 @@ import kotlinx.serialization.json.* import kotlin.native.concurrent.* @SharedImmutable -internal val JsonAlternativeNamesKey = DescriptorSchemaCache.Key>() +internal val JsonDeserializationNamesKey = DescriptorSchemaCache.Key>() -@OptIn(ExperimentalSerializationApi::class) -internal fun SerialDescriptor.buildAlternativeNamesMap(): Map { +@SharedImmutable +internal val JsonSerializationNamesKey = DescriptorSchemaCache.Key>() + +internal fun SerialDescriptor.buildDeserializationNamesMap(json: Json): Map { fun MutableMap.putOrThrow(name: String, index: Int) { if (name in this) { throw JsonException( "The suggested name '$name' for property ${getElementName(index)} is already one of the names for property " + - "${getElementName(getValue(name))} in ${this@buildAlternativeNamesMap}" + "${getElementName(getValue(name))} in ${this@buildDeserializationNamesMap}" ) } this[name] = index } - var builder: MutableMap? = null + val builder: MutableMap = + mutableMapOf() // can be not concurrent because it is only read after creation and safely published to concurrent map + val strategy = namingStrategy(json) for (i in 0 until elementsCount) { getElementAnnotations(i).filterIsInstance().singleOrNull()?.names?.forEach { name -> - if (builder == null) builder = createMapForCache(elementsCount) - builder!!.putOrThrow(name, i) + builder.putOrThrow(name, i) } + strategy?.let { builder.putOrThrow(it.serialNameForJson(this, i, getElementName(i)), i) } } - return builder ?: emptyMap() + return builder.ifEmpty { emptyMap() } } +internal fun Json.deserializationNamesMap(descriptor: SerialDescriptor): Map = + schemaCache.getOrPut(descriptor, JsonDeserializationNamesKey) { descriptor.buildDeserializationNamesMap(this) } + +internal fun SerialDescriptor.serializationNamesMap(json: Json, strategy: JsonNamingStrategy): Array = json.schemaCache.getOrPut(this, JsonSerializationNamesKey) { + Array(elementsCount) { i -> + val baseName = getElementName(i) + strategy.serialNameForJson(this, i, baseName) + } +} + +internal fun SerialDescriptor.getJsonElementName(json: Json, index: Int): String { + val strategy = namingStrategy(json) + return if (strategy == null) getElementName(index) else serializationNamesMap(json, strategy)[index] +} + +internal fun SerialDescriptor.namingStrategy(json: Json) = + if (kind == StructureKind.CLASS) json.configuration.namingStrategy else null + /** * Serves same purpose as [SerialDescriptor.getElementIndex] but respects * [JsonNames] annotation and [JsonConfiguration.useAlternativeNames] state. */ @OptIn(ExperimentalSerializationApi::class) internal fun SerialDescriptor.getJsonNameIndex(json: Json, name: String): Int { + fun getJsonNameIndexSlowPath(): Int = + json.deserializationNamesMap(this)[name] ?: CompositeDecoder.UNKNOWN_NAME + + val strategy = namingStrategy(json) + if (strategy != null) return getJsonNameIndexSlowPath() val index = getElementIndex(name) // Fast path, do not go through ConcurrentHashMap.get // Note, it blocks ability to detect collisions between the primary name and alternate, @@ -48,9 +76,7 @@ internal fun SerialDescriptor.getJsonNameIndex(json: Json, name: String): Int { if (index != CompositeDecoder.UNKNOWN_NAME) return index if (!json.configuration.useAlternativeNames) return index // Slow path - val alternativeNamesMap = - json.schemaCache.getOrPut(this, JsonAlternativeNamesKey, this::buildAlternativeNamesMap) - return alternativeNamesMap[name] ?: CompositeDecoder.UNKNOWN_NAME + return getJsonNameIndexSlowPath() } /** @@ -74,7 +100,7 @@ internal inline fun Json.tryCoerceValue( if (!elementDescriptor.isNullable && peekNull()) return true if (elementDescriptor.kind == SerialKind.ENUM) { val enumValue = peekString() - ?: return false // if value is not a string, decodeEnum() will throw correct exception + ?: return false // if value is not a string, decodeEnum() will throw correct exception val enumIndex = elementDescriptor.getJsonNameIndex(this, enumValue) if (enumIndex == CompositeDecoder.UNKNOWN_NAME) { onEnumCoercing() diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/SchemaCache.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/SchemaCache.kt index de65fb686..b514f6e68 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/SchemaCache.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/SchemaCache.kt @@ -16,11 +16,13 @@ private typealias DescriptorData = MutableMap, T * To be able to work with it from multiple threads in Kotlin/Native, use @[ThreadLocal] in appropriate places. */ internal class DescriptorSchemaCache { - private val map: MutableMap> = createMapForCache(1) + // 16 is default CHM size, as we do not know number of descriptors in an application (but it's likely not 1) + private val map: MutableMap> = createMapForCache(16) @Suppress("UNCHECKED_CAST") public operator fun set(descriptor: SerialDescriptor, key: Key, value: T) { - map.getOrPut(descriptor, { createMapForCache(1) })[key as Key] = value as Any + // Initial capacity = number of known DescriptorSchemaCache.Key instances + map.getOrPut(descriptor, { createMapForCache(2) })[key as Key] = value as Any } public fun getOrPut(descriptor: SerialDescriptor, key: Key, defaultValue: () -> T): T { diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonEncoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonEncoder.kt index bc954ce9c..10a464d02 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonEncoder.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonEncoder.kt @@ -139,7 +139,7 @@ internal class StreamingJsonEncoder( if (!composer.writingFirst) composer.print(COMMA) composer.nextItem() - encodeString(descriptor.getElementName(index)) + encodeString(descriptor.getJsonElementName(json, index)) composer.print(COLON) composer.space() } diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt index 6541cb3b3..9e0cb2ad5 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt @@ -134,7 +134,7 @@ private sealed class AbstractJsonTreeDecoder( override fun decodeTaggedChar(tag: String): Char = getPrimitiveValue(tag).primitive("char") { content.single() } - private inline fun JsonPrimitive.primitive(primitive: String, block: JsonPrimitive.() -> T?): T { + private inline fun JsonPrimitive.primitive(primitive: String, block: JsonPrimitive.() -> T?): T { try { return block() ?: unparsedPrimitive(primitive) } catch (e: IllegalArgumentException) { @@ -224,18 +224,28 @@ private open class JsonTreeDecoder( return !forceNull && super.decodeNotNullMark() } - override fun elementName(desc: SerialDescriptor, index: Int): String { - val mainName = desc.getElementName(index) - if (!configuration.useAlternativeNames) return mainName - // Fast path, do not go through ConcurrentHashMap.get - // Note, it blocks ability to detect collisions between the primary name and alternate, - // but it eliminates a significant performance penalty (about -15% without this optimization) - if (mainName in value.keys) return mainName + override fun elementName(descriptor: SerialDescriptor, index: Int): String { + val strategy = descriptor.namingStrategy(json) + val baseName = descriptor.getElementName(index) + if (strategy == null) { + if (!configuration.useAlternativeNames) return baseName + // Fast path, do not go through ConcurrentHashMap.get + // Note, it blocks ability to detect collisions between the primary name and alternate, + // but it eliminates a significant performance penalty (about -15% without this optimization) + if (baseName in value.keys) return baseName + } // Slow path - val alternativeNamesMap = - json.schemaCache.getOrPut(desc, JsonAlternativeNamesKey, desc::buildAlternativeNamesMap) - val nameInObject = value.keys.find { alternativeNamesMap[it] == index } - return nameInObject ?: mainName + val deserializationNamesMap = json.deserializationNamesMap(descriptor) + value.keys.find { deserializationNamesMap[it] == index }?.let { + return it + } + + val fallbackName = strategy?.serialNameForJson( + descriptor, + index, + baseName + ) // Key not found exception should be thrown with transformed name, not original + return fallbackName ?: baseName } override fun currentElement(tag: String): JsonElement = value.getValue(tag) @@ -252,12 +262,14 @@ private open class JsonTreeDecoder( override fun endStructure(descriptor: SerialDescriptor) { if (configuration.ignoreUnknownKeys || descriptor.kind is PolymorphicKind) return // Validate keys + val strategy = descriptor.namingStrategy(json) + @Suppress("DEPRECATION_ERROR") - val names: Set = - if (!configuration.useAlternativeNames) - descriptor.jsonCachedSerialNames() - else - descriptor.jsonCachedSerialNames() + json.schemaCache[descriptor, JsonAlternativeNamesKey]?.keys.orEmpty() + val names: Set = when { + strategy == null && !configuration.useAlternativeNames -> descriptor.jsonCachedSerialNames() + strategy != null -> json.deserializationNamesMap(descriptor).keys + else -> descriptor.jsonCachedSerialNames() + json.schemaCache[descriptor, JsonDeserializationNamesKey]?.keys.orEmpty() + } for (key in value.keys) { if (key !in names && key != polyDiscriminator) { @@ -272,7 +284,7 @@ private class JsonTreeMapDecoder(json: Json, override val value: JsonObject) : J private val size: Int = keys.size * 2 private var position = -1 - override fun elementName(desc: SerialDescriptor, index: Int): String { + override fun elementName(descriptor: SerialDescriptor, index: Int): String { val i = index / 2 return keys[i] } @@ -298,7 +310,7 @@ private class JsonTreeListDecoder(json: Json, override val value: JsonArray) : A private val size = value.size private var currentIndex = -1 - override fun elementName(desc: SerialDescriptor, index: Int): String = (index).toString() + override fun elementName(descriptor: SerialDescriptor, index: Int): String = index.toString() override fun currentElement(tag: String): JsonElement { return value[tag.toInt()] diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt index 643e158e1..d3c8a3d30 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt @@ -36,6 +36,9 @@ private sealed class AbstractJsonTreeEncoder( private var polymorphicDiscriminator: String? = null + override fun elementName(descriptor: SerialDescriptor, index: Int): String = + descriptor.getJsonElementName(json, index) + override fun encodeJsonElement(element: JsonElement) { encodeSerializableValue(JsonElementSerializer, element) } diff --git a/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicDecoders.kt b/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicDecoders.kt index a6658c7cb..1447881c1 100644 --- a/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicDecoders.kt +++ b/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicDecoders.kt @@ -100,18 +100,27 @@ private open class DynamicInput( return forceNull } - override fun elementName(desc: SerialDescriptor, index: Int): String { - val mainName = desc.getElementName(index) - if (!json.configuration.useAlternativeNames) return mainName - // Fast path, do not go through Map.get - // Note, it blocks ability to detect collisions between the primary name and alternate, - // but it eliminates a significant performance penalty (about -15% without this optimization) - if (hasName(mainName)) return mainName + override fun elementName(descriptor: SerialDescriptor, index: Int): String { + val strategy = descriptor.namingStrategy(json) + val mainName = descriptor.getElementName(index) + if (strategy == null) { + if (!json.configuration.useAlternativeNames) return mainName + // Fast path, do not go through Map.get + // Note, it blocks ability to detect collisions between the primary name and alternate, + // but it eliminates a significant performance penalty (about -15% without this optimization) + if (hasName(mainName)) return mainName + } // Slow path - val alternativeNamesMap = - json.schemaCache.getOrPut(desc, JsonAlternativeNamesKey, desc::buildAlternativeNamesMap) - val nameInObject = (keys as Array).find { alternativeNamesMap[it] == index } - return nameInObject ?: mainName + val deserializationNamesMap = json.deserializationNamesMap(descriptor) + (keys as Array).find { deserializationNamesMap[it] == index }?.let { + return it + } + val fallbackName = strategy?.serialNameForJson( + descriptor, + index, + mainName + ) // Key not found exception should be thrown with transformed name, not original + return fallbackName ?: mainName } override fun decodeTaggedEnum(tag: String, enumDescriptor: SerialDescriptor): Int { @@ -191,7 +200,7 @@ private class DynamicMapInput( private var currentPosition = -1 private val isKey: Boolean get() = currentPosition % 2 == 0 - override fun elementName(desc: SerialDescriptor, index: Int): String { + override fun elementName(descriptor: SerialDescriptor, index: Int): String { val i = index / 2 return keys[i] as String } @@ -261,7 +270,7 @@ private class DynamicListInput( override val size = value.length as Int private var currentPosition = -1 - override fun elementName(desc: SerialDescriptor, index: Int): String = (index).toString() + override fun elementName(descriptor: SerialDescriptor, index: Int): String = (index).toString() override fun decodeElementIndex(descriptor: SerialDescriptor): Int { while (currentPosition < size - 1) { diff --git a/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicEncoders.kt b/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicEncoders.kt index e58158384..4c4841d47 100644 --- a/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicEncoders.kt +++ b/formats/json/jsMain/src/kotlinx/serialization/json/internal/DynamicEncoders.kt @@ -81,7 +81,7 @@ private class DynamicObjectEncoder( when { current.writeMode == WriteMode.MAP -> currentElementIsMapKey = current.index % 2 == 0 current.writeMode == WriteMode.LIST && descriptor.kind is PolymorphicKind -> currentName = index.toString() - else -> currentName = descriptor.getElementName(index) + else -> currentName = descriptor.getJsonElementName(json, index) } return true From b1c07803498c4ed057ee49175a10caee227af4ed Mon Sep 17 00:00:00 2001 From: Leonid Startsev Date: Fri, 13 Jan 2023 16:35:48 +0100 Subject: [PATCH 2/5] !fixup Remove AllLowercase built-in strategy --- formats/json/api/kotlinx-serialization-json.api | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/formats/json/api/kotlinx-serialization-json.api b/formats/json/api/kotlinx-serialization-json.api index cf088c5b2..0d8c25e46 100644 --- a/formats/json/api/kotlinx-serialization-json.api +++ b/formats/json/api/kotlinx-serialization-json.api @@ -90,8 +90,7 @@ public final class kotlinx/serialization/json/JsonBuilder { public final fun getEncodeDefaults ()Z public final fun getExplicitNulls ()Z public final fun getIgnoreUnknownKeys ()Z - public final fun getNamingStrategyForEnums ()Lkotlinx/serialization/json/JsonNamingStrategy; - public final fun getNamingStrategyForProperties ()Lkotlinx/serialization/json/JsonNamingStrategy; + public final fun getNamingStrategy ()Lkotlinx/serialization/json/JsonNamingStrategy; public final fun getPrettyPrint ()Z public final fun getPrettyPrintIndent ()Ljava/lang/String; public final fun getSerializersModule ()Lkotlinx/serialization/modules/SerializersModule; @@ -106,8 +105,7 @@ public final class kotlinx/serialization/json/JsonBuilder { public final fun setExplicitNulls (Z)V public final fun setIgnoreUnknownKeys (Z)V public final fun setLenient (Z)V - public final fun setNamingStrategyForEnums (Lkotlinx/serialization/json/JsonNamingStrategy;)V - public final fun setNamingStrategyForProperties (Lkotlinx/serialization/json/JsonNamingStrategy;)V + public final fun setNamingStrategy (Lkotlinx/serialization/json/JsonNamingStrategy;)V public final fun setPrettyPrint (Z)V public final fun setPrettyPrintIndent (Ljava/lang/String;)V public final fun setSerializersModule (Lkotlinx/serialization/modules/SerializersModule;)V @@ -133,8 +131,7 @@ public final class kotlinx/serialization/json/JsonConfiguration { public final fun getEncodeDefaults ()Z public final fun getExplicitNulls ()Z public final fun getIgnoreUnknownKeys ()Z - public final fun getNamingStrategyForEnums ()Lkotlinx/serialization/json/JsonNamingStrategy; - public final fun getNamingStrategyForProperties ()Lkotlinx/serialization/json/JsonNamingStrategy; + public final fun getNamingStrategy ()Lkotlinx/serialization/json/JsonNamingStrategy; public final fun getPrettyPrint ()Z public final fun getPrettyPrintIndent ()Ljava/lang/String; public final fun getUseAlternativeNames ()Z @@ -254,7 +251,6 @@ public abstract interface class kotlinx/serialization/json/JsonNamingStrategy { } public final class kotlinx/serialization/json/JsonNamingStrategy$Builtins { - public final fun getAllLowercase ()Lkotlinx/serialization/json/JsonNamingStrategy; public final fun getSnakeCase ()Lkotlinx/serialization/json/JsonNamingStrategy; } From 32d5031f2dc09afcf1e2b8bca0d93164887855e4 Mon Sep 17 00:00:00 2001 From: Leonid Startsev Date: Fri, 20 Jan 2023 18:39:23 +0100 Subject: [PATCH 3/5] Improve SnakeCase strategy --- .../features/JsonNamingStrategyTest.kt | 17 ++++++- .../serialization/json/JsonNamingStrategy.kt | 45 ++++++++++++++----- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyTest.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyTest.kt index 1b9da667e..330d5d2bb 100644 --- a/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyTest.kt +++ b/formats/json-tests/commonTest/src/kotlinx/serialization/features/JsonNamingStrategyTest.kt @@ -74,11 +74,24 @@ class JsonNamingStrategyTest : JsonTestBase() { "_A" to "_a", "property" to "property", "twoWords" to "two_words", + "threeDistinctWords" to "three_distinct_words", + "ThreeDistinctWords" to "three_distinct_words", "Oneword" to "oneword", "camel_Case_Underscores" to "camel_case_underscores", "_many____underscores__" to "_many____underscores__", - "URLmapping" to "urlmapping", // can be fixed, maybe later - "InURLBetween" to "in_urlbetween", // can be fixed, maybe later + "URLmapping" to "ur_lmapping", + "URLMapping" to "url_mapping", + "IOStream" to "io_stream", + "IOstream" to "i_ostream", + "myIo2Stream" to "my_io2_stream", + "myIO2Stream" to "my_io2_stream", + "myIO2stream" to "my_io2stream", + "myIO2streamMax" to "my_io2stream_max", + "InURLBetween" to "in_url_between", + "myHTTP2APIKey" to "my_http2_api_key", + "myHTTP2fastApiKey" to "my_http2fast_api_key", + "myHTTP23APIKey" to "my_http23_api_key", + "myHttp23ApiKey" to "my_http23_api_key", "theWWW" to "the_www", "theWWW_URL_xxx" to "the_www_url_xxx", "hasDigit123AndPostfix" to "has_digit123_and_postfix" diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt b/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt index 550e35658..4bd7be508 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt @@ -72,13 +72,22 @@ public fun interface JsonNamingStrategy { * A strategy that transforms serial names from camel case to snake case — lowercase characters with words separated by underscores. * The descriptor parameter is not used. * - * It applies to every character following transformation rules: + * **Transformation rules** * - * 1. If character `C` is in upper case, and the previous character exists, was not uppercase, and was not underscore, character `C` is transformed into underscore + c.lowercaseChar(): `_c`. - * 2. If character `C` is in upper case but does not match other conditions from 1., it is transformed into lowercase: `c`. Thus, upper case acronyms like URL are transformed correctly. - * 3. Otherwise, the character remains intact. + * Words bounds are defined by uppercase characters. If there is a single uppercase char, it is transformed into lowercase one with underscore in front: + * `twoWords` -> `two_words`. No underscore is added if it was a beginning of the name: `MyProperty` -> `my_property`. Also, no underscore is added if it was already there: + * `camel_Case_Underscores` -> `camel_case_underscores`. * - * **Note on cases:** Whether a character is in upper case is determined by the result of [Char.isUpperCase] function. + * **Acronyms** + * + * Since acronym rules are quite complex, it is recommended to lowercase all acronyms in source code. + * If there is an uppercase acronym — a sequence of uppercase chars — they are considered as a whole word from the start to second-to-last character of the sequence: + * `URLMapping` -> `url_mapping`, `myHTTPAuth` -> `my_http_auth`. Non-letter characters allow the word to continue: + * `myHTTP2APIKey` -> `my_http2_api_key`, `myHTTP2fastApiKey` -> `my_http2fast_api_key`. + * + * **Note on cases** + * + * Whether a character is in upper case is determined by the result of [Char.isUpperCase] function. * Lowercase transformation is performed by [Char.lowercaseChar], not by [Char.lowercase], * and therefore does not support one-to-many and many-to-one character mappings. * See the documentation of these functions for details. @@ -87,18 +96,34 @@ public fun interface JsonNamingStrategy { public val SnakeCase: JsonNamingStrategy = object : JsonNamingStrategy { override fun serialNameForJson(descriptor: SerialDescriptor, elementIndex: Int, serialName: String): String = buildString(serialName.length * 2) { - var previousWasUppercase = false + var bufferedChar: Char? = null + var previousUpperCharsCount = 0 + serialName.forEach { c -> if (c.isUpperCase()) { - if (!previousWasUppercase && isNotEmpty() && last() != '_') + if (previousUpperCharsCount == 0 && isNotEmpty() && last() != '_') append('_') - previousWasUppercase = true - append(c.lowercaseChar()) + + bufferedChar?.let(::append) + + previousUpperCharsCount++ + bufferedChar = c.lowercaseChar() } else { - previousWasUppercase = false + if (bufferedChar != null) { + if (previousUpperCharsCount > 1 && c.isLetter()) { + append('_') + } + append(bufferedChar) + previousUpperCharsCount = 0 + bufferedChar = null + } append(c) } } + + if(bufferedChar != null) { + append(bufferedChar) + } } override fun toString(): String = "kotlinx.serialization.json.JsonNamingStrategy.SnakeCase" From a9c96c5d71f6ea6d021e19d7511250a05ee5a5fd Mon Sep 17 00:00:00 2001 From: Leonid Startsev Date: Fri, 20 Jan 2023 18:47:15 +0100 Subject: [PATCH 4/5] Improve serialization/deserialization names maps --- .../json/internal/JsonNamesMap.kt | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt index 66a136d03..bf616f98e 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonNamesMap.kt @@ -17,7 +17,7 @@ internal val JsonDeserializationNamesKey = DescriptorSchemaCache.Key>() -internal fun SerialDescriptor.buildDeserializationNamesMap(json: Json): Map { +private fun SerialDescriptor.buildDeserializationNamesMap(json: Json): Map { fun MutableMap.putOrThrow(name: String, index: Int) { if (name in this) { throw JsonException( @@ -40,19 +40,24 @@ internal fun SerialDescriptor.buildDeserializationNamesMap(json: Json): Map = schemaCache.getOrPut(descriptor, JsonDeserializationNamesKey) { descriptor.buildDeserializationNamesMap(this) } -internal fun SerialDescriptor.serializationNamesMap(json: Json, strategy: JsonNamingStrategy): Array = json.schemaCache.getOrPut(this, JsonSerializationNamesKey) { - Array(elementsCount) { i -> - val baseName = getElementName(i) - strategy.serialNameForJson(this, i, baseName) +internal fun SerialDescriptor.serializationNamesIndices(json: Json, strategy: JsonNamingStrategy): Array = + json.schemaCache.getOrPut(this, JsonSerializationNamesKey) { + Array(elementsCount) { i -> + val baseName = getElementName(i) + strategy.serialNameForJson(this, i, baseName) + } } -} internal fun SerialDescriptor.getJsonElementName(json: Json, index: Int): String { val strategy = namingStrategy(json) - return if (strategy == null) getElementName(index) else serializationNamesMap(json, strategy)[index] + return if (strategy == null) getElementName(index) else serializationNamesIndices(json, strategy)[index] } internal fun SerialDescriptor.namingStrategy(json: Json) = From c8ade783f644e19c69378e24e9a54c0edc7afc56 Mon Sep 17 00:00:00 2001 From: Leonid Startsev Date: Mon, 23 Jan 2023 18:49:39 +0100 Subject: [PATCH 5/5] ~clarification --- .../src/kotlinx/serialization/json/JsonNamingStrategy.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt b/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt index 4bd7be508..060572af4 100644 --- a/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt +++ b/formats/json/commonMain/src/kotlinx/serialization/json/JsonNamingStrategy.kt @@ -15,7 +15,7 @@ import kotlinx.serialization.descriptors.* * Original serial names are never used after transformation, so they are ignored in a Json input. * If the original serial name is present in the Json input but transformed is not, * [MissingFieldException] still would be thrown. If one wants to preserve the original serial name for deserialization, - * one should use the [JsonNames] annotation. + * one should use the [JsonNames] annotation, as its values are not transformed. * * ### Common pitfalls in conjunction with other Json features *