Pickler: add transientNone for optional fields (#3816)

doc/endpoint/pickler.md

@@ -235,7 +235,9 @@ you can proceed with `Pickler.derived[T]`.
 
 ## Divergences from default µPickle behavior
 
-* Tapir pickler serialises None values as `null`, instead of wrapping the value in an array
+* Tapir pickler serialises fields of type `Option[T]` as direct value `T` or skips serialisation if field value is `None`. 
+  This default behavior can be changed by setting `.withTransientNone(false)` in `PicklerConfiguration`, which would result in serialising `None` as `null`.
+  This differs from uPickle, where optional values are wrapped in arrays.
 * Value classes (case classes extending AnyVal) will be serialised as simple values
 * Discriminator field value is a short class name, instead of full package with class name
 

json/pickler/src/main/scala/sttp/tapir/json/pickler/PicklerConfiguration.scala

@@ -3,7 +3,13 @@ package sttp.tapir.json.pickler
 import sttp.tapir.generic.Configuration
 import upickle.core.Annotator
 
-final case class PicklerConfiguration(genericDerivationConfig: Configuration) {
+/** Configuration parameters for Pickler.
+  * @param genericDerivationConfig
+  *   basic configuration for schema and codec derivation
+  * @param transientNone
+  *   skip serialization of Option fields if their value is None. If false, None will be serialized as null.
+  */
+final case class PicklerConfiguration(genericDerivationConfig: Configuration, transientNone: Boolean = true) {
   export genericDerivationConfig.{toEncodedName, toDiscriminatorValue}
 
   def discriminator: String = genericDerivationConfig.discriminator.getOrElse(Annotator.defaultTagKey)
@@ -31,6 +37,7 @@ final case class PicklerConfiguration(genericDerivationConfig: Configuration) {
   def withFullKebabCaseDiscriminatorValues: PicklerConfiguration = PicklerConfiguration(
     genericDerivationConfig.withFullKebabCaseDiscriminatorValues
   )
+  def withTransientNone(transientNone: Boolean): PicklerConfiguration = copy(transientNone = transientNone)
 }
 
 object PicklerConfiguration {

json/pickler/src/main/scala/sttp/tapir/json/pickler/Writers.scala

@@ -4,7 +4,6 @@ import _root_.upickle.core.Annotator.Checker
 import _root_.upickle.core.{ObjVisitor, Visitor, _}
 import _root_.upickle.implicits.{WritersVersionSpecific, macros => upickleMacros}
 import sttp.tapir.Schema
-import sttp.tapir.Schema.SName
 import sttp.tapir.SchemaType.SProduct
 import sttp.tapir.generic.Configuration
 import sttp.tapir.internal.EnumerationMacros.*
@@ -47,7 +46,8 @@ private[pickler] trait Writers extends WritersVersionSpecific with UpickleHelper
             v,
             ctx,
             childWriters,
-            childDefaults
+            childDefaults,
+            config.transientNone
           )
           ctx.visitEnd(-1)
         }
@@ -61,7 +61,8 @@ private[pickler] trait Writers extends WritersVersionSpecific with UpickleHelper
           v,
           ctx,
           childWriters,
-          childDefaults
+          childDefaults,
+          config.transientNone                  
         )
     }
 

json/pickler/src/main/scala/sttp/tapir/json/pickler/macros.scala

@@ -2,12 +2,10 @@ package sttp.tapir.json.pickler
 
 import _root_.upickle.implicits.*
 import _root_.upickle.implicits.{macros => uMacros}
-import sttp.tapir.internal.EnumerationMacros.*
 import sttp.tapir.SchemaType
 import sttp.tapir.SchemaType.SProduct
 
 import scala.quoted.*
-import compiletime.*
 
 /** Macros, mostly copied from uPickle, and modified to allow our customizations like passing writers/readers as parameters, adjusting
   * encoding/decoding logic to make it coherent with the schema.
@@ -22,9 +20,10 @@ private[pickler] object macros:
       inline v: T,
       inline ctx: _root_.upickle.core.ObjVisitor[_, R],
       childWriters: List[Any],
-      childDefaults: List[Option[Any]]
+      childDefaults: List[Option[Any]],
+      transientNone: Boolean
   ): Unit =
-    ${ writeSnippetsImpl[R, T]('sProduct, 'thisOuter, 'self, 'v, 'ctx, 'childWriters, 'childDefaults) }
+    ${ writeSnippetsImpl[R, T]('sProduct, 'thisOuter, 'self, 'v, 'ctx, 'childWriters, 'childDefaults, 'transientNone) }
 
   private[pickler] def writeSnippetsImpl[R, T](
       sProduct: Expr[SProduct[T]],
@@ -33,27 +32,32 @@ private[pickler] object macros:
       v: Expr[T],
       ctx: Expr[_root_.upickle.core.ObjVisitor[_, R]],
       childWriters: Expr[List[?]],
-      childDefaults: Expr[List[Option[?]]]
+      childDefaults: Expr[List[Option[?]]],
+      transientNone: Expr[Boolean]
   )(using Quotes, Type[T], Type[R]): Expr[Unit] =
 
     import quotes.reflect.*
+    val optionSymbol = TypeRepr.of[Option[_]].typeSymbol
     Expr.block(
       for (((rawLabel, label), i) <- uMacros.fieldLabelsImpl0[T].zipWithIndex) yield {
-        val tpe0 = TypeRepr.of[T].memberType(rawLabel).asType
+        val memberTypeRepr = TypeRepr.of[T].memberType(rawLabel)
+        val tpe0 = memberTypeRepr.asType
         tpe0 match
           case '[tpe] =>
             Literal(IntConstant(i)).tpe.asType match
               case '[IsInt[index]] =>
                 val encodedName = '{ ${ sProduct }.fields(${ Expr(i) }).name.encodedName }
                 val select = Select.unique(v.asTerm, rawLabel.name).asExprOf[Any]
-                '{
+                val snippet = '{
                   ${ self }.writeSnippetMappedName[R, tpe](
                     ${ ctx },
                     ${ encodedName },
                     ${ childWriters }(${ Expr(i) }),
                     ${ select }
                   )
                 }
+                if memberTypeRepr.typeSymbol == optionSymbol then '{ if !${ transientNone } || ${ select } != None then $snippet }
+                else snippet
       },
       '{ () }
     )
@@ -72,7 +76,9 @@ private[pickler] object macros:
   ) = {
     import quotes.reflect.*
 
+    val optionSymbol = TypeRepr.of[Option[_]].typeSymbol
     val defaults = uMacros.getDefaultParamsImpl0[T]
+    val members = TypeRepr.of[T].typeSymbol.caseFields
     val statements = uMacros
       .fieldLabelsImpl0[T]
       .zipWithIndex
@@ -86,7 +92,18 @@ private[pickler] object macros:
             }
           }),
           if (defaults.contains(label)) '{ ${ x }.storeValueIfNotFound(${ Expr(i) }, ${ defaults(label) }) }
-          else '{}
+          else {
+            members
+              .find(_.name == label)
+              .collect { case member =>
+                member.tree match {
+                  case v: ValDef if v.tpt.tpe.typeSymbol == optionSymbol =>
+                    '{ ${ x }.storeValueIfNotFound(${ Expr(i) }, None) }
+                  case _ => '{}
+                }
+              }
+              .getOrElse('{})
+          }
         )
       }
 

json/pickler/src/test/scala/sttp/tapir/json/pickler/Fixtures.scala

@@ -44,7 +44,7 @@ object Fixtures:
       fieldC: InnerCaseClass
   )
   case class InnerCaseClass(fieldInner: String, @default(4) fieldInnerInt: Int)
-  case class FlatClassWithOption(fieldA: String, fieldB: Option[Int])
+  case class FlatClassWithOption(fieldA: String, fieldB: Option[Int], fieldC: Boolean)
   case class NestedClassWithOption(innerField: Option[FlatClassWithOption])
 
   case class FlatClassWithList(fieldA: String, fieldB: List[Int])

json/pickler/src/test/scala/sttp/tapir/json/pickler/PicklerBasicTest.scala

@@ -102,18 +102,36 @@ class PicklerBasicTest extends AnyFlatSpec with Matchers {
     // when
     val pickler1 = Pickler.derived[FlatClassWithOption]
     val pickler2 = Pickler.derived[NestedClassWithOption]
-    val jsonStr1 = pickler1.toCodec.encode(FlatClassWithOption("fieldA value", Some(-4018)))
-    val jsonStr2 = pickler2.toCodec.encode(NestedClassWithOption(Some(FlatClassWithOption("fieldA value2", Some(-3014)))))
-    val jsonStr3 = pickler1.toCodec.encode(FlatClassWithOption("fieldA value", None))
+    val jsonStr1 = pickler1.toCodec.encode(FlatClassWithOption("fieldA value", Some(-4018), true))
+    val jsonStr2 = pickler2.toCodec.encode(NestedClassWithOption(Some(FlatClassWithOption("fieldA value2", None, true))))
+    val jsonStr3 = pickler1.toCodec.encode(FlatClassWithOption("fieldA value", None, true))
 
     // then
     {
       given derivedFlatClassSchema: Schema[FlatClassWithOption] = Schema.derived[FlatClassWithOption]
       pickler1.schema shouldBe derivedFlatClassSchema
       pickler2.schema shouldBe Schema.derived[NestedClassWithOption]
-      jsonStr1 shouldBe """{"fieldA":"fieldA value","fieldB":-4018}"""
-      jsonStr2 shouldBe """{"innerField":{"fieldA":"fieldA value2","fieldB":-3014}}"""
-      jsonStr3 shouldBe """{"fieldA":"fieldA value","fieldB":null}"""
+      jsonStr1 shouldBe """{"fieldA":"fieldA value","fieldB":-4018,"fieldC":true}"""
+      jsonStr2 shouldBe """{"innerField":{"fieldA":"fieldA value2","fieldC":true}}"""
+      jsonStr3 shouldBe """{"fieldA":"fieldA value","fieldC":true}"""
+      pickler1.toCodec.decode("""{"fieldA":"fieldA value3","fieldC":true}""") shouldBe Value(FlatClassWithOption("fieldA value3", None, true))
+      pickler1.toCodec.decode("""{"fieldA":"fieldA value4", "fieldB": null, "fieldC": true}""") shouldBe Value(FlatClassWithOption("fieldA value4", None, true))
+    }
+  }
+
+  it should "serialize Options to nulls if transientNone = false" in {
+    import generic.auto.* // for Pickler auto-derivation
+
+    // when
+    given PicklerConfiguration = PicklerConfiguration.default.withTransientNone(false)
+    val pickler = Pickler.derived[FlatClassWithOption]
+    val jsonStr1 = pickler.toCodec.encode(FlatClassWithOption("fieldA value", Some(-2545), true))
+    val jsonStr2 = pickler.toCodec.encode(FlatClassWithOption("fieldA value2", None, true))
+
+    // then
+    {
+      jsonStr1 shouldBe """{"fieldA":"fieldA value","fieldB":-2545,"fieldC":true}"""
+      jsonStr2 shouldBe """{"fieldA":"fieldA value2","fieldB":null,"fieldC":true}"""
     }
   }