Implement Proto3 Field Presence

Closes ahamez#49

lib/protox/define_decoder.ex

@@ -142,9 +142,9 @@ defmodule Protox.DefineDecoder do
     # Fragment to parse known fields.
     known_tags_case =
       fields
-      |> Enum.map(fn {tag, _, name, kind, type} ->
-        single = make_single_case(msg_var, keep_set_fields, tag, name, kind, type)
-        delimited = make_delimited_case(msg_var, keep_set_fields, single, tag, name, kind, type)
+      |> Enum.map(fn field ->
+        single = make_single_case(msg_var, keep_set_fields, field)
+        delimited = make_delimited_case(msg_var, keep_set_fields, single, field)
 
         delimited ++ single
       end)
@@ -175,25 +175,25 @@ defmodule Protox.DefineDecoder do
     end
   end
 
-  defp make_single_case(_msg_var, _keep_set_fields, _tag, _name, _kind, {:message, _}),
+  defp make_single_case(_msg_var, _keep_set_fields, {_tag, _label, _name, _kind, {:message, _}}),
     do: quote(do: [])
 
-  defp make_single_case(_msg_var, _keep_set_fields, _tag, _name, _kind, :string),
+  defp make_single_case(_msg_var, _keep_set_fields, {_tag, _label, _name, _kind, :string}),
     do: quote(do: [])
 
-  defp make_single_case(_msg_var, _keep_set_fields, _tag, _name, _kind, :bytes),
+  defp make_single_case(_msg_var, _keep_set_fields, {_tag, _label, _name, _kind, :bytes}),
     do: quote(do: [])
 
-  defp make_single_case(_msg_var, _keep_set_fields, _tag, _name, _kind, {x, _})
+  defp make_single_case(_msg_var, _keep_set_fields, {_tag, _label, _name, _kind, {x, _}})
        when x != :enum,
        do: quote(do: [])
 
-  defp make_single_case(msg_var, keep_set_fields, tag, name, kind, type) do
+  defp make_single_case(msg_var, keep_set_fields, field = {tag, _label, name, _kind, type}) do
     bytes_var = quote do: bytes
     field_var = quote do: field
     value_var = quote do: value
     parse_single = make_parse_single(bytes_var, type)
-    update_field = make_update_field(name, kind, type, msg_var, value_var)
+    update_field = make_update_field(field, msg_var, value_var)
 
     # No need to maintain a list of set fields for proto3
     case_return =
@@ -214,35 +214,48 @@ defmodule Protox.DefineDecoder do
          msg_var,
          keep_set_fields,
          single,
-         tag,
-         name,
-         kind,
-         type = {:message, _}
+         field = {
+           _tag,
+           _label,
+           _name,
+           _kind,
+           {:message, _}
+         }
        ) do
-    make_delimited_case_impl(msg_var, keep_set_fields, single, tag, name, kind, type)
+    make_delimited_case_impl(msg_var, keep_set_fields, single, field)
   end
 
-  defp make_delimited_case(msg_var, keep_set_fields, single, tag, name, kind, :bytes) do
-    make_delimited_case_impl(msg_var, keep_set_fields, single, tag, name, kind, :bytes)
+  defp make_delimited_case(msg_var, keep_set_fields, single, field = {_, _, _, _, :bytes}) do
+    make_delimited_case_impl(msg_var, keep_set_fields, single, field)
   end
 
-  defp make_delimited_case(msg_var, keep_set_fields, single, tag, name, kind, :string) do
-    make_delimited_case_impl(msg_var, keep_set_fields, single, tag, name, kind, :string)
+  defp make_delimited_case(msg_var, keep_set_fields, single, field = {_, _, _, _, :string}) do
+    make_delimited_case_impl(msg_var, keep_set_fields, single, field)
   end
 
-  defp make_delimited_case(_msg_var, _keep_set_fields, _single, _tag, _name, {:default, _}, _) do
+  defp make_delimited_case(
+         _msg_var,
+         _keep_set_fields,
+         _single,
+         {_tag, _label, _name, {:default, _}, _}
+       ) do
     []
   end
 
-  defp make_delimited_case(msg_var, keep_set_fields, single, tag, name, kind, type) do
-    make_delimited_case_impl(msg_var, keep_set_fields, single, tag, name, kind, type)
+  defp make_delimited_case(msg_var, keep_set_fields, single, field) do
+    make_delimited_case_impl(msg_var, keep_set_fields, single, field)
   end
 
-  defp make_delimited_case_impl(msg_var, keep_set_fields, single, tag, name, kind, type) do
+  defp make_delimited_case_impl(
+         msg_var,
+         keep_set_fields,
+         single,
+         field = {tag, _label, name, _kind, type}
+       ) do
     bytes_var = quote do: bytes
     field_var = quote do: field
     value_var = quote do: value
-    update_field = make_update_field(name, kind, type, msg_var, value_var)
+    update_field = make_update_field(field, msg_var, value_var)
 
     case_return =
       case keep_set_fields do
@@ -271,14 +284,14 @@ defmodule Protox.DefineDecoder do
     end
   end
 
-  defp make_update_field(name, :map, _type, msg_var, value_var) do
+  defp make_update_field({_, _, name, :map, _}, msg_var, value_var) do
     quote do
       {entry_key, entry_value} = unquote(value_var)
       {unquote(name), Map.put(unquote(msg_var).unquote(name), entry_key, entry_value)}
     end
   end
 
-  defp make_update_field(name, {:oneof, parent_field}, {:message, _}, msg_var, value_var) do
+  defp make_update_field({_, _, name, {:oneof, parent_field}, {:message, _}}, msg_var, value_var) do
     quote do
       case unquote(msg_var).unquote(parent_field) do
         {unquote(name), previous_value} ->
@@ -291,21 +304,27 @@ defmodule Protox.DefineDecoder do
     end
   end
 
-  defp make_update_field(name, {:oneof, parent_field}, _type, _msg_var, value_var) do
-    quote(do: {unquote(parent_field), {unquote(name), unquote(value_var)}})
+  defp make_update_field({_, label, name, {:oneof, parent_field}, _}, _msg_var, value_var) do
+    case label do
+      :proto3_optional ->
+        quote(do: {unquote(name), unquote(value_var)})
+
+      _ ->
+        quote(do: {unquote(parent_field), {unquote(name), unquote(value_var)}})
+    end
   end
 
-  defp make_update_field(name, {:default, _}, {:message, _}, msg_var, value_var) do
+  defp make_update_field({_, _, name, {:default, _}, {:message, _}}, msg_var, value_var) do
     quote do
       {unquote(name), Protox.Message.merge(unquote(msg_var).unquote(name), unquote(value_var))}
     end
   end
 
-  defp make_update_field(name, {:default, _}, _, _msg_var, value_var) do
+  defp make_update_field({_, _, name, {:default, _}, _}, _msg_var, value_var) do
     quote(do: {unquote(name), unquote(value_var)})
   end
 
-  defp make_update_field(name, _kind, _type, msg_var, value_var) do
+  defp make_update_field({_, _, name, _kind, _type}, msg_var, value_var) do
     quote do
       {unquote(name), unquote(msg_var).unquote(name) ++ List.wrap(unquote(value_var))}
     end

lib/protox/define_encoder.ex

@@ -87,6 +87,12 @@ defmodule Protox.DefineEncoder do
     make_encode_oneof_fun(ast, oneofs)
   end
 
+  defp parent_name(_, [{_, :proto3_optional, _, {_, name}, _}]), do: name
+  defp parent_name(name, _), do: name
+
+  defp parent_data_key(_, [{_, :proto3_optional, child_name, _, _}]), do: child_name
+  defp parent_data_key(parent_name, _), do: parent_name
+
   defp make_encode_oneof_funs(oneofs) do
     for {parent_name, children} <- oneofs do
       nil_case =
@@ -106,11 +112,13 @@ defmodule Protox.DefineEncoder do
            end)
            |> List.flatten())
 
+      parent_name = parent_name(parent_name, children)
       encode_parent_fun_name = String.to_atom("encode_#{parent_name}")
+      parent_data_key = parent_data_key(parent_name, children)
 
       quote do
         defp unquote(encode_parent_fun_name)(acc, msg) do
-          case msg.unquote(parent_name) do
+          case msg.unquote(parent_data_key) do
             unquote(children_case_ast)
           end
         end
@@ -119,18 +127,18 @@ defmodule Protox.DefineEncoder do
   end
 
   defp make_encode_field_funs(fields, required_fields, syntax) do
-    for {tag, _, name, kind, type} <- fields do
+    for {_, _, name, _, _} = field <- fields do
       required = name in required_fields
       fun_name = String.to_atom("encode_#{name}")
-      fun_ast = make_encode_field_body(kind, tag, name, type, required, syntax)
+      fun_ast = make_encode_field_body(field, required, syntax)
 
       quote do
         defp unquote(fun_name)(acc, msg), do: unquote(fun_ast)
       end
     end
   end
 
-  defp make_encode_field_body({:default, default}, tag, name, type, required, syntax) do
+  defp make_encode_field_body({tag, _, name, {:default, default}, type}, required, syntax) do
     key = Protox.Encode.make_key_bytes(tag, type)
     var = quote do: field_value
     encode_value_ast = get_encode_value_body(type, var)
@@ -170,20 +178,41 @@ defmodule Protox.DefineEncoder do
   end
 
   # Generate the AST to encode child `_child_name` of oneof `parent_field`
-  defp make_encode_field_body({:oneof, parent_field}, tag, _child_name, type, _required, _syntax) do
+  defp make_encode_field_body(
+         {tag, label, child_name, {:oneof, parent_field}, type},
+         _required,
+         _syntax
+       ) do
     key = Protox.Encode.make_key_bytes(tag, type)
     var = quote do: field_value
     encode_value_ast = get_encode_value_body(type, var)
 
-    # The dispatch on the correct child is performed by the parent encoding function,
-    # this is why we don't check if the child is set.
-    quote do
-      {_, unquote(var)} = msg.unquote(parent_field)
-      [acc, unquote(key), unquote(encode_value_ast)]
+    case label do
+      :proto3_optional ->
+        quote do
+          case msg.unquote(child_name) do
+            {_, unquote(var)} ->
+              [acc, unquote(key), unquote(encode_value_ast)]
+
+            unquote(var) when not is_nil(unquote(var)) ->
+              [acc, unquote(key), unquote(encode_value_ast)]
+
+            _ ->
+              [acc]
+          end
+        end
+
+      _ ->
+        # The dispatch on the correct child is performed by the parent encoding function,
+        # this is why we don't check if the child is set.
+        quote do
+          {_, unquote(var)} = msg.unquote(parent_field)
+          [acc, unquote(key), unquote(encode_value_ast)]
+        end
     end
   end
 
-  defp make_encode_field_body(:packed, tag, name, type, _required, _syntax) do
+  defp make_encode_field_body({tag, _label, name, :packed, type}, _required, _syntax) do
     key = Protox.Encode.make_key_bytes(tag, :packed)
     encode_packed_ast = make_encode_packed_body(type)
 
@@ -195,7 +224,7 @@ defmodule Protox.DefineEncoder do
     end
   end
 
-  defp make_encode_field_body(:unpacked, tag, name, type, _required, _syntax) do
+  defp make_encode_field_body({tag, _label, name, :unpacked, type}, _required, _syntax) do
     encode_repeated_ast = make_encode_repeated_body(tag, type)
 
     quote do
@@ -206,7 +235,7 @@ defmodule Protox.DefineEncoder do
     end
   end
 
-  defp make_encode_field_body(:map, tag, name, type, _required, _syntax) do
+  defp make_encode_field_body({tag, _label, name, :map, type}, _required, _syntax) do
     # Each key/value entry of a map has the same layout as a message.
     # https://developers.google.com/protocol-buffers/docs/proto3#backwards-compatibility
 

lib/protox/define_message.ex

@@ -118,10 +118,10 @@ defmodule Protox.DefineMessage do
   # Generate fields of the struct which is created for a message.
   defp make_struct_fields(fields, syntax, unknown_fields, keep_unknown_fields) do
     struct_fields =
-      for {_, _, name, kind, _} <- fields do
+      for {_, label, name, kind, _} <- fields do
         case kind do
           :map -> {name, Macro.escape(%{})}
-          {:oneof, parent} -> {parent, nil}
+          {:oneof, parent} -> make_oneof_field(label, name, parent)
           :packed -> {name, []}
           :unpacked -> {name, []}
           {:default, _} when syntax == :proto2 -> {name, nil}
@@ -138,6 +138,9 @@ defmodule Protox.DefineMessage do
     Enum.uniq(struct_fields)
   end
 
+  defp make_oneof_field(:proto3_optional, name, _), do: {name, nil}
+  defp make_oneof_field(_, _, parent), do: {parent, nil}
+
   # Get the list of fields that are marked as `required`.
   defp make_required_fields(fields) do
     for {_, :required, name, _, _} <- fields, do: name

lib/protox/defs.ex

@@ -11,7 +11,11 @@ defmodule Protox.Defs do
       end)
 
     {
-      oneofs |> Enum.group_by(fn {_, _, _, {:oneof, parent}, _} -> parent end) |> Map.to_list(),
+      oneofs
+      |> Enum.group_by(fn {_, label, name, {:oneof, parent}, _} ->
+        oneof_groupby(label, name, parent)
+      end)
+      |> Map.to_list(),
       fields
     }
   end
@@ -22,4 +26,7 @@ defmodule Protox.Defs do
       _ -> false
     end)
   end
+
+  defp oneof_groupby(:proto3_optional, name, _parent), do: name
+  defp oneof_groupby(_, _, parent), do: parent
 end

lib/protox/descriptor.ex

@@ -106,7 +106,8 @@ defmodule Protox.Descriptor do
           {2, :none, :extendee, {:default, nil}, :string},
           {7, :none, :default_value, {:default, nil}, :string},
           {9, :none, :oneof_index, {:default, nil}, :int32},
-          {8, :none, :options, {:default, nil}, {:message, Protox.Google.Protobuf.FieldOptions}}
+          {8, :none, :options, {:default, nil}, {:message, Protox.Google.Protobuf.FieldOptions}},
+          {17, :none, :proto3_optional, {:default, false}, :bool}
         ]
       },
       {

lib/protox/parse.ex

@@ -210,7 +210,7 @@ defmodule Protox.Parse do
         nil ->
           type = get_type(descriptor)
           kind = get_kind(syntax, upper, descriptor)
-          {descriptor.label, kind, type}
+          {field_label(descriptor), kind, type}
 
         map_type ->
           {nil, :map, map_type}
@@ -224,6 +224,9 @@ defmodule Protox.Parse do
     }
   end
 
+  defp field_label(%{proto3_optional: true}), do: :proto3_optional
+  defp field_label(%{label: label}), do: label
+
   defp map_entry(nil, _, _), do: nil
 
   defp map_entry(upper, prefix, descriptor) do

test/example_test.exs

@@ -21,6 +21,9 @@ defmodule ExampleTest do
         FooOrBar g = 7;
         repeated int32 h = 8;
         map<string, int32> i = 9;
+        optional bool j = 10;
+        optional bool k = 11;
+        optional bool l = 12;
       }
 
       message Envelope {
@@ -45,7 +48,9 @@ defmodule ExampleTest do
            f: true,
            g: :FOO,
            h: [1, 2, 3],
-           i: %{"foo" => 42, "bar" => 33}
+           i: %{"foo" => 42, "bar" => 33},
+           k: false,
+           l: true
          }}
     }
 
@@ -55,7 +60,9 @@ defmodule ExampleTest do
     }
 
     encoded_sub_msg = sub_msg |> Envelope.encode!() |> :binary.list_to_bin()
-    assert Envelope.decode!(encoded_sub_msg) == sub_msg
+    decoded_sub_msg = Envelope.decode!(encoded_sub_msg)
+    assert decoded_sub_msg == sub_msg
+    assert %{envelope: {:sub_msg, %SubMsg{j: nil, k: false, l: true}}} = decoded_sub_msg
 
     encoded_str = str |> Envelope.encode!() |> :binary.list_to_bin()
     assert Envelope.decode!(encoded_str) == str