Skip to content

Commit

Permalink
Add an option, allow_preencoded_submsgs
Browse files Browse the repository at this point in the history
The idea is to be able to pre-encode submessages that
occur frequently, and supply them to encoding as binaries,
to increase performance by not encoding them more than once.
  • Loading branch information
tomas-abrahamsson committed Feb 4, 2023
1 parent 5557e31 commit 2ee8064
Show file tree
Hide file tree
Showing 5 changed files with 176 additions and 17 deletions.
39 changes: 36 additions & 3 deletions src/gpb_compile.erl
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@
{maps_unset_optional, omitted | present_undefined} |
{maps_oneof, tuples | flat} |
{maps_key_type, atom | binary} |
boolean_opt(allow_preencoded_submsgs) |
%% Verification of input
{verify, optionally | always | never} |
boolean_opt(verify_decode_required_present) |
Expand Down Expand Up @@ -371,7 +372,9 @@ file(File) ->
%% <tt>{<a href="#option-maps_oneof">maps_oneof</a>,
%% tuples|flat}</tt>,
%% <tt>{<a href="#option-maps_key_type">maps_key_type</a>,
%% atom|binary}</tt>
%% atom|binary}</tt>,
%% <tt><a href="#option-allow_preencoded_submsgs"
%% >allow_preencoded_submsgs</a></tt>
%% <br/>
%% See also <tt><a href="#option-use_packages">use_packages</a></tt>.
%% </dd>
Expand All @@ -398,7 +401,7 @@ file(File) ->
%% <tt>{<a href="#option-module_name_suffix">module_name_suffix</a>,
%% {@link name_part()}}</tt>,
%% <tt>{<a href="#option-module_name">module_name</a>
%% {@link new_name()}}</tt>,
%% {@link new_name()}}</tt>
%% </dd>
%% <dt>What to generate and how</dt>
%% <dd><tt><a href="#option-use_packages">use_packages</a></tt>,
Expand Down Expand Up @@ -685,6 +688,16 @@ file(File) ->
%% Corresponding command line option:
%% <a href="#cmdline-option-maps_key_type">-maps_key_type</a>.
%%
%% <h4><a id="option-allow_preencoded_submsgs"/>`allow_preencoded_submsgs'</h4>
%%
%% Allow pre-encoded submsgs to save cpu during encoding. A sub-message
%% can then be specified as a binary. It is not possible to combine this
%% option neither with the option `nif' nor with encoding to json.
%%
%% Corresponding command line option:
%% <a href="#cmdline-option-allow-preencoded-submsgs"
%% >-allow-preencoded-submsgs</a>.
%%
%% <h4>Related options</h4>
%% <ul>
%% <li><a href="#option-use_packages">`use_packages'</a></li>
Expand Down Expand Up @@ -1865,7 +1878,8 @@ verify_opts(Defs, Opts) ->
fun() -> verify_opts_epb_compat(Defs, Opts) end,
fun() -> verify_opts_flat_oneof(Opts) end,
fun() -> verify_opts_no_gen_mergers(Opts) end,
fun() -> verify_opts_no_gen_verifiers(Opts) end]).
fun() -> verify_opts_no_gen_verifiers(Opts) end,
fun() -> verify_opts_allow_preencoded_submsgs(Opts) end]).

while_ok(Funs) ->
lists:foldl(fun(F, ok) -> F();
Expand Down Expand Up @@ -1968,6 +1982,15 @@ verify_opts_no_gen_verifiers(Opts) ->
{verify,Verify}, {gen_verifiers,false}}}
end.

verify_opts_allow_preencoded_submsgs(Opts) ->
DoNif = proplists:get_bool(nif, Opts),
AllowPreencodedSubmsgs = proplists:get_bool(allow_preencoded_submsgs, Opts),
case {DoNif, AllowPreencodedSubmsgs} of
{true, true} -> {error, {invalid_options,
nif, allow_preencoded_submsgs}};
_ -> ok
end.

%% @equiv msg_defs(Mod, Defs, [])
%% @doc Deprecated, use proto_defs/2 instead.
-spec msg_defs(module(), gpb_defs:defs()) -> comp_ret().
Expand Down Expand Up @@ -2362,6 +2385,8 @@ fmt_err({invalid_options, nif, {gen_mergers, false}}) ->
fmt_err({invalid_options, {verify,Verify}, {gen_verifiers,false}}) ->
?f("Option error: It is not possible to omit verifiers when verify = ~p",
[Verify]);
fmt_err({invalid_options, nif, allow_preencoded_submsgs}) ->
"Option error: Not supported: both allow_preencoded_submsgs and nif";
fmt_err({epb_compatibility_impossible, {with_msg_named, msg}}) ->
"Not possible to generate epb compatible functions when a message "
"is named 'msg' because of collision with the standard gpb functions "
Expand Down Expand Up @@ -2512,6 +2537,12 @@ c() ->
%% <dd>Specifies the key type for maps.<br/>
%% Corresponding Erlang-level option:
%% <a href="#option-maps_key_type">maps_key_type</a></dd>
%% <dt><a id="cmdline-option-allow-preencoded-submsgs"/>
%% `-allow-preencoded-submsgs'</dt>
%% <dd>Allow pre-encoded submsgs to save cpu during encoding<br/>
%% Corresponding Erlang-level option:
%% <a href="#option-allow_preencoded_submsgs"
%% >allow_preencoded_submsgs</a></dd>
%% </dl>
%%
%% Verification of input
Expand Down Expand Up @@ -3224,6 +3255,8 @@ opt_specs() ->
{"maps_key_type", {atom, binary}, maps_key_type,
"atom | binary\n"
" Specifies the key type for maps.\n"},
{"allow-preencoded-submsgs", undefined, allow_preencoded_submsgs, "\n"
" Allow pre-encoded submsgs to save cpu during encoding.\n"},
{{section, "Verification of inputs"}},
{"v", {optionally, always, never}, verify, " optionally | always | never\n"
" Specify how the generated encoder should\n"
Expand Down
14 changes: 12 additions & 2 deletions src/gpb_gen_encoders.erl
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ format_msg_encoder(MsgName, MsgDef, Defs, AnRes, Opts, IncludeStarter) ->
replace_tree('M', MsgVar)])
end
end,
AllowPreencodedSubmsgs = proplists:get_bool(allow_preencoded_submsgs, Opts),
[[[[[gpb_codegen:format_fn(
gpb_lib:mk_fn(encode_msg_, MsgName),
fun(Msg) ->
Expand All @@ -317,10 +318,19 @@ format_msg_encoder(MsgName, MsgDef, Defs, AnRes, Opts, IncludeStarter) ->
"\n"] || IncludeStarter],
gpb_codegen:format_fn(
FnName,
fun('<msg-matching>', Bin, TrUserData) ->
fun('Preencoded', _Bin, _TrUserData) when is_binary('Preencoded') ->
'Preencoded';
('<msg-matching>', Bin, TrUserData) ->
'<encode-param-exprs>'
end,
[replace_tree('<msg-matching>', FieldMatching),
[repeat_clauses(
'Preencoded',
if AllowPreencodedSubmsgs ->
[[replace_tree('Preencoded', gpb_lib:var("Preencoded", []))]];
not AllowPreencodedSubmsgs ->
[] % don't include this clause at all
end),
replace_tree('<msg-matching>', FieldMatching),
splice_trees('<encode-param-exprs>', EncodeExprs)])].

field_encode_expr(MsgName, MsgVar, #?gpb_field{name=FName}=Field,
Expand Down
17 changes: 12 additions & 5 deletions src/gpb_gen_types.erl
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@
map_type_fields :: '2tuples' | maps,
module :: module(),
nif :: boolean(),
verify_decode_required_present :: boolean()}).
verify_decode_required_present :: boolean(),
allow_preencoded_submsgs :: boolean()}).

-record(type_text,
{text :: string(),
Expand Down Expand Up @@ -208,14 +209,16 @@ t_env(Opts) ->
Mod = proplists:get_value(module, Opts),
Nif = proplists:get_bool(nif, Opts),
DecVfy = proplists:get_bool(verify_decode_required_present, Opts),
AllowPreencodedSubmsgs = proplists:get_bool(allow_preencoded_submsgs, Opts),
#t_env{type_specs = TypeSpecs,
can_do_map_presence = TypespecsCanIndicateMapItemPresence,
mapping_and_unset = MappingAndUnset,
map_key_type = KeyType,
map_type_fields = MapTypeFieldsRepr,
module = Mod,
nif = Nif,
verify_decode_required_present = DecVfy}.
verify_decode_required_present = DecVfy,
allow_preencoded_submsgs = AllowPreencodedSubmsgs}.

calc_keytype_override([], _TEnv) ->
no_override;
Expand Down Expand Up @@ -766,13 +769,17 @@ float_spec() ->
msg_to_typestr(M, AnRes, TEnv) ->
MsgType = rename_msg_type(M, AnRes),
#t_env{mapping_and_unset=MappingAndUnset,
module = Mod} = TEnv,
module = Mod,
allow_preencoded_submsgs=AllowPreencodedSubmsgs} = TEnv,
OrBinary = if AllowPreencodedSubmsgs -> " | binary()";
true -> ""
end,
case MappingAndUnset of
records ->
%% Prefix with module since records live in an hrl file
?f("~p:~p()", [Mod, MsgType]);
?f("~p:~p()~s", [Mod, MsgType, OrBinary]);
#maps{} ->
?f("~p()", [MsgType])
?f("~p()~s", [MsgType, OrBinary])
end.

enum_typestr(E, Defs, #t_env{nif=Nif}) ->
Expand Down
47 changes: 40 additions & 7 deletions src/gpb_gen_verifiers.erl
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,41 @@ format_verifiers(Defs, AnRes, Opts) ->
].

format_msg_verifiers(Defs, AnRes, Opts) ->
[format_msg_verifier(MsgName, MsgDef, AnRes, Opts)
|| {_Type, MsgName, MsgDef} <- gpb_lib:msgs_or_groups(Defs)].
[[case Type == msg andalso can_occur_as_submsg(MsgName, AnRes) of
true -> format_submsg_verifier_wrapper(MsgName, Opts);
false -> []
end,
format_msg_verifier(MsgName, MsgDef, AnRes, Opts)]
|| {Type, MsgName, MsgDef} <- gpb_lib:msgs_or_groups(Defs)].

can_occur_as_submsg(MsgName, #anres{used_types=UsedTypes}) ->
gpb_lib:smember({msg, MsgName}, UsedTypes).

format_submsg_verifier_wrapper(MsgName, Opts) ->
FnNameSub = gpb_lib:mk_fn(v_submsg_, MsgName),
FnName = gpb_lib:mk_fn(v_msg_, MsgName),
case proplists:get_bool(allow_preencoded_submsgs, Opts) of
true ->
[gpb_lib:nowarn_unused_function(FnNameSub, 3),
gpb_lib:nowarn_dialyzer_attr(FnNameSub, 3,Opts),
gpb_codegen:format_fn(
FnNameSub,
fun(Preencoded, _Path, _TrUserData) when is_binary(Preencoded) ->
ok;
(Msg, Path, TrUserData) ->
'FnName'(Msg, Path, TrUserData)
end,
[replace_term('FnName', FnName)])];
false ->
[gpb_lib:nowarn_unused_function(FnNameSub, 3),
gpb_lib:nowarn_dialyzer_attr(FnNameSub, 3,Opts),
gpb_codegen:format_fn(
FnNameSub,
fun(Msg, Path, TrUserData) ->
'FnName'(Msg, Path, TrUserData)
end,
[replace_term('FnName', FnName)])]
end.

format_msg_verifier(MsgName, MsgDef0, AnRes, Opts) ->
MsgDef1 = drop_field_for_unknown_if_present(MsgDef0),
Expand Down Expand Up @@ -254,7 +287,7 @@ field_verifier(MsgName,
FVar, MsgVar, TrUserDataVar, AnRes, Opts) ->
FVerifierFn =
case Type of
{msg,FMsgName} -> gpb_lib:mk_fn(v_msg_, FMsgName);
{msg,FMsgName} -> gpb_lib:mk_fn(v_submsg_, FMsgName);
{group,GName} -> gpb_lib:mk_fn(v_msg_, GName);
{enum,EnumName} -> gpb_lib:mk_fn(v_enum_, EnumName);
{map,KT,VT} -> gpb_lib:mk_fn(v_, gpb_lib:map_type_to_msg_name(
Expand Down Expand Up @@ -524,7 +557,7 @@ field_oneof_present_undefined_verifier(MsgName, FName, OFields,
FVerifierFn =
case Type of
{msg,FMsgName} ->
gpb_lib:mk_fn(v_msg_, FMsgName);
gpb_lib:mk_fn(v_submsg_, FMsgName);
{enum,EnumName} ->
gpb_lib:mk_fn(v_enum_, EnumName);
Type ->
Expand Down Expand Up @@ -581,7 +614,7 @@ field_oneof_omitted_tuples_verifier(MsgName, FName, OFields,
[begin
FVerifierFn =
case Type of
{msg,FMsgName} -> gpb_lib:mk_fn(v_msg_, FMsgName);
{msg,FMsgName} -> gpb_lib:mk_fn(v_submsg_, FMsgName);
{enum,EnumName} -> gpb_lib:mk_fn(v_enum_, EnumName);
Type -> gpb_lib:mk_fn(v_type_, Type)
end,
Expand Down Expand Up @@ -645,7 +678,7 @@ field_oneof_omitted_flat_verifier(MsgName, FName, OFields,
[begin
FVerifierFn =
case Type of
{msg,FMsgName} -> gpb_lib:mk_fn(v_msg_, FMsgName);
{msg,FMsgName} -> gpb_lib:mk_fn(v_submsg_, FMsgName);
{enum,EnumName} -> gpb_lib:mk_fn(v_enum_, EnumName);
Type -> gpb_lib:mk_fn(v_type_, Type)
end,
Expand Down Expand Up @@ -817,7 +850,7 @@ format_map_verifier(KeyType, ValueType, MapsOrTuples, AnRes, Opts) ->
FnName = gpb_lib:mk_fn(v_, MsgName),
KeyVerifierFn = gpb_lib:mk_fn(v_type_, KeyType),
ValueVerifierFn1 = case ValueType of
{msg,FMsgName} -> gpb_lib:mk_fn(v_msg_, FMsgName);
{msg,FMsgName} -> gpb_lib:mk_fn(v_submsg_,FMsgName);
{enum,EnumName} -> gpb_lib:mk_fn(v_enum_, EnumName);
Type -> gpb_lib:mk_fn(v_type_, Type)
end,
Expand Down
76 changes: 76 additions & 0 deletions test/gpb_compile_tests.erl
Original file line number Diff line number Diff line change
Expand Up @@ -2400,6 +2400,77 @@ gen_verifiers_test() ->
unload_code(M),
ok.

%% --- pre-encoded submsgs during encoding ------------

preencoded_msg_with_verify_test() ->
M = compile_iolist(["message Top { optional Sub s = 1; }
message Sub { required uint32 a = 1; }
"],
[allow_preencoded_submsgs]),
Top = M:encode_msg({'Top', {'Sub', 17}}),
Sub = M:encode_msg({'Sub', 17}),
ok = M:verify_msg({'Top', Sub}),
Top = M:encode_msg({'Top', Sub}, [{verify, true}]),
unload_code(M).

preencoded_msg_with_no_fields_test() ->
M = compile_iolist(["message Top { optional Sub s = 1; }
message Sub { }
"],
[allow_preencoded_submsgs]),
Top = M:encode_msg({'Top', {'Sub'}}),
Sub = M:encode_msg({'Sub'}),
ok = M:verify_msg({'Top', Sub}),
Top = M:encode_msg({'Top', Sub}, [{verify, true}]),
unload_code(M).

preencoded_msg_in_oneof_test() ->
M = compile_iolist(["message Top { oneof c { Sub s = 1; } }
message Sub { required uint32 a = 1; }
"],
[allow_preencoded_submsgs]),
Top = M:encode_msg({'Top', {s, {'Sub', 17}}}),
Sub = M:encode_msg({'Sub', 17}),
ok = M:verify_msg({'Top', {s, Sub}}),
Top = M:encode_msg({'Top', {s, Sub}}),
unload_code(M).

preencoded_msg_in_maptype_field_test() ->
M = compile_iolist(["message Top { map<uint32, Sub> m = 1; }
message Sub { required uint32 a = 1; }
"],
[allow_preencoded_submsgs]),
Top = M:encode_msg({'Top', [{42, {'Sub', 17}}]}),
Sub = M:encode_msg({'Sub', 17}),
ok = M:verify_msg({'Top', [{42, Sub}]}),
Top = M:encode_msg({'Top', [{42, Sub}]}),
unload_code(M).

preencoded_msg_with_repeated_test() ->
M = compile_iolist(["message Top { repeated Sub r = 1; }
message Sub { required uint32 a = 1; }
"],
[allow_preencoded_submsgs]),
Top = M:encode_msg({'Top', [{'Sub', 17}, {'Sub', 18}]}),
Sub17 = M:encode_msg({'Sub', 17}),
Sub18 = M:encode_msg({'Sub', 18}),
ok = M:verify_msg({'Top', [Sub17, Sub18]}),
Top = M:encode_msg({'Top', [Sub17, Sub18]}),
unload_code(M).

-ifndef(NO_HAVE_MAPS).
preencoded_msg_with_maps_test() ->
M = compile_iolist(["message Top { optional Sub s = 1; }
message Sub { required uint32 a = 1; }
"],
[allow_preencoded_submsgs, maps]),
Top = M:encode_msg(#{s => #{a => 17}}, 'Top'),
Sub = M:encode_msg(#{a => 17}, 'Sub'),
ok = M:verify_msg(#{s => Sub}, 'Top'),
Top = M:encode_msg(#{s => Sub}, 'Top', [{verify, true}]),
unload_code(M).
-endif. % -ifndef(NO_HAVE_MAPS).

%% --- locate_import and read_import ----------

read_import_test() ->
Expand Down Expand Up @@ -5431,6 +5502,11 @@ no_gen_mergers_test() ->
gpb_compile:parse_opts_and_args(["-nif", "-no-gen-mergers",
"x.proto"]).

allow_preencoded_submsgs_options_test() ->
{ok, {[allow_preencoded_submsgs], ["x.proto"]}} =
gpb_compile:parse_opts_and_args(["-allow-preencoded-submsgs",
"x.proto"]).

no_gen_intospections_test() ->
{ok, {[{gen_introspect, false}], ["x.proto"]}} =
gpb_compile:parse_opts_and_args(["-no-gen-introspect", "x.proto"]),
Expand Down

0 comments on commit 2ee8064

Please sign in to comment.