diff --git a/piqilib/piq.ml b/piqilib/piq.ml index d9990d4f..d6fe5fe4 100644 --- a/piqilib/piq.ml +++ b/piqilib/piq.ml @@ -205,7 +205,9 @@ let rec to_portable_ast (ast:piq_ast) :Piq_piqi.piq_node = `typed Piq_piqi.Typed.({typename = typename; value = to_portable_ast value}) | `list l -> `list (List.map to_portable_ast l) - (* these shouldn't be used for plain Piq ASTs *) + | `form (`name name, args) -> + `splice {Piq_piqi.Splice.name = name; Piq_piqi.Splice.item = List.map to_portable_ast args} + (* XXX, TODO: any shouldn't be used in plain Piq ASTs? *) | `form _ -> assert false | `any _ -> assert false in @@ -213,6 +215,8 @@ let rec to_portable_ast (ast:piq_ast) :Piq_piqi.piq_node = let addloc loc ast = + (* TODO: this is not enough -- need to add location to the option values + * themselves *) Piqloc.setloc loc; (match ast with | `int (x, _) -> Piqloc.add x @@ -232,39 +236,47 @@ let addloc loc ast = Piqloc.add typename | `list l -> Piqloc.add l - (* these shouldn't be used for plain Piq ASTs *) - | `form _ -> assert false + | `form ((`name s) as name, args) -> + Piqloc.add s; + Piqloc.add name + (* XXX, TODO: any shouldn't be used in plain Piq ASTs? *) + | `form (_, _) -> assert false | `any _ -> assert false ); Piqloc.add ast let rec of_portable_ast (piq_node:Piq_piqi.piq_node) :piq_ast = - let ast = - match piq_node.Piq_piqi.Piq_node.piq with - | `int x -> `int (x, "") - | `uint x -> `uint (x, "") - | `float x -> `float (x, "") - | `bool x -> `bool x - | `string x -> `string (x, "") - | `raw_string x -> `raw_string x - | `word x -> `word x - | `text x -> `text x - | `binary x -> `binary (x, "") - | `name x -> `name x - | `typename x -> `typename x - | `named {Piq_piqi.Named.name = name; Piq_piqi.Named.value = value} -> - `named Piq_ast.Named.({name = name; value = of_portable_ast value}) - | `typed {Piq_piqi.Typed.typename = typename; Piq_piqi.Typed.value = value} -> - `typed Piq_ast.Typed.({typename = typename; value = of_portable_ast value}) - | `list l -> - `list (List.map of_portable_ast l) + let rec aux piq_node = + let ast = + match piq_node.Piq_piqi.Piq_node.piq with + | `int x -> `int (x, "") + | `uint x -> `uint (x, "") + | `float x -> `float (x, "") + | `bool x -> `bool x + | `string x -> `string (x, "") + | `raw_string x -> `raw_string x + | `word x -> `word x + | `text x -> `text x + | `binary x -> `binary (x, "") + | `name x -> `name x + | `typename x -> `typename x + | `named {Piq_piqi.Named.name = name; Piq_piqi.Named.value = value} -> + `named Piq_ast.Named.({name = name; value = aux value}) + | `typed {Piq_piqi.Typed.typename = typename; Piq_piqi.Typed.value = value} -> + `typed Piq_ast.Typed.({typename = typename; value = aux value}) + | `list l -> + `list (List.map aux l) + | `splice {Piq_piqi.Splice.name = name; Piq_piqi.Splice.item = items} -> + `form (`name name, List.map aux items) + in + let open Piq_piqi.Loc in + (match piq_node.Piq_piqi.Piq_node.loc with + | None -> () + | Some {file = file; line = line; column = column} -> + let loc = (file, line, column) in + addloc loc ast + ); + ast in - let open Piq_piqi.Loc in - (match piq_node.Piq_piqi.Piq_node.loc with - | None -> () - | Some {file = file; line = line; column = column} -> - let loc = (file, line, column) in - addloc loc ast - ); - ast + aux piq_node diff --git a/piqilib/piq.piqi b/piqilib/piq.piqi index f35ace34..5dde11c4 100644 --- a/piqilib/piq.piqi +++ b/piqilib/piq.piqi @@ -48,6 +48,9 @@ .name list .type piq-list ] + .option [ + .type splice + ] ] @@ -106,6 +109,18 @@ ] +.record [ + .name splice + + .field [ .type name ] + .field [ + .name item + .type piq-node + .repeated + ] +] + + .record [ .name typed diff --git a/piqilib/piq_parser.ml b/piqilib/piq_parser.ml index e2dce17a..b12e2586 100644 --- a/piqilib/piq_parser.ml +++ b/piqilib/piq_parser.ml @@ -180,7 +180,7 @@ let expand_names (x: piq_ast) :piq_ast = | `list l -> `list (List.map aux l) | `form (name, args) -> - (* at this stage, after we've run expand_forms, this can not be a + (* at this stage, after we've run expand_splices, this can not be a * named or typed form, so leaving name without a transformation *) `form (name, List.map aux args) | _ -> @@ -220,8 +220,8 @@ let cons_named_or_typed name v = cons_typed n v -(* expand named and typed forms *) -let expand_forms (x: piq_ast) :piq_ast = +(* expand named and typed splices *) +let expand_splices (x: piq_ast) :piq_ast = let rec aux0 obj = match obj with | `form (name, args) when args = [] -> @@ -283,13 +283,13 @@ let expand_forms (x: piq_ast) :piq_ast = (* expand built-in syntax abbreviations *) let expand x = - (* expand (...) when possible *) - let x = expand_forms x in + (* expand (.foo ...) or .foo* [ ... ] when possible *) + let x = expand_splices x in (* expand multi-component names *) let x = expand_names x in (* (* check if expansion produces correct location bindings *) - let x = expand_forms x in + let x = expand_splices x in let x = expand_names x in *) x @@ -377,7 +377,7 @@ let parse_number s = * a simple piq parser *) -let read_next ?(expand_abbr=true) ?(skip_trailing_comma=false) (fname, lexstream) = +let read_next ?(skip_trailing_comma=false) (fname, lexstream) = let location = ref (0,0) in let loc () = let line, col = !location in @@ -586,11 +586,7 @@ let read_next ?(expand_abbr=true) ?(skip_trailing_comma=false) (fname, lexstream let ast = parse_common t in (* skip an optional trailing comma *) if skip_trailing_comma && peek_token () = L.Comma then junk_token (); - let res = - if expand_abbr - then expand ast (* expand built-in syntax abbreviations *) - else ast (* return as it is *) - in Some res + Some ast in try parse_top () with L.Error (s, (line, col)) -> @@ -598,9 +594,9 @@ let read_next ?(expand_abbr=true) ?(skip_trailing_comma=false) (fname, lexstream error_at (fname, line, col) s -let read_all ?(expand_abbr=true) piq_parser = +let read_all piq_parser = let rec aux accu = - match read_next piq_parser ~expand_abbr ~skip_trailing_comma:true with + match read_next piq_parser ~skip_trailing_comma:true with | None -> List.rev accu | Some x -> aux (x::accu) diff --git a/piqilib/piq_piqi.ml b/piqilib/piq_piqi.ml index c6c402f5..69d43aa2 100644 --- a/piqilib/piq_piqi.ml +++ b/piqilib/piq_piqi.ml @@ -24,11 +24,13 @@ module rec Piq_piqi: | `typename of Piq_piqi.name | `typed of Piq_piqi.typed | `list of Piq_piqi.piq_list + | `splice of Piq_piqi.splice ] type piq_node = Piq_node.t type loc = Loc.t type piq_list = Piq_piqi.piq_node list type named = Named.t + type splice = Splice.t type typed = Typed.t end = Piq_piqi and Piq_node: @@ -53,6 +55,13 @@ and Named: mutable value: Piq_piqi.piq_node; } end = Named +and Splice: + sig + type t = { + mutable name: Piq_piqi.name; + mutable item: Piq_piqi.piq_node list; + } + end = Splice and Typed: sig type t = { @@ -129,6 +138,9 @@ and parse_piq x = | 14 -> let res = parse_piq_list x in `list res + | 15 -> + let res = parse_splice x in + `splice res | _ -> Piqirun.error_variant x code and parse_piq_node x = @@ -171,6 +183,16 @@ and parse_named x = Named.value = _value; } +and parse_splice x = + let x = Piqirun.parse_record x in + let _name, x = Piqirun.parse_required_field 1 parse_name x in + let _item, x = Piqirun.parse_repeated_field 2 parse_piq_node x in + Piqirun.check_unparsed_fields x; + { + Splice.name = _name; + Splice.item = _item; + } + and parse_typed x = let x = Piqirun.parse_record x in let _typename, x = Piqirun.parse_required_field 1 parse_name x in @@ -220,6 +242,7 @@ and gen__piq code (x:Piq_piqi.piq) = | `typename x -> gen__name 12 x | `typed x -> gen__typed 13 x | `list x -> gen__piq_list 14 x + | `splice x -> gen__splice 15 x )] and gen__piq_node code x = @@ -244,6 +267,11 @@ and gen__named code x = let _value = Piqirun.gen_required_field 2 gen__piq_node x.Named.value in Piqirun.gen_record code (_name :: _value :: []) +and gen__splice code x = + let _name = Piqirun.gen_required_field 1 gen__name x.Splice.name in + let _item = Piqirun.gen_repeated_field 2 gen__piq_node x.Splice.item in + Piqirun.gen_record code (_name :: _item :: []) + and gen__typed code x = let _typename = Piqirun.gen_required_field 1 gen__name x.Typed.typename in let _value = Piqirun.gen_required_field 2 gen__piq_node x.Typed.value in @@ -265,6 +293,7 @@ let gen_word x = gen__word (-1) x let gen_name x = gen__name (-1) x let gen_piq_list x = gen__piq_list (-1) x let gen_named x = gen__named (-1) x +let gen_splice x = gen__splice (-1) x let gen_typed x = gen__typed (-1) x @@ -296,6 +325,11 @@ and default_named () = Named.name = default_name (); Named.value = default_piq_node (); } +and default_splice () = + { + Splice.name = default_name (); + Splice.item = []; + } and default_typed () = { Typed.typename = default_name (); diff --git a/piqilib/piqi.ml b/piqilib/piqi.ml index dbb9032d..4b143846 100644 --- a/piqilib/piqi.ml +++ b/piqilib/piqi.ml @@ -685,8 +685,8 @@ let check_defs ~piqi idtable defs = let read_piqi_common fname piq_parser :piq_ast = - (* don't expand abbreviations until we construct the containing object *) - let res = Piq_parser.read_all piq_parser ~expand_abbr:false in + (* NOTE: not expanding abbreviations until we construct the containing object *) + let res = Piq_parser.read_all piq_parser in if res = [] then piqi_warning ("piqi file is empty: " ^ fname); @@ -967,7 +967,7 @@ let apply_extensions obj obj_def obj_parse_f obj_gen_f extension_entries custom_ (* Piqloc.trace := false; *) debug "apply_extensions(0)\n"; let obj_ast = mlobj_to_ast obj_def obj_gen_f obj in - let extension_asts = List.map Piqobj.piq_of_piqi_any extension_entries in + let extension_asts = List.map (fun x -> Piq_parser.expand (Piqobj.piq_of_piqi_any x)) extension_entries in let override l = if not override diff --git a/piqilib/piqi_boot.ml b/piqilib/piqi_boot.ml index 472dbcac..95a316e9 100644 --- a/piqilib/piqi_boot.ml +++ b/piqilib/piqi_boot.ml @@ -2331,7 +2331,12 @@ let piq = typename = Some "piq-list"; deprecated = false; piqtype = None; unparsed_piq_ast = None; piq_format = None; piq_alias = None; protobuf_name = None; code = None; json_name = None; - getopt_letter = None; getopt_doc = None; proto_name = None}]; + getopt_letter = None; getopt_doc = None; proto_name = None}; + {Piqi_impl_piqi.Option.name = None; typename = Some "splice"; + deprecated = false; piqtype = None; unparsed_piq_ast = None; + piq_format = None; piq_alias = None; protobuf_name = None; + code = None; json_name = None; getopt_letter = None; + getopt_doc = None; proto_name = None}]; parent = None; is_func_param = false; unparsed_piq_ast = None; protobuf_name = None; protobuf_custom = []; json_name = None; proto_custom = []; proto_name = None}; @@ -2427,6 +2432,28 @@ let piq = piq_allow_unnesting = None; protobuf_name = None; protobuf_custom = []; json_name = None; proto_custom = []; proto_name = None}; + `record + {Piqi_impl_piqi.Record.name = Some "splice"; + field = + [{Piqi_impl_piqi.Field.name = None; typename = Some "name"; + mode = `required; default = None; deprecated = false; + piqtype = None; unparsed_piq_ast = None; piq_format = None; + piq_positional = None; piq_alias = None; protobuf_name = None; + code = None; protobuf_packed = false; json_name = None; + json_omit_missing = None; getopt_letter = None; getopt_doc = None; + proto_name = None; wire_packed = false}; + {Piqi_impl_piqi.Field.name = Some "item"; + typename = Some "piq-node"; mode = `repeated; default = None; + deprecated = false; piqtype = None; unparsed_piq_ast = None; + piq_format = None; piq_positional = None; piq_alias = None; + protobuf_name = None; code = None; protobuf_packed = false; + json_name = None; json_omit_missing = None; getopt_letter = None; + getopt_doc = None; proto_name = None; wire_packed = false}]; + parent = None; wire_field = []; is_func_param = false; + unparsed_piq_ast = None; piq_positional = None; + piq_allow_unnesting = None; protobuf_name = None; + protobuf_custom = []; json_name = None; proto_custom = []; + proto_name = None}; `record {Piqi_impl_piqi.Record.name = Some "typed"; field = diff --git a/piqilib/piqi_convert.ml b/piqilib/piqi_convert.ml index 4a09a757..363d2eb5 100644 --- a/piqilib/piqi_convert.ml +++ b/piqilib/piqi_convert.ml @@ -179,6 +179,7 @@ let load_piq_from_ast (user_piqtype: T.piqtype option) ast :obj = let load_piq (user_piqtype: T.piqtype option) ?(skip_trailing_comma=false) piq_parser :obj = let ast = read_piq_ast piq_parser user_piqtype ~skip_trailing_comma in + let ast = Piq_parser.expand ast in load_piq_from_ast user_piqtype ast diff --git a/piqilib/piqi_getopt.ml b/piqilib/piqi_getopt.ml index 1a60b674..491cd870 100644 --- a/piqilib/piqi_getopt.ml +++ b/piqilib/piqi_getopt.ml @@ -387,6 +387,7 @@ let parse_args (piqtype: T.piqtype) (args: piq_ast list) :Piqobj.obj = let loc = (getopt_filename, 0, 1) in Piqloc.addlocret loc res in + let ast = Piq_parser.expand ast in let piqobj = U.with_bool Config.piq_relaxed_parsing true (fun () -> Piqobj_of_piq.parse_obj piqtype ast) in diff --git a/src/pp.ml b/src/pp.ml index dcb9ba5d..421cc53d 100644 --- a/src/pp.ml +++ b/src/pp.ml @@ -22,6 +22,8 @@ open C (* command-line parameters *) let flag_normalize = ref false let flag_expand_abbr = ref false +let flag_expand_names = ref false +let flag_expand_splices = ref false let flag_parse_literals = ref false let input_format = ref "" let output_format = ref "" @@ -32,9 +34,22 @@ let normalize_ast ast = let transform_ast ast = - if !flag_normalize - then normalize_ast ast - else ast + let ast = + if !flag_expand_names + then Piq_parser.expand_names ast + else ast + in + let ast = + if !flag_expand_splices + then Piq_parser.expand_splices ast + else ast + in + let ast = + if !flag_normalize + then normalize_ast ast + else ast + in + ast let make_reader piq_input_format = @@ -46,7 +61,7 @@ let make_reader piq_input_format = (* piq reder/parser *) let piq_parser = Piq_parser.init_from_channel fname ch in let piq_ast_parser () = - match Piq_parser.read_next piq_parser ~expand_abbr:!flag_expand_abbr ~skip_trailing_comma:true with + match Piq_parser.read_next piq_parser ~skip_trailing_comma:true with | Some ast -> ast | None -> raise Piqi_convert.EOF in @@ -135,7 +150,13 @@ let speclist = Main.common_speclist @ "normalize all words while pretty-printing (convert CamelCase to camel-case)"; "--expand-abbr", Arg.Set flag_expand_abbr, - "expand built-in syntax abbreviations"; + "expand built-in syntax abbreviations, equivalent to --expand-names --expand-splices"; + + "--expand-names", Arg.Set flag_expand_names, + "expand chained names, e.g. .foo.bar ..."; + + "--expand-splices", Arg.Set flag_expand_splices, + "expand splices, e.g. foo* [ ... ]"; "--parse-literals", Arg.Set flag_parse_literals, "deprecated: this option has no effect"; @@ -151,6 +172,13 @@ let speclist = Main.common_speclist @ let run () = Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:2; + + if !flag_expand_abbr + then ( + flag_expand_names := true; + flag_expand_splices := true; + ); + prettyprint_file () diff --git a/src/to_proto.ml b/src/to_proto.ml index 2ba2b66e..9ece52d5 100644 --- a/src/to_proto.ml +++ b/src/to_proto.ml @@ -207,6 +207,7 @@ let gen_default x = match x.default, x.piqtype with | Some any, Some piqtype -> let ast = Piqobj.piq_of_piqi_any any in + let ast = Piq_parser.expand ast in let piqobj = Piqobj_of_piq.parse_obj piqtype ast in gen_default_obj piqobj | _, _ -> iol [] (* there is no default *)