Add support for slices and chained names in serialized Piq ASTs

piqilib/piq.ml

@@ -205,14 +205,18 @@ let rec to_portable_ast (ast:piq_ast) :Piq_piqi.piq_node =
           `typed Piq_piqi.Typed.({typename = typename; value = to_portable_ast value})
       | `list l ->
           `list (List.map to_portable_ast l)
-      (* these shouldn't be used for plain Piq ASTs *)
+      | `form (`name name, args) ->
+          `splice {Piq_piqi.Splice.name = name; Piq_piqi.Splice.item = List.map to_portable_ast args}
+      (* XXX, TODO: any shouldn't be used in plain Piq ASTs? *)
       | `form _ -> assert false
       | `any _ -> assert false
   in
   Piq_piqi.Piq_node.({piq = piq; loc = loc})
 
 
 let addloc loc ast =
+  (* TODO: this is not enough -- need to add location to the option values
+   * themselves *)
   Piqloc.setloc loc;
   (match ast with
     | `int (x, _) -> Piqloc.add x
@@ -232,39 +236,47 @@ let addloc loc ast =
         Piqloc.add typename
     | `list l ->
         Piqloc.add l
-    (* these shouldn't be used for plain Piq ASTs *)
-    | `form _ -> assert false
+    | `form ((`name s) as name, args) ->
+        Piqloc.add s;
+        Piqloc.add name
+    (* XXX, TODO: any shouldn't be used in plain Piq ASTs? *)
+    | `form (_, _) -> assert false
     | `any _ -> assert false
   );
   Piqloc.add ast
 
 
 let rec of_portable_ast (piq_node:Piq_piqi.piq_node) :piq_ast =
-  let ast =
-    match piq_node.Piq_piqi.Piq_node.piq with
-      | `int x -> `int (x, "")
-      | `uint x -> `uint (x, "")
-      | `float x -> `float (x, "")
-      | `bool x -> `bool x
-      | `string x -> `string (x, "")
-      | `raw_string x -> `raw_string x
-      | `word x -> `word x
-      | `text x -> `text x
-      | `binary x -> `binary (x, "")
-      | `name x -> `name x
-      | `typename x -> `typename x
-      | `named {Piq_piqi.Named.name = name; Piq_piqi.Named.value = value} ->
-          `named Piq_ast.Named.({name = name; value = of_portable_ast value})
-      | `typed {Piq_piqi.Typed.typename = typename; Piq_piqi.Typed.value = value} ->
-          `typed Piq_ast.Typed.({typename = typename; value = of_portable_ast value})
-      | `list l ->
-          `list (List.map of_portable_ast l)
+  let rec aux piq_node =
+    let ast =
+      match piq_node.Piq_piqi.Piq_node.piq with
+        | `int x -> `int (x, "")
+        | `uint x -> `uint (x, "")
+        | `float x -> `float (x, "")
+        | `bool x -> `bool x
+        | `string x -> `string (x, "")
+        | `raw_string x -> `raw_string x
+        | `word x -> `word x
+        | `text x -> `text x
+        | `binary x -> `binary (x, "")
+        | `name x -> `name x
+        | `typename x -> `typename x
+        | `named {Piq_piqi.Named.name = name; Piq_piqi.Named.value = value} ->
+            `named Piq_ast.Named.({name = name; value = aux value})
+        | `typed {Piq_piqi.Typed.typename = typename; Piq_piqi.Typed.value = value} ->
+            `typed Piq_ast.Typed.({typename = typename; value = aux value})
+        | `list l ->
+            `list (List.map aux l)
+        | `splice {Piq_piqi.Splice.name = name; Piq_piqi.Splice.item = items} ->
+            `form (`name name, List.map aux items)
+    in
+    let open Piq_piqi.Loc in
+    (match piq_node.Piq_piqi.Piq_node.loc with
+      | None -> ()
+      | Some {file = file; line = line; column = column} ->
+          let loc = (file, line, column) in
+          addloc loc ast
+    );
+    ast
   in
-  let open Piq_piqi.Loc in
-  (match piq_node.Piq_piqi.Piq_node.loc with
-    | None -> ()
-    | Some {file = file; line = line; column = column} ->
-        let loc = (file, line, column) in
-        addloc loc ast
-  );
-  ast
+  aux piq_node

piqilib/piq.piqi

@@ -48,6 +48,9 @@
         .name list
         .type piq-list
     ]
+    .option [
+        .type splice
+    ]
 ]
 
 
@@ -106,6 +109,18 @@
 ]
 
 
+.record [
+    .name splice
+
+    .field [ .type name ]
+    .field [
+        .name item
+        .type piq-node
+        .repeated
+    ]
+]
+
+
 .record [
     .name typed
 

piqilib/piq_parser.ml

@@ -180,7 +180,7 @@ let expand_names (x: piq_ast) :piq_ast =
       | `list l ->
           `list (List.map aux l)
       | `form (name, args) ->
-          (* at this stage, after we've run expand_forms, this can not be a
+          (* at this stage, after we've run expand_splices, this can not be a
            * named or typed form, so leaving name without a transformation *)
           `form (name, List.map aux args)
       | _ ->
@@ -220,8 +220,8 @@ let cons_named_or_typed name v =
         cons_typed n v
 
 
-(* expand named and typed forms *)
-let expand_forms (x: piq_ast) :piq_ast =
+(* expand named and typed splices *)
+let expand_splices (x: piq_ast) :piq_ast =
   let rec aux0 obj =
     match obj with
       | `form (name, args) when args = [] ->
@@ -283,13 +283,13 @@ let expand_forms (x: piq_ast) :piq_ast =
 
 (* expand built-in syntax abbreviations *)
 let expand x =
-  (* expand (...) when possible *)
-  let x = expand_forms x in
+  (* expand (.foo ...) or .foo* [ ... ] when possible *)
+  let x = expand_splices x in
   (* expand multi-component names *)
   let x = expand_names x in
   (*
     (* check if expansion produces correct location bindings *)
-    let x = expand_forms x in
+    let x = expand_splices x in
     let x = expand_names x in
   *)
   x
@@ -377,7 +377,7 @@ let parse_number s =
  * a simple piq parser
  *)
 
-let read_next ?(expand_abbr=true) ?(skip_trailing_comma=false) (fname, lexstream) =
+let read_next ?(skip_trailing_comma=false) (fname, lexstream) =
   let location = ref (0,0) in
   let loc () =
     let line, col = !location in
@@ -586,21 +586,17 @@ let read_next ?(expand_abbr=true) ?(skip_trailing_comma=false) (fname, lexstream
           let ast = parse_common t in
           (* skip an optional trailing comma *)
           if skip_trailing_comma && peek_token () = L.Comma then junk_token ();
-          let res =
-            if expand_abbr
-            then expand ast (* expand built-in syntax abbreviations *)
-            else ast (* return as it is *)
-          in Some res
+          Some ast
   in
   try parse_top ()
   with L.Error (s, (line, col)) ->
     (* convert lexer's errors *)
     error_at (fname, line, col) s
 
 
-let read_all ?(expand_abbr=true) piq_parser =
+let read_all piq_parser =
   let rec aux accu =
-    match read_next piq_parser ~expand_abbr ~skip_trailing_comma:true with
+    match read_next piq_parser ~skip_trailing_comma:true with
       | None -> List.rev accu
       | Some x ->
           aux (x::accu)

piqilib/piq_piqi.ml

@@ -24,11 +24,13 @@ module rec Piq_piqi:
         | `typename of Piq_piqi.name
         | `typed of Piq_piqi.typed
         | `list of Piq_piqi.piq_list
+        | `splice of Piq_piqi.splice
       ]
     type piq_node = Piq_node.t
     type loc = Loc.t
     type piq_list = Piq_piqi.piq_node list
     type named = Named.t
+    type splice = Splice.t
     type typed = Typed.t
   end = Piq_piqi
 and Piq_node:
@@ -53,6 +55,13 @@ and Named:
       mutable value: Piq_piqi.piq_node;
     }
   end = Named
+and Splice:
+  sig
+    type t = {
+      mutable name: Piq_piqi.name;
+      mutable item: Piq_piqi.piq_node list;
+    }
+  end = Splice
 and Typed:
   sig
     type t = {
@@ -129,6 +138,9 @@ and parse_piq x =
     | 14 ->
         let res = parse_piq_list x in
         `list res
+    | 15 ->
+        let res = parse_splice x in
+        `splice res
     | _ -> Piqirun.error_variant x code
 
 and parse_piq_node x =
@@ -171,6 +183,16 @@ and parse_named x =
     Named.value = _value;
   }
 
+and parse_splice x =
+  let x = Piqirun.parse_record x in
+  let _name, x = Piqirun.parse_required_field 1 parse_name x in
+  let _item, x = Piqirun.parse_repeated_field 2 parse_piq_node x in
+  Piqirun.check_unparsed_fields x;
+  {
+    Splice.name = _name;
+    Splice.item = _item;
+  }
+
 and parse_typed x =
   let x = Piqirun.parse_record x in
   let _typename, x = Piqirun.parse_required_field 1 parse_name x in
@@ -220,6 +242,7 @@ and gen__piq code (x:Piq_piqi.piq) =
     | `typename x -> gen__name 12 x
     | `typed x -> gen__typed 13 x
     | `list x -> gen__piq_list 14 x
+    | `splice x -> gen__splice 15 x
   )]
 
 and gen__piq_node code x =
@@ -244,6 +267,11 @@ and gen__named code x =
   let _value = Piqirun.gen_required_field 2 gen__piq_node x.Named.value in
   Piqirun.gen_record code (_name :: _value :: [])
 
+and gen__splice code x =
+  let _name = Piqirun.gen_required_field 1 gen__name x.Splice.name in
+  let _item = Piqirun.gen_repeated_field 2 gen__piq_node x.Splice.item in
+  Piqirun.gen_record code (_name :: _item :: [])
+
 and gen__typed code x =
   let _typename = Piqirun.gen_required_field 1 gen__name x.Typed.typename in
   let _value = Piqirun.gen_required_field 2 gen__piq_node x.Typed.value in
@@ -265,6 +293,7 @@ let gen_word x = gen__word (-1) x
 let gen_name x = gen__name (-1) x
 let gen_piq_list x = gen__piq_list (-1) x
 let gen_named x = gen__named (-1) x
+let gen_splice x = gen__splice (-1) x
 let gen_typed x = gen__typed (-1) x
 
 
@@ -296,6 +325,11 @@ and default_named () =
     Named.name = default_name ();
     Named.value = default_piq_node ();
   }
+and default_splice () =
+  {
+    Splice.name = default_name ();
+    Splice.item = [];
+  }
 and default_typed () =
   {
     Typed.typename = default_name ();

piqilib/piqi.ml

@@ -685,8 +685,8 @@ let check_defs ~piqi idtable defs =
 
 
 let read_piqi_common fname piq_parser :piq_ast =
-  (* don't expand abbreviations until we construct the containing object *)
-  let res = Piq_parser.read_all piq_parser ~expand_abbr:false in
+  (* NOTE: not expanding abbreviations until we construct the containing object *)
+  let res = Piq_parser.read_all piq_parser in
 
   if res = []
   then piqi_warning ("piqi file is empty: " ^ fname);
@@ -967,7 +967,7 @@ let apply_extensions obj obj_def obj_parse_f obj_gen_f extension_entries custom_
   (* Piqloc.trace := false; *)
   debug "apply_extensions(0)\n";
   let obj_ast = mlobj_to_ast obj_def obj_gen_f obj in
-  let extension_asts = List.map Piqobj.piq_of_piqi_any extension_entries in
+  let extension_asts = List.map (fun x -> Piq_parser.expand (Piqobj.piq_of_piqi_any x)) extension_entries in
 
   let override l =
     if not override

piqilib/piqi_boot.ml

@@ -2331,7 +2331,12 @@ let piq =
            typename = Some "piq-list"; deprecated = false; piqtype = None;
            unparsed_piq_ast = None; piq_format = None; piq_alias = None;
            protobuf_name = None; code = None; json_name = None;
-           getopt_letter = None; getopt_doc = None; proto_name = None}];
+           getopt_letter = None; getopt_doc = None; proto_name = None};
+          {Piqi_impl_piqi.Option.name = None; typename = Some "splice";
+           deprecated = false; piqtype = None; unparsed_piq_ast = None;
+           piq_format = None; piq_alias = None; protobuf_name = None;
+           code = None; json_name = None; getopt_letter = None;
+           getopt_doc = None; proto_name = None}];
         parent = None; is_func_param = false; unparsed_piq_ast = None;
         protobuf_name = None; protobuf_custom = []; json_name = None;
         proto_custom = []; proto_name = None};
@@ -2427,6 +2432,28 @@ let piq =
         piq_allow_unnesting = None; protobuf_name = None;
         protobuf_custom = []; json_name = None; proto_custom = [];
         proto_name = None};
+     `record
+       {Piqi_impl_piqi.Record.name = Some "splice";
+        field =
+         [{Piqi_impl_piqi.Field.name = None; typename = Some "name";
+           mode = `required; default = None; deprecated = false;
+           piqtype = None; unparsed_piq_ast = None; piq_format = None;
+           piq_positional = None; piq_alias = None; protobuf_name = None;
+           code = None; protobuf_packed = false; json_name = None;
+           json_omit_missing = None; getopt_letter = None; getopt_doc = None;
+           proto_name = None; wire_packed = false};
+          {Piqi_impl_piqi.Field.name = Some "item";
+           typename = Some "piq-node"; mode = `repeated; default = None;
+           deprecated = false; piqtype = None; unparsed_piq_ast = None;
+           piq_format = None; piq_positional = None; piq_alias = None;
+           protobuf_name = None; code = None; protobuf_packed = false;
+           json_name = None; json_omit_missing = None; getopt_letter = None;
+           getopt_doc = None; proto_name = None; wire_packed = false}];
+        parent = None; wire_field = []; is_func_param = false;
+        unparsed_piq_ast = None; piq_positional = None;
+        piq_allow_unnesting = None; protobuf_name = None;
+        protobuf_custom = []; json_name = None; proto_custom = [];
+        proto_name = None};
      `record
        {Piqi_impl_piqi.Record.name = Some "typed";
         field =

piqilib/piqi_convert.ml

@@ -179,6 +179,7 @@ let load_piq_from_ast (user_piqtype: T.piqtype option) ast :obj =
 
 let load_piq (user_piqtype: T.piqtype option) ?(skip_trailing_comma=false) piq_parser :obj =
   let ast = read_piq_ast piq_parser user_piqtype ~skip_trailing_comma in
+  let ast = Piq_parser.expand ast in
   load_piq_from_ast user_piqtype ast
 
 

piqilib/piqi_getopt.ml

@@ -387,6 +387,7 @@ let parse_args (piqtype: T.piqtype) (args: piq_ast list) :Piqobj.obj =
           let loc = (getopt_filename, 0, 1) in
           Piqloc.addlocret loc res
   in
+  let ast = Piq_parser.expand ast in
   let piqobj = U.with_bool Config.piq_relaxed_parsing true
     (fun () -> Piqobj_of_piq.parse_obj piqtype ast)
   in