From 2e16c519212fdb955c94810a2a4010c1e2829a11 Mon Sep 17 00:00:00 2001 From: Anton Lavrik Date: Thu, 13 Jan 2011 02:16:39 -0600 Subject: [PATCH] Finished implementation of "piqi getopt" command; added tests and examples. --- examples/Makefile | 2 + examples/test_getopt | 92 +++++++++++++++++++++++++++++++++++++++++ piqi/piq.ml | 8 +--- piqi/piq_parser.ml | 49 +++++++++++----------- piqi/piqi_convert.ml | 96 +++++++++++++++++++++++-------------------- piqi/piqi_getopt.ml | 50 ++++++++++++++++------ piqi/piqobj_of_piq.ml | 21 ++++++++-- 7 files changed, 228 insertions(+), 90 deletions(-) create mode 100755 examples/test_getopt diff --git a/examples/Makefile b/examples/Makefile index a516d0ca..b9455ed2 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -43,6 +43,8 @@ test_complex: ./test_piqi addressbook.proto ./test_piq addressbook + ./test_getopt + clean: rm -f *.pb *.wire *.json *.proto* addressbook.* diff --git a/examples/test_getopt b/examples/test_getopt new file mode 100755 index 00000000..2dc6a5b1 --- /dev/null +++ b/examples/test_getopt @@ -0,0 +1,92 @@ +#!/bin/sh + +set -ex + +# +# pretty-printing args as Piq AST +# + +piqi getopt # empty input +piqi getopt -- # empty input + +piqi getopt -- foo +piqi getopt -- 10 -20 30.0 -0.inf 0.nan '"foo\r"' '"\xff"' -fum [ -foo bar [] ] -baz + + +# +# converting args to Piq objects +# +# NOTE: converting to encodings other than Piq (controlled by -t parameter) is +# tested as a part of "piqi convert" testing as they share the same +# functionality +# + +# built-in types + +piqi getopt --piqtype bool -- true +piqi getopt --piqtype int -- -10 +piqi getopt --piqtype float -- -0.inf + +piqi getopt --piqtype string -- foo # parsing words as strings +piqi getopt --piqtype string -- foo\ bar +piqi getopt --piqtype string -- '"\tfoo\x20\u0045"' +piqi getopt --piqtype string -- "привет" +piqi getopt --piqtype binary -- '"\x00ab\tcd\xff\xfe"' + +piqi getopt --piqtype piq-word -- foo-bar +piqi getopt --piqtype piq-any -- 10 -20 30.0 -0.inf 0.nan '"foo\r"' '"\xff"' -fum [ -foo bar [] ] -baz +#NOTE: piq-text is not supported + +# complex types + +piqi getopt --piqtype complex/t -- [ .re 0 .im 0 ] +piqi getopt --piqtype complex/t -- .re 0 .im 0 +piqi getopt --piqtype complex/t -- -re 0 -im 0 +piqi getopt --piqtype complex/t -- 0 0 +piqi getopt --piqtype complex/foo -- [] +piqi getopt --piqtype complex/foo -- [ ] +piqi getopt --piqtype complex/foo --add-defaults -- [ ] + +piqi getopt --piqtype def/r -- -i 0 -flag +piqi getopt --piqtype def/r -- -i 0 -flag +piqi getopt --piqtype def/r -- -i 0 -flag + +piqi getopt --piqtype def/int-list -- [] +piqi getopt --piqtype def/int-list -- [ 1 2 3 4 5 ] +piqi getopt --piqtype def/int-list -- 1 2 3 4 5 +piqi getopt --piqtype def/int-list-list -- [ [] [ 1 2 3 ] ] +piqi getopt --piqtype def/int-list-list -- [] [ 1 2 3 ] + +piqi getopt --piqtype record-variant-list/r -- -a 0 -b 1 +piqi getopt --piqtype record-variant-list/l -- -a 0 -b 1 +piqi getopt --piqtype record-variant-list/v -- -a 0 +piqi getopt --piqtype record-variant-list/v -- -b 1 + +piqi getopt --piqtype person/person -- \ + -name "J. Random Hacker" \ + -id 0 \ + -email "j.r.hacker@example.com" \ + -phone [ -number "(111) 123 45 67" ] \ + -phone [ \ + -number "(222) 123 45 67" \ + -mobile \ + ] \ + -phone [ \ + -number "(333) 123 45 67" \ + -work \ + ] + +piqi getopt --piqtype person/person -- \ + .name "Joe User" \ + .id 1 \ + .phone [ "(444) 123 45 67" ] \ + .phone [ "(555) 123 45 67" .work ] + + +piqi getopt --piqtype function/bar-input -- [ 10 ] +piqi getopt --piqtype function/bar-output -- 1 +piqi getopt --piqtype function/bar-error -- 100.0 + +piqi getopt --piqtype function/baz-input -- [] +piqi getopt --piqtype function/baz-input --add-defaults -- [] + diff --git a/piqi/piq.ml b/piqi/piq.ml index 804c6320..9a44ecb4 100644 --- a/piqi/piq.ml +++ b/piqi/piq.ml @@ -46,10 +46,6 @@ let read_piq_ast piq_parser :T.ast = | None -> raise EOF -let piqobj_of_ast ?piqtype ast :Piqobj.obj = - Piqobj_of_piq.parse_typed_obj ast ?piqtype - - let default_piqtype = ref None @@ -100,12 +96,12 @@ let rec load_piq_obj piq_parser :obj = | `typename x -> error x "invalid piq object" | `typed _ -> - let obj = piqobj_of_ast ast in + let obj = Piqobj_of_piq.parse_typed_obj ast in Typed_piqobj obj | _ -> match !default_piqtype with | Some piqtype -> - let obj = piqobj_of_ast ~piqtype ast in + let obj = Piqobj_of_piq.parse_obj piqtype ast in Piqobj obj | None -> error ast "type of object is unknown" diff --git a/piqi/piq_parser.ml b/piqi/piq_parser.ml index 3c70fdb4..ecee4441 100644 --- a/piqi/piq_parser.ml +++ b/piqi/piq_parser.ml @@ -84,17 +84,17 @@ let make_any ast = Piqloc.addrefret ast res -let check_name loc n = +let check_name n = (* XXX: this should refer to piq rather than piqi name *) if Piqi_name.is_valid_name n ~allow:"." then () - else error_at loc ("invalid name: " ^ quote n) + else error n ("invalid name: " ^ quote n) -let check_typename loc n = +let check_typename n = if Piqi_name.is_valid_typename n ~allow:"." then () - else error_at loc ("invalid type name: " ^ quote n) + else error n ("invalid type name: " ^ quote n) let piq_addrefret dst (src:T.ast) = @@ -127,52 +127,50 @@ let piq_reference f x = else piq_addrefret x res -let make_named loc n v :T.ast = - check_name loc n; +let make_named n v :T.ast = + check_name n; match v with | None -> `name n | Some v -> let res = T.Named#{name = n; value = v} in - Piqloc.addloc loc res; + Piqloc.addref n res; `named res -let make_typed loc n v :T.ast = - check_typename loc n; +let make_typed n v :T.ast = + check_typename n; match v with | None -> `typename n | Some v -> let res = T.Typed#{typename = n; value = make_any v} in - Piqloc.addloc loc res; + Piqloc.addref n res; `typed res -let make_named_or_typed c loc n v = - Piqloc.addloc loc n; +let make_named_or_typed c name n v = + Piqloc.addref name n; let res = match c with - | "." -> make_named loc n v - | ":" -> make_typed loc n v + | "." -> make_named n v + | ":" -> make_typed n v | _ -> assert false in - Piqloc.addlocret loc res + Piqloc.addrefret name res -(* TODO: adjust locations according to the name component's offset *) -let expand_name obj c n v = - if not (String.contains n '.') && not (String.contains n ':') +let expand_name obj c name value = + if not (String.contains name '.') && not (String.contains name ':') then obj else - let loc = Piqloc.find n in (* TODO: optimize *) let rec aux = function | [c; n] -> - make_named_or_typed c loc n v + make_named_or_typed c name n value | c::n::t -> let v = aux t in - make_named_or_typed c loc n (Some v) + make_named_or_typed c name n (Some v) | _ -> assert false in - aux (tokenize_name c n) + aux (tokenize_name c name) let expand_obj_names (obj :T.ast) :T.ast = @@ -391,7 +389,7 @@ let read_next ?(expand_abbr=true) (fname, lexstream) = let text = parse_text line text in Piqloc.addloc text_loc text; Piqloc.addret (`text text) - | L.EOF -> error "unexpected end of file" + | L.EOF -> error "unexpected end of input" (* TODO, XXX: move this functionality to the lexer *) (* join adjacent text lines *) @@ -485,7 +483,7 @@ let read_next ?(expand_abbr=true) (fname, lexstream) = (* cut the first character which is '.' or ':' *) let n = String.sub s 1 (String.length s - 1) in Piqloc.addloc loc n; - let res = make_f loc n (parse_named_part ()) in + let res = make_f n (parse_named_part ()) in (* let res = expand_obj_names res in *) @@ -497,7 +495,8 @@ let read_next ?(expand_abbr=true) (fname, lexstream) = (* name delimiters *) | L.Word s when s.[0] = '.' || s.[0] = ':' -> (* other name or type *) None - | L.Rbr | L.Rpar -> (* closing parenthesis or bracket *) + | L.Rbr | L.Rpar (* closing parenthesis or bracket *) + | L.EOF -> (* end of input *) None (* something else *) | _ -> diff --git a/piqi/piqi_convert.ml b/piqi/piqi_convert.ml index b898f2a7..02b07ca1 100644 --- a/piqi/piqi_convert.ml +++ b/piqi/piqi_convert.ml @@ -58,18 +58,20 @@ let speclist = Main.common_speclist @ ] +let find_piqtype typename = + if not (Piqi_name.is_valid_typename typename) + then + piqi_error ("invalid type name: " ^ typename); + + try Piqi_db.find_piqtype typename + with Not_found -> + piqi_error ("unknown type: " ^ typename) + + let get_piqtype typename = if typename = "piqi" (* special case *) then !Piqi.piqi_def (* return Piqi type from embedded self-definition *) - else ( - if not (Piqi_name.is_valid_typename typename) - then - piqi_error ("invalid type name: " ^ typename); - - try Piqi_db.find_piqtype typename - with Not_found -> - piqi_error ("unknown type: " ^ typename) - ) + else find_piqtype typename let do_load_piqi fname = @@ -135,7 +137,7 @@ let make_reader load_f input_param = (fun () -> load_f input_param) -let init_json_writer enc = +let init_json_writer () = Piqi_json.init (); (* XXX: We need to resolve all defaults before converting to JSON * since it is a dynamic encoding *) @@ -143,7 +145,7 @@ let init_json_writer enc = () -let init_json_reader enc = +let init_json_reader () = Piqi_json.init (); if !typename <> "" then @@ -152,34 +154,34 @@ let init_json_reader enc = else () -let get_reader = function - | "pb" when !typename = "" -> - piqi_error "--piqtype parameter must be specified for \"pb\" input encoding" - | "pb" -> - let piqtype = get_piqtype !typename in - let wireobj = Piq.open_pb !ifile in - make_reader (load_pb piqtype) wireobj - | "json" | "piq-json" -> - init_json_reader (); - let json_parser = Piqi_json.open_json !ifile in - make_reader Piq.load_json_obj json_parser - | _ when !typename <> "" -> - piqi_error "--piqtype parameter is applicable only to \"pb\" or \"json\" input encodings" - | "piq" -> - let piq_parser = Piq.open_piq !ifile in - make_reader Piq.load_piq_obj piq_parser - | "piqi" -> - make_reader load_piqi !ifile - | "wire" -> - let buf = Piq.open_wire !ifile in - make_reader Piq.load_wire_obj buf - | _ -> - piqi_error "unknown input encoding" - - -let get_writer input_encoding = - let is_piqi_input = (input_encoding = "piqi") in - match !output_encoding with +let make_reader input_encoding = + match input_encoding with + | "pb" when !typename = "" -> + piqi_error "--piqtype parameter must be specified for \"pb\" input encoding" + | "pb" -> + let piqtype = get_piqtype !typename in + let wireobj = Piq.open_pb !ifile in + make_reader (load_pb piqtype) wireobj + | "json" | "piq-json" -> + init_json_reader (); + let json_parser = Piqi_json.open_json !ifile in + make_reader Piq.load_json_obj json_parser + | _ when !typename <> "" -> + piqi_error "--piqtype parameter is applicable only to \"pb\" or \"json\" input encodings" + | "piq" -> + let piq_parser = Piq.open_piq !ifile in + make_reader Piq.load_piq_obj piq_parser + | "piqi" -> + make_reader load_piqi !ifile + | "wire" -> + let buf = Piq.open_wire !ifile in + make_reader Piq.load_wire_obj buf + | _ -> + piqi_error "unknown input encoding" + + +let make_writer ?(is_piqi_input=false) output_encoding = + match output_encoding with | "" (* default output encoding is "piq" *) | "piq" -> Piq.write_piq | "wire" -> Piq.write_wire @@ -188,10 +190,10 @@ let get_writer input_encoding = Piq.write_json is_piqi_input | "piq-json" -> init_json_writer (); - output_encoding := "json"; Piq.write_piq_json | "pb" -> write_pb is_piqi_input - | _ -> piqi_error "unknown output encoding" + | _ -> + piqi_error "unknown output encoding" let seen = ref [] (* the list of seen elements *) @@ -289,14 +291,20 @@ let convert_file () = else Piqi_file.get_extension !ifile in validate_options input_encoding; - let reader = get_reader input_encoding in - let writer = get_writer input_encoding in + let reader = make_reader input_encoding in + let is_piqi_input = (input_encoding = "piqi") in + let writer = make_writer !output_encoding ~is_piqi_input in (* open output file *) let ofile = match !ofile with | "" when !output_encoding = "" -> "" (* print "piq" to stdout by default *) | "" when !ifile <> "" && !ifile <> "-" -> - !ifile ^ "." ^ !output_encoding + let output_extension = + match !output_encoding with + | "piq-json" -> "json" + | x -> x + in + !ifile ^ "." ^ output_extension | x -> x in let och = Main.open_output ofile in diff --git a/piqi/piqi_getopt.ml b/piqi/piqi_getopt.ml index f08e4fb7..6a7ebf09 100644 --- a/piqi/piqi_getopt.ml +++ b/piqi/piqi_getopt.ml @@ -92,13 +92,21 @@ let parse_name_arg s = else error ("invalid name: " ^ quote s) -let parse_arg = function - (* NOTE: we don't support '(' and ')' and '[]' is handeled separately below *) - | "[" -> Piq_lexer.Lbr - | "]" -> Piq_lexer.Rbr - | s when s.[0] = '"' -> parse_string_arg s - | s when s.[0] = '-' -> parse_name_arg s - | s -> parse_word_arg s +let parse_arg s = + let len = String.length s in + + if len = 0 + then error "empty argument"; + if s = "-" || s = "--" + then error ("invalid argument: " ^ s); + + match s with + (* NOTE: we don't support '(' and ')' and '[]' is handeled separately below *) + | "[" -> Piq_lexer.Lbr + | "]" -> Piq_lexer.Rbr + | s when s.[0] = '"' -> parse_string_arg s + | s when s.[0] = '-' && (s.[1] < '0' || s.[1] > '9') -> parse_name_arg s + | s -> parse_word_arg s let parse_argv start = @@ -159,18 +167,36 @@ let getopt_piq () = piq_ast +let validate_options () = + if !typename = "" (* pretty-print mode *) + then ( + if !output_encoding <> "" + then piqi_error "option -t can not be used without --piqtype"; + ) + + let getopt_command () = + validate_options (); (* open output file *) - let ch = Main.open_output !ofile in + let och = Main.open_output !ofile in (* interpret command-line arguments after "--" as Piq data *) let piq_ast = getopt_piq () in - (* TODO: parse the Piq AST according to "--piqtype" and convert to the output - * format according to "-t" *) match piq_ast with | None -> () (* no data *) + | Some ast when !typename = "" -> + (* with no --piqtype parameter given, just pretty-print the Piq AST *) + Piqi_pp.prettyprint_ast och ast; + output_char och '\n' | Some ast -> - Piqi_pp.prettyprint_ast ch ast; - output_char ch '\n' + let writer = Piqi_convert.make_writer !output_encoding in + let piqtype = Piqi_convert.find_piqtype !typename in + (* parse the Piq AST according to "--piqtype" and convert to the output + * format according to "-t" *) + Piqobj_of_piq.resolve_defaults := !Piqi_convert.flag_add_defaults; + Piqobj_of_piq.parse_words_as_strings := true; + let piqobj = Piqobj_of_piq.parse_obj piqtype ast in + (* write the object *) + writer och (Piq.Typed_piqobj piqobj) (* find the position of the first argument after "--" *) diff --git a/piqi/piqobj_of_piq.ml b/piqi/piqobj_of_piq.ml index a29bc137..455a930a 100644 --- a/piqi/piqobj_of_piq.ml +++ b/piqi/piqobj_of_piq.ml @@ -91,9 +91,17 @@ let get_unknown_fields () = res -(* XXX: resolve_defaults should be disabled by default *) +(* + * global constants + *) + +(* use default values to initialize optional fields that are missing *) let resolve_defaults = ref false +(* allow treating word literal as strings -- this mode is used by "piqi getopt" + *) +let parse_words_as_strings = ref false + (* ------------------------------------------------------------------- *) (* ------------------------------------------------------------------- *) @@ -150,6 +158,8 @@ let parse_string (x:T.ast) = match x with | `ascii_string s | `utf8_string s | `text s -> s | `binary s -> error s "string contains non-unicode binary data" + (* this mode is used by "piqi getopt" *) + | `word s when !parse_words_as_strings -> s | o -> error o "string expected" @@ -574,7 +584,7 @@ and make_option_finder f o = let open T.Option in match o.typeref with | None -> false - | Some x -> f (unalias (piqtype x)) + | Some x -> f (unalias (piqtype x)) (* TODO: optimize *) and parse_bool_option options x = @@ -593,7 +603,12 @@ and parse_float_option options x = and parse_word_option options x = - let f = make_option_finder ((=) `word) in + let test_f = + if !parse_words_as_strings (* this mode is used by "piqi getopt" *) + then (function `word | `string -> true | _ -> false) + else ((=) `word) + in + let f = make_option_finder test_f in parse_typed_option options f x