From 31c01b510fd0a5ea4f84e0c61b80284cc5ace4ee Mon Sep 17 00:00:00 2001 From: Anton Lavrik Date: Wed, 12 Jan 2011 02:32:33 -0600 Subject: [PATCH] Adding "piqi getopt" command that treats command-line arguments as Piq data, pretty-prints and converts it to various encodings (work in progress). --- piqi/Makefile | 1 + piqi/piq_parser.ml | 69 ++++++++------ piqi/piqi_convert.ml | 4 +- piqi/piqi_getopt.ml | 214 +++++++++++++++++++++++++++++++++++++++++++ piqi/piqi_main.ml | 2 +- 5 files changed, 261 insertions(+), 29 deletions(-) create mode 100644 piqi/piqi_getopt.ml diff --git a/piqi/Makefile b/piqi/Makefile index 57b9bced..73dc056a 100644 --- a/piqi/Makefile +++ b/piqi/Makefile @@ -21,6 +21,7 @@ SOURCES += \ piqi_check.ml \ piqi_expand.ml \ piqi_light.ml \ + piqi_getopt.ml \ \ piqi_to_proto.ml \ \ diff --git a/piqi/piq_parser.ml b/piqi/piq_parser.ml index 65f71c10..3c70fdb4 100644 --- a/piqi/piq_parser.ml +++ b/piqi/piq_parser.ml @@ -22,15 +22,6 @@ open Piqi_common module L = Piq_lexer -let location (fname, lexbuf) = - let line, col = L.location lexbuf in - (fname, line, col) - - -let string_of_char c = - String.make 1 c - - (* tokenize '[.:]'-separated string; return separator character as string * between separated tokens; * @@ -354,10 +345,23 @@ let make_string loc t s = * a simple piq parser *) -let read_next ?(expand_abbr=true) ((fname, lexbuf) as f) = - let loc () = location f in - let token () = L.token lexbuf in - let rollback tok = L.rollback lexbuf tok in +let read_next ?(expand_abbr=true) (fname, lexstream) = + let location = ref (0,0) in + let loc () = + let line, col = !location in + (fname, line, col) + in + let next_token () = + let tok, loc = Stream.next lexstream in + location := loc; tok + in + let peek_token () = + match Stream.peek lexstream with + | None -> assert false + | Some (tok, loc) -> + location := loc; tok + in + let junk_token () = Stream.junk lexstream in let error s = error_at (loc ()) s in let rec parse_common = function @@ -392,19 +396,19 @@ let read_next ?(expand_abbr=true) ((fname, lexbuf) as f) = (* TODO, XXX: move this functionality to the lexer *) (* join adjacent text lines *) and parse_text prev_line accu = - let tok = token () in + let tok = peek_token () in let _,line,_ = loc () in match tok with | L.Text text when prev_line + 1 = line -> (* add next line to the text unless there's a line between them *) - parse_text line (accu ^ "\n" ^ text) - | t -> (* something else -- end of text block *) - rollback t; accu + junk_token (); parse_text line (accu ^ "\n" ^ text) + | _ -> (* something else -- end of text block *) + accu and parse_control () = let startloc = loc () in let rec aux accu = - let t = token () in + let t = next_token () in match t with | L.Rpar -> let l = List.rev accu in @@ -488,21 +492,21 @@ let read_next ?(expand_abbr=true) ((fname, lexbuf) as f) = Piqloc.addlocret loc res and parse_named_part () = - let t = token () in + let t = peek_token () in match t with (* name delimiters *) | L.Word s when s.[0] = '.' || s.[0] = ':' -> (* other name or type *) - rollback t; None + None | L.Rbr | L.Rpar -> (* closing parenthesis or bracket *) - rollback t; None + None (* something else *) | _ -> - Some (parse_common t) (* parse named object *) + junk_token (); Some (parse_common t) (* parse named object *) and parse_list () = let startloc = loc () in let rec aux accu = - let t = token () in + let t = next_token () in match t with | L.Rbr -> let l = List.rev accu in @@ -513,7 +517,7 @@ let read_next ?(expand_abbr=true) ((fname, lexbuf) as f) = in aux [] in let parse_top () = - let t = token () in + let t = next_token () in match t with | L.EOF -> None | _ -> @@ -542,12 +546,25 @@ let read_all ?(expand_abbr=true) piq_parser = in aux [] +let make_lexstream lexbuf = + let f _counter = + let tok = L.token lexbuf in + let loc = L.location lexbuf in + Some (tok, loc) + in + Stream.from f + + let init_from_channel fname ch = let lexbuf = L.init_from_channel ch in - (fname, lexbuf) + (fname, make_lexstream lexbuf) let init_from_string fname s = let lexbuf = L.init_from_string s in - (fname, lexbuf) + (fname, make_lexstream lexbuf) + + +let init_from_token_list fname l = + (fname, Stream.of_list l) diff --git a/piqi/piqi_convert.ml b/piqi/piqi_convert.ml index 26471148..b898f2a7 100644 --- a/piqi/piqi_convert.ml +++ b/piqi/piqi_convert.ml @@ -40,10 +40,10 @@ let speclist = Main.common_speclist @ arg_o; "-f", Arg.Set_string input_encoding, - "piq|wire|pb input encoding"; + "piq|wire|pb|json|piq-json input encoding"; "-t", Arg.Set_string output_encoding, - "piq|wire|pb output encoding (piq is used by default)"; + "piq|wire|pb|json|piq-json output encoding (piq is used by default)"; "--piqtype", Arg.Set_string typename, " type of converted object when converting from .pb or plain .json"; diff --git a/piqi/piqi_getopt.ml b/piqi/piqi_getopt.ml new file mode 100644 index 00000000..f08e4fb7 --- /dev/null +++ b/piqi/piqi_getopt.ml @@ -0,0 +1,214 @@ +(*pp camlp4o -I $PIQI_ROOT/camlp4 pa_labelscope.cmo pa_openin.cmo *) +(* + Copyright 2009, 2010, 2011 Anton Lavrik + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*) + + +(* + * Command-line element parsing, pretty-printing and converting to various + * output formats. + *) + + +module C = Piqi_common +open C + + +(* global constants *) +let getopt_filename = "argv" (* fake filename for error reporting *) + + +let error s = + (* using fake location here, the actual location (i.e. the index of the + * argument) will be correctly provided by the exception handler below *) + let loc = (0,0) in + raise (Piq_lexer.Error (s, loc)) + + +let parse_string_arg s = + let lexbuf = Piq_lexer.init_from_string s in + let res = + try + Piq_lexer.token lexbuf + with + Piq_lexer.Error (err, _loc) -> error (err ^ ": " ^ s) + in + match res with + | Piq_lexer.String (t, s') when String.length s' + 2 = String.length s -> + res + | Piq_lexer.String _ -> + error ("trailing characters after string: " ^ s) (* s is alread quoted *) + | _ -> + assert false (* something that starts with '"' have to be a string *) + + +(* specifies the same as Piq_lexer's "let regexp word =" *) +let is_valid_word s = + let is_valid_char = function + | '(' | ')' | '[' | ']' | '{' | '}' + | '"' | '%' | '#' | '\000'..'\032' | '\127' -> false + | _ -> true + in + let len = String.length s in + let rec check_char i = + if i >= len + then true + else + if is_valid_char s.[i] + then check_char (i + 1) + else false + in + if len = 0 + then false + else check_char 0 + + +let parse_word_arg s = + if is_valid_word s + then Piq_lexer.Word s + else + (* return unquoted arg that is not a word as a Unicode string *) + Piq_lexer.String (Piq_lexer.String_u, s) + + +let parse_name_arg s = + if is_valid_word s + then ( + s.[0] <- '.'; (* replace '-' with '.' to get a Piq name *) + Piq_lexer.Word s + ) + else error ("invalid name: " ^ quote s) + + +let parse_arg = function + (* NOTE: we don't support '(' and ')' and '[]' is handeled separately below *) + | "[" -> Piq_lexer.Lbr + | "]" -> Piq_lexer.Rbr + | s when s.[0] = '"' -> parse_string_arg s + | s when s.[0] = '-' -> parse_name_arg s + | s -> parse_word_arg s + + +let parse_argv start = + let make_token i tok = + (* 1-based token position in the argv starting from the position after "--" *) + let loc = (0, i - start + 1) in + (tok, loc) + in + let parse_make_arg i x = + let tok = + try parse_arg x + with Piq_lexer.Error (err, _loc) -> + C.error_at (getopt_filename, 0, i) err + in + make_token i tok + in + let len = Array.length Sys.argv in + let rec aux i = + if i >= len + then [make_token i Piq_lexer.EOF] + else + match Sys.argv.(i) with + | "[]" -> (* split it into two tokens '[' and ']' *) + Sys.argv.(i) <- "]"; + (parse_make_arg i "[") :: (aux i) + | x -> + (parse_make_arg i x) :: (aux (i+1)) + in + aux start + + +(* index of the "--" element in argv array *) +let argv_start_index = ref 0 + +(* command-line arguments *) +let output_encoding = ref "" +let typename = ref "" + + +module Main = Piqi_main +open Main + + +let getopt_piq () = + let tokens = parse_argv !argv_start_index in + let piq_parser = Piq_parser.init_from_token_list getopt_filename tokens in + let piq_objects = Piq_parser.read_all piq_parser in + let piq_ast = + match piq_objects with + | [] -> None + | [x] -> Some x + | l -> + (* if there's more that one Piq objects, wrap them into a list *) + let res = `list l in + Piqloc.addref (List.hd l) res; (* preserve the location info *) + Some res + in + piq_ast + + +let getopt_command () = + (* open output file *) + let ch = Main.open_output !ofile in + (* interpret command-line arguments after "--" as Piq data *) + let piq_ast = getopt_piq () in + (* TODO: parse the Piq AST according to "--piqtype" and convert to the output + * format according to "-t" *) + match piq_ast with + | None -> () (* no data *) + | Some ast -> + Piqi_pp.prettyprint_ast ch ast; + output_char ch '\n' + + +(* find the position of the first argument after "--" *) +let rest_fun arg = + if !argv_start_index = 0 (* first argument after first occurence of "--" *) + then argv_start_index := !Arg.current + 1 + else () + + +let usage = "Usage: piqi getopt [options] -- [] \nOptions:" + + +let speclist = Main.common_speclist @ + [ + arg_o; + + "-t", Arg.Set_string output_encoding, + "piq|wire|pb|json|piq-json output encoding (piq is used by default)"; + + "--piqtype", Arg.Set_string typename, + " type of the object represented by data arguments"; + + "--add-defaults", Arg.Set Piqi_convert.flag_add_defaults, + "add field default values while converting records"; + + "--", Arg.Rest rest_fun, + "separator between piqi command-line arguments adn data arguments"; + ] + + +let run () = + Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:0; + if !argv_start_index = 0 (* "--" is not present in the list of arguments *) + then argv_start_index := Array.length Sys.argv; + getopt_command () + + +let _ = + Main.register_command run "getopt" + "interpret command-line arguments as Piq data, pretty-print and convert to various encodings" + diff --git a/piqi/piqi_main.ml b/piqi/piqi_main.ml index f24c20e8..fcb821e1 100644 --- a/piqi/piqi_main.ml +++ b/piqi/piqi_main.ml @@ -220,7 +220,7 @@ let usage () = (List.rev !commands); print_cmd "version" "print version"; prerr_endline - "\nMore information is available on http://piqi.org\n" + "\nMore information is available at http://piqi.org/doc/tools/\n" let exit_usage () =