Skip to content

Commit

Permalink
Adding "piqi getopt" command that treats command-line arguments
Browse files Browse the repository at this point in the history
as Piq data, pretty-prints and converts it to various encodings
(work in progress).
  • Loading branch information
alavrik committed Jan 12, 2011
1 parent 9a95e39 commit 31c01b5
Show file tree
Hide file tree
Showing 5 changed files with 261 additions and 29 deletions.
1 change: 1 addition & 0 deletions piqi/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ SOURCES += \
piqi_check.ml \
piqi_expand.ml \
piqi_light.ml \
piqi_getopt.ml \
\
piqi_to_proto.ml \
\
Expand Down
69 changes: 43 additions & 26 deletions piqi/piq_parser.ml
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,6 @@ open Piqi_common
module L = Piq_lexer


let location (fname, lexbuf) =
let line, col = L.location lexbuf in
(fname, line, col)


let string_of_char c =
String.make 1 c


(* tokenize '[.:]'-separated string; return separator character as string
* between separated tokens;
*
Expand Down Expand Up @@ -354,10 +345,23 @@ let make_string loc t s =
* a simple piq parser
*)

let read_next ?(expand_abbr=true) ((fname, lexbuf) as f) =
let loc () = location f in
let token () = L.token lexbuf in
let rollback tok = L.rollback lexbuf tok in
let read_next ?(expand_abbr=true) (fname, lexstream) =
let location = ref (0,0) in
let loc () =
let line, col = !location in
(fname, line, col)
in
let next_token () =
let tok, loc = Stream.next lexstream in
location := loc; tok
in
let peek_token () =
match Stream.peek lexstream with
| None -> assert false
| Some (tok, loc) ->
location := loc; tok
in
let junk_token () = Stream.junk lexstream in
let error s = error_at (loc ()) s in

let rec parse_common = function
Expand Down Expand Up @@ -392,19 +396,19 @@ let read_next ?(expand_abbr=true) ((fname, lexbuf) as f) =
(* TODO, XXX: move this functionality to the lexer *)
(* join adjacent text lines *)
and parse_text prev_line accu =
let tok = token () in
let tok = peek_token () in
let _,line,_ = loc () in
match tok with
| L.Text text when prev_line + 1 = line ->
(* add next line to the text unless there's a line between them *)
parse_text line (accu ^ "\n" ^ text)
| t -> (* something else -- end of text block *)
rollback t; accu
junk_token (); parse_text line (accu ^ "\n" ^ text)
| _ -> (* something else -- end of text block *)
accu

and parse_control () =
let startloc = loc () in
let rec aux accu =
let t = token () in
let t = next_token () in
match t with
| L.Rpar ->
let l = List.rev accu in
Expand Down Expand Up @@ -488,21 +492,21 @@ let read_next ?(expand_abbr=true) ((fname, lexbuf) as f) =
Piqloc.addlocret loc res

and parse_named_part () =
let t = token () in
let t = peek_token () in
match t with
(* name delimiters *)
| L.Word s when s.[0] = '.' || s.[0] = ':' -> (* other name or type *)
rollback t; None
None
| L.Rbr | L.Rpar -> (* closing parenthesis or bracket *)
rollback t; None
None
(* something else *)
| _ ->
Some (parse_common t) (* parse named object *)
junk_token (); Some (parse_common t) (* parse named object *)

and parse_list () =
let startloc = loc () in
let rec aux accu =
let t = token () in
let t = next_token () in
match t with
| L.Rbr ->
let l = List.rev accu in
Expand All @@ -513,7 +517,7 @@ let read_next ?(expand_abbr=true) ((fname, lexbuf) as f) =
in aux []
in
let parse_top () =
let t = token () in
let t = next_token () in
match t with
| L.EOF -> None
| _ ->
Expand Down Expand Up @@ -542,12 +546,25 @@ let read_all ?(expand_abbr=true) piq_parser =
in aux []


let make_lexstream lexbuf =
let f _counter =
let tok = L.token lexbuf in
let loc = L.location lexbuf in
Some (tok, loc)
in
Stream.from f


let init_from_channel fname ch =
let lexbuf = L.init_from_channel ch in
(fname, lexbuf)
(fname, make_lexstream lexbuf)


let init_from_string fname s =
let lexbuf = L.init_from_string s in
(fname, lexbuf)
(fname, make_lexstream lexbuf)


let init_from_token_list fname l =
(fname, Stream.of_list l)

4 changes: 2 additions & 2 deletions piqi/piqi_convert.ml
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,10 @@ let speclist = Main.common_speclist @
arg_o;

"-f", Arg.Set_string input_encoding,
"piq|wire|pb input encoding";
"piq|wire|pb|json|piq-json input encoding";

"-t", Arg.Set_string output_encoding,
"piq|wire|pb output encoding (piq is used by default)";
"piq|wire|pb|json|piq-json output encoding (piq is used by default)";

"--piqtype", Arg.Set_string typename,
"<typename> type of converted object when converting from .pb or plain .json";
Expand Down
214 changes: 214 additions & 0 deletions piqi/piqi_getopt.ml
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
(*pp camlp4o -I $PIQI_ROOT/camlp4 pa_labelscope.cmo pa_openin.cmo *)
(*
Copyright 2009, 2010, 2011 Anton Lavrik
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*)


(*
* Command-line element parsing, pretty-printing and converting to various
* output formats.
*)


module C = Piqi_common
open C


(* global constants *)
let getopt_filename = "argv" (* fake filename for error reporting *)


let error s =
(* using fake location here, the actual location (i.e. the index of the
* argument) will be correctly provided by the exception handler below *)
let loc = (0,0) in
raise (Piq_lexer.Error (s, loc))


let parse_string_arg s =
let lexbuf = Piq_lexer.init_from_string s in
let res =
try
Piq_lexer.token lexbuf
with
Piq_lexer.Error (err, _loc) -> error (err ^ ": " ^ s)
in
match res with
| Piq_lexer.String (t, s') when String.length s' + 2 = String.length s ->
res
| Piq_lexer.String _ ->
error ("trailing characters after string: " ^ s) (* s is alread quoted *)
| _ ->
assert false (* something that starts with '"' have to be a string *)


(* specifies the same as Piq_lexer's "let regexp word =" *)
let is_valid_word s =
let is_valid_char = function
| '(' | ')' | '[' | ']' | '{' | '}'
| '"' | '%' | '#' | '\000'..'\032' | '\127' -> false
| _ -> true
in
let len = String.length s in
let rec check_char i =
if i >= len
then true
else
if is_valid_char s.[i]
then check_char (i + 1)
else false
in
if len = 0
then false
else check_char 0


let parse_word_arg s =
if is_valid_word s
then Piq_lexer.Word s
else
(* return unquoted arg that is not a word as a Unicode string *)
Piq_lexer.String (Piq_lexer.String_u, s)


let parse_name_arg s =
if is_valid_word s
then (
s.[0] <- '.'; (* replace '-' with '.' to get a Piq name *)
Piq_lexer.Word s
)
else error ("invalid name: " ^ quote s)


let parse_arg = function
(* NOTE: we don't support '(' and ')' and '[]' is handeled separately below *)
| "[" -> Piq_lexer.Lbr
| "]" -> Piq_lexer.Rbr
| s when s.[0] = '"' -> parse_string_arg s
| s when s.[0] = '-' -> parse_name_arg s
| s -> parse_word_arg s


let parse_argv start =
let make_token i tok =
(* 1-based token position in the argv starting from the position after "--" *)
let loc = (0, i - start + 1) in
(tok, loc)
in
let parse_make_arg i x =
let tok =
try parse_arg x
with Piq_lexer.Error (err, _loc) ->
C.error_at (getopt_filename, 0, i) err
in
make_token i tok
in
let len = Array.length Sys.argv in
let rec aux i =
if i >= len
then [make_token i Piq_lexer.EOF]
else
match Sys.argv.(i) with
| "[]" -> (* split it into two tokens '[' and ']' *)
Sys.argv.(i) <- "]";
(parse_make_arg i "[") :: (aux i)
| x ->
(parse_make_arg i x) :: (aux (i+1))
in
aux start


(* index of the "--" element in argv array *)
let argv_start_index = ref 0

(* command-line arguments *)
let output_encoding = ref ""
let typename = ref ""


module Main = Piqi_main
open Main


let getopt_piq () =
let tokens = parse_argv !argv_start_index in
let piq_parser = Piq_parser.init_from_token_list getopt_filename tokens in
let piq_objects = Piq_parser.read_all piq_parser in
let piq_ast =
match piq_objects with
| [] -> None
| [x] -> Some x
| l ->
(* if there's more that one Piq objects, wrap them into a list *)
let res = `list l in
Piqloc.addref (List.hd l) res; (* preserve the location info *)
Some res
in
piq_ast


let getopt_command () =
(* open output file *)
let ch = Main.open_output !ofile in
(* interpret command-line arguments after "--" as Piq data *)
let piq_ast = getopt_piq () in
(* TODO: parse the Piq AST according to "--piqtype" and convert to the output
* format according to "-t" *)
match piq_ast with
| None -> () (* no data *)
| Some ast ->
Piqi_pp.prettyprint_ast ch ast;
output_char ch '\n'


(* find the position of the first argument after "--" *)
let rest_fun arg =
if !argv_start_index = 0 (* first argument after first occurence of "--" *)
then argv_start_index := !Arg.current + 1
else ()


let usage = "Usage: piqi getopt [options] -- [<data arguments>] \nOptions:"


let speclist = Main.common_speclist @
[
arg_o;

"-t", Arg.Set_string output_encoding,
"piq|wire|pb|json|piq-json output encoding (piq is used by default)";

"--piqtype", Arg.Set_string typename,
"<typename> type of the object represented by data arguments";

"--add-defaults", Arg.Set Piqi_convert.flag_add_defaults,
"add field default values while converting records";

"--", Arg.Rest rest_fun,
"separator between piqi command-line arguments adn data arguments";
]


let run () =
Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:0;
if !argv_start_index = 0 (* "--" is not present in the list of arguments *)
then argv_start_index := Array.length Sys.argv;
getopt_command ()


let _ =
Main.register_command run "getopt"
"interpret command-line arguments as Piq data, pretty-print and convert to various encodings"

2 changes: 1 addition & 1 deletion piqi/piqi_main.ml
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ let usage () =
(List.rev !commands);
print_cmd "version" "print version";
prerr_endline
"\nMore information is available on http://piqi.org\n"
"\nMore information is available at http://piqi.org/doc/tools/\n"


let exit_usage () =
Expand Down

0 comments on commit 31c01b5

Please sign in to comment.