diff --git a/Makefile b/Makefile index 32c7c716..0e52f874 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,12 @@ include $(PIQI_ROOT)/make/Makefile.dirs DIRS = \ camlp4 \ piqirun-ocaml \ - piqicc piqic piqi \ + piqicc piqic piqi-tools \ install: -install -d $(PIQI_PREFIX)/bin - install piqi/piqi $(PIQI_PREFIX)/bin + install piqi-tools/piqi $(PIQI_PREFIX)/bin install piqic/piqic $(PIQI_PREFIX)/bin diff --git a/README b/README index 4f5c17a8..5a64f00f 100644 --- a/README +++ b/README @@ -64,8 +64,9 @@ FILES camlp4/ camlp4 extensions piqicc/ Piqi compiler compiler piqic/ Piq interface compiler for OCaml and Erlang - piqi/ "piqi" command-line utility (piqi tools) - piqi.org/ Piqi self-specifications + piqi-tools/ "piqi" command-line utility (piqi tools) + piqi.org/ Piqi self-specification + piqilib/ Piqi common library piqirun-ocaml/ Piqi runtime library for OCaml piqirun-erlang/ Piqi runtime library for Erlang examples/ examples diff --git a/bin/piqi b/bin/piqi index b4b955f3..71ef8470 120000 --- a/bin/piqi +++ b/bin/piqi @@ -1 +1 @@ -../piqi/piqi \ No newline at end of file +../piqi-tools/piqi \ No newline at end of file diff --git a/examples/Makefile b/examples/Makefile index b9455ed2..e30d8b60 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -32,6 +32,7 @@ test_complex: ./test_piqi_proto record-variant-list ./test_piq record-variant-list + ln -sf ../piqilib/piqobj.piqi . ./test_piqi piqobj ./test_piqi function @@ -47,4 +48,4 @@ test_complex: clean: - rm -f *.pb *.wire *.json *.proto* addressbook.* + rm -f *.pb *.wire *.json *.proto* addressbook.* piqobj.piqi diff --git a/examples/piqobj.piqi b/examples/piqobj.piqi deleted file mode 120000 index b67464ad..00000000 --- a/examples/piqobj.piqi +++ /dev/null @@ -1 +0,0 @@ -../piqi/piqobj.piqi \ No newline at end of file diff --git a/piqi-tools/.gitignore b/piqi-tools/.gitignore new file mode 100644 index 00000000..c63b2dd5 --- /dev/null +++ b/piqi-tools/.gitignore @@ -0,0 +1,52 @@ +piqi +piqtype.* +piqobj.ml +descriptor.* +piqi_version.ml +piqi_json_parser.ml + +piqi_c_impl.c +piqi_main.ml +piqi.ml +piq_parser.ml +piqi_common.ml +piqobj_to_piq.ml +piqi_c.mli +piqi_name.ml +piqi_file.ml +piqobj_common.ml +piqobj_of_wire.ml +piqloc.ml +piq_lexer.ml +piqi_config.ml +piq_gen.ml +piqobj_to_wire.ml +piqi_iolist.ml +piqi_wire.ml +piqobj_of_piq.ml +piqi_db.ml +piqi_c.ml +piqi_ext.ml +piqi_func.ml + +# piqi lib + +piqi-boot.piqi +piqi.piqi +piqi-impl.piqi + +piqi_json_common.ml +piqi_json_parser.mll +piqi_json_gen.ml +piqi_json.ml + +piqi_base64.ml +piqobj_to_json.ml +piqobj_of_json.ml + +piqi_pp.ml +piq.ml + +piqi_light.ml +piqi_getopt.ml + diff --git a/piqi-tools/Makefile b/piqi-tools/Makefile new file mode 100644 index 00000000..8f4a306c --- /dev/null +++ b/piqi-tools/Makefile @@ -0,0 +1,53 @@ +include $(PIQI_ROOT)/make/Makefile.ocaml +include $(PIQI_ROOT)/piqilib/Makefile.piqilib + + +RESULT = piqi + +SOURCES += \ + piqi_json_pp.ml \ + \ + piqi_pp_cmd.ml \ + piqi_convert.ml \ + piqi_check.ml \ + piqi_expand.ml \ + piqi_light_cmd.ml \ + piqi_getopt_cmd.ml \ + \ + piqi_to_proto.ml \ + \ + descriptor.ml \ + piqi_of_proto.ml \ + \ + piqi_run.ml \ + + + +PIQI_MODULES = descriptor.piqi +PIQI_ML_SOURCES = $(PIQI_MODULES:%.piqi=%.ml) + + +PRE_TARGETS += $(PIQI_SOURCES) $(PIQI_PIQI) $(PIQI_ML_SOURCES) + + +PIQIC = ../piqic/piqic +PIQIC_FLAGS = -I $(PIQI_ROOT) + + +#OCAMLBLDFLAGS = -custom + + +all: nc + + +debug: dc top + + +$(PIQI_ML_SOURCES): %.ml: %.piqi + $(PIQIC) ocaml $(PIQIC_FLAGS) --pp -o $@ $< + + +clean:: + + +include $(OCAMLMAKEFILE) diff --git a/piqi/descriptor.piqi b/piqi-tools/descriptor.piqi similarity index 100% rename from piqi/descriptor.piqi rename to piqi-tools/descriptor.piqi diff --git a/piqi/piqi_check.ml b/piqi-tools/piqi_check.ml similarity index 100% rename from piqi/piqi_check.ml rename to piqi-tools/piqi_check.ml diff --git a/piqi/piqi_convert.ml b/piqi-tools/piqi_convert.ml similarity index 100% rename from piqi/piqi_convert.ml rename to piqi-tools/piqi_convert.ml diff --git a/piqi/piqi_expand.ml b/piqi-tools/piqi_expand.ml similarity index 50% rename from piqi/piqi_expand.ml rename to piqi-tools/piqi_expand.ml index 139133be..e2685e02 100644 --- a/piqi/piqi_expand.ml +++ b/piqi-tools/piqi_expand.ml @@ -20,60 +20,6 @@ module C = Piqi_common open C -let init_res_piqi orig_piqi = - let open P in - { - modname = orig_piqi.modname; - proto_package = orig_piqi.proto_package; - piqdef = []; - extend = []; - includ = []; - import = []; - func = []; - - custom_field = []; - - (* piqi-impl (implementation related) extensions *) - extended_piqdef = []; - resolved_piqdef = []; - imported_piqdef = []; - resolved_import = []; - resolved_func = []; - included_piqi = []; - original_piqi = None; - } - - -let expand_piqi ?(includes_only=false) piqi = - let open P in - let all_piqi = piqi.included_piqi in - let orig_piqi = some_of piqi.original_piqi in - - (* create a new piqi module from the original piqi module *) - let res_piqi = init_res_piqi orig_piqi in - - (* copy all imports to the resulting module *) - let imports = Piqi.get_imports all_piqi in - res_piqi.import <- imports; - - (* copy all definitions to the resulting module *) - res_piqi.piqdef <- - if includes_only - then Piqi.get_piqdefs all_piqi - else piqi.extended_piqdef; - - (* copy all extensions to the resulting module *) - res_piqi.extend <- - if includes_only - then Piqi.get_extensions all_piqi - else []; - - (* copy all functions to the resulting module *) - res_piqi.func <- Piqi_ext.get_functions all_piqi; - - res_piqi - - module Main = Piqi_main open Main @@ -95,7 +41,7 @@ let speclist = Main.common_speclist @ let expand_file filename = let ch = Main.open_output !ofile in let piqi = Piqi.load_piqi filename in - let res_piqi = expand_piqi piqi ~includes_only:!flag_includes_only in + let res_piqi = Piqi_ext.expand_piqi piqi ~includes_only:!flag_includes_only in Piqi_pp.prettyprint_piqi ch res_piqi diff --git a/piqi-tools/piqi_getopt_cmd.ml b/piqi-tools/piqi_getopt_cmd.ml new file mode 100644 index 00000000..8859005f --- /dev/null +++ b/piqi-tools/piqi_getopt_cmd.ml @@ -0,0 +1,112 @@ +(*pp camlp4o -I $PIQI_ROOT/camlp4 pa_labelscope.cmo pa_openin.cmo *) +(* + Copyright 2009, 2010, 2011 Anton Lavrik + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*) + + +(* + * Command-line element parsing, pretty-printing and converting to various + * output formats. + *) + + +module C = Piqi_common +open C + + +(* command-line arguments *) +let output_encoding = ref "" +let typename = ref "" + + +(* index of the "--" element in argv array *) +let argv_start_index = ref 0 + + +module Main = Piqi_main +open Main + + +let validate_options () = + if !typename = "" (* pretty-print mode *) + then ( + if !output_encoding <> "" + then piqi_error "option -t can not be used without --piqtype"; + ) + + +let getopt_command () = + validate_options (); + (* open output file *) + let och = Main.open_output !ofile in + (* interpret command-line arguments after "--" as Piq data *) + let piq_ast = Piqi_getopt.getopt_piq !argv_start_index in + match piq_ast with + | None -> () (* no data *) + | Some ast when !typename = "" -> + (* with no --piqtype parameter given, just pretty-print the Piq AST *) + Piqi_pp.prettyprint_ast och ast; + output_char och '\n' + | Some ast -> + let writer = Piqi_convert.make_writer !output_encoding in + let piqtype = Piqi_convert.find_piqtype !typename in + (* parse the Piq AST according to "--piqtype" and convert to the output + * format according to "-t" *) + Piqobj_of_piq.resolve_defaults := !Piqi_convert.flag_add_defaults; + Piqobj_of_piq.parse_words_as_strings := true; + let piqobj = Piqobj_of_piq.parse_obj piqtype ast in + (* write the object *) + writer och (Piq.Typed_piqobj piqobj) + + +(* find the position of the first argument after "--" *) +let rest_fun arg = + if !argv_start_index = 0 (* first argument after first occurence of "--" *) + then argv_start_index := !Arg.current + 1 + else () + + +let usage = "Usage: piqi getopt [options] -- [] \nOptions:" + + +let speclist = Main.common_speclist @ + [ + arg_o; + + "-t", Arg.Set_string output_encoding, + "piq|wire|pb|json|piq-json output encoding (piq is used by default)"; + + "--piqtype", Arg.Set_string typename, + " type of the object represented by data arguments"; + + "--add-defaults", Arg.Set Piqi_convert.flag_add_defaults, + "add field default values while converting records"; + + "--", Arg.Rest rest_fun, + "separator between piqi command-line arguments and data arguments"; + ] + + +let run () = + Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:0; + if !argv_start_index = 0 (* "--" is not present in the list of arguments *) + then argv_start_index := Array.length Sys.argv; + getopt_command () + + +let _ = + Main.register_command run "getopt" + "interpret command-line arguments as Piq data, pretty-print and convert to various encodings" + diff --git a/piqi-tools/piqi_json_pp.ml b/piqi-tools/piqi_json_pp.ml new file mode 100644 index 00000000..fe139f9b --- /dev/null +++ b/piqi-tools/piqi_json_pp.ml @@ -0,0 +1,68 @@ +(*pp camlp4o -I $PIQI_ROOT/camlp4 pa_labelscope.cmo pa_openin.cmo *) +(* + Copyright 2009, 2010, 2011 Anton Lavrik + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*) + + +open Piqi_common + + +module Main = Piqi_main +open Main + + +let usage = "Usage: piqi json-pp [options] [<.json file>] [output file]\nOptions:" + + +let speclist = Main.common_speclist @ + [ + arg_o; + arg__; + ] + + +let prettyprint_json ch json = + Piqi_json_gen.pretty_to_channel ch json + + +let prettyprint_json ch json_parser = + let rec aux () = + match Piqi_json.read_json_obj json_parser with + | None -> () + | Some json -> + (* reset location db to allow GC to collect previously read objects *) + Piqloc.reset (); + prettyprint_json ch json; + output_string ch "\n\n"; + aux () + in aux () + + +let prettyprint_file filename = + let ch = Main.open_output !ofile in + (* switch parser/generator to pretty-print mode *) + Config.pp_mode := true; + let json_parser = Piqi_json.open_json filename in + prettyprint_json ch json_parser + + +let run () = + Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:2; + prettyprint_file !ifile + + +let _ = + Main.register_command run "json-pp" "pretty-print %.json" + diff --git a/piqi/piqi_ext.ml b/piqi-tools/piqi_light_cmd.ml similarity index 53% rename from piqi/piqi_ext.ml rename to piqi-tools/piqi_light_cmd.ml index 0422d7a2..7e85c296 100644 --- a/piqi/piqi_ext.ml +++ b/piqi-tools/piqi_light_cmd.ml @@ -16,20 +16,34 @@ *) -(* This module contains functionality that Piqi module would contain, but if it - * did, it would break Piqicc bootstrap that doesn't support advanced Piqi - * functionality such as functions. *) +(* + * Light syntax for Piqi DDL: Piqi to Piqi-light pretty-printer + *) + + +let print_piqi_file ch filename = + let piqi = Piqi.load_piqi filename in + Piqi_light.gen_piqi ch piqi + + +module Main = Piqi_main +open Main -(* XXX: include this module from Piqi? *) +let usage = "Usage: piqi light [options] [<.piqi file>] [output-file]\nOptions:" +let speclist = Main.common_speclist @ + [ + arg_o; + ] -module C = Piqi_common -open C +let run () = + Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:2; + let ch = Main.open_output !ofile in + print_piqi_file ch !ifile -let get_functions modules = - flatmap (fun x -> x.P#func) modules +let _ = + Main.register_command run "light" + "pretty-print %.piqi using Piqi-light syntax" -let get_resolved_functions modules = - flatmap (fun x -> x.P#resolved_func) modules diff --git a/piqi/piqi_of_proto.ml b/piqi-tools/piqi_of_proto.ml similarity index 100% rename from piqi/piqi_of_proto.ml rename to piqi-tools/piqi_of_proto.ml diff --git a/piqi-tools/piqi_pp_cmd.ml b/piqi-tools/piqi_pp_cmd.ml new file mode 100644 index 00000000..bf94334f --- /dev/null +++ b/piqi-tools/piqi_pp_cmd.ml @@ -0,0 +1,121 @@ +(*pp camlp4o -I $PIQI_ROOT/camlp4 pa_labelscope.cmo pa_openin.cmo *) +(* + Copyright 2009, 2010, 2011 Anton Lavrik + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*) + + +open Piqi_common + + +(* command-line parameters *) +let flag_normalize = ref false +let flag_expand_abbr = ref false + + +let open_piq fname = + let ch = Piqi_main.open_input fname in + let piq_parser = Piq_parser.init_from_channel fname ch in + piq_parser + + +let read_piq_obj piq_parser = + let res = Piq_parser.read_next piq_parser in + (* reset location db to allow GC to collect previously read objects *) + Piqloc.reset (); + res + + +let map_words (x:T.ast) f = + let rec aux = function + | `word s -> `word (f s) + | `name s -> `name (f s) + | `named ({T.Named.value = v} as x) -> + `named {x with T.Named.value = aux v} + | `typed ({T.Typed.value = v} as x) -> + let ast = some_of v.T.Any.ast in + let v = {v with T.Any.ast = Some (aux ast)} in + `typed {x with T.Typed.value = v} + | `list l -> `list (List.map aux l) + | `control l -> `control (List.map aux l) + | x -> x + in aux x + + +let normalize ast = + map_words ast Piqi_name.normalize_name + + +let transform_ast ast = + let ast = + if !flag_normalize + then normalize ast + else ast + in + let ast = + if !flag_expand_abbr + then Piq_parser.expand ast + else ast + in ast + + +let prettyprint_piq ch piq_parser = + let rec aux () = + match read_piq_obj piq_parser with + | None -> () + | Some ast -> + let ast = transform_ast ast in + Piqi_pp.prettyprint_ast ch ast; + output_char ch '\n'; + aux () + in aux () + + +module Main = Piqi_main +open Main + + +let prettyprint_file filename = + let ch = Main.open_output !ofile in + (* switch piq parser/generator to pretty-print mode *) + Config.pp_mode := true; + let piq_parser = open_piq filename in + prettyprint_piq ch piq_parser + + +let usage = "Usage: piqi pp [options] [<.piqi|.piq file>] [output file]\nOptions:" + + +let speclist = Main.common_speclist @ + [ + arg_o; + + "--normalize-words", Arg.Set flag_normalize, + "normalize all words while pretty-printing (convert CamelCase to camel-case)"; + + "--expand-abbr", Arg.Set flag_expand_abbr, + "expand built-in syntax abbreviations"; + + arg__; + ] + + +let run () = + Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:2; + prettyprint_file !ifile + + +let _ = + Main.register_command run "pp" "pretty-print %.piqi or %.piq" + diff --git a/piqi/piqi_run.ml b/piqi-tools/piqi_run.ml similarity index 100% rename from piqi/piqi_run.ml rename to piqi-tools/piqi_run.ml diff --git a/piqi/piqi_to_proto.ml b/piqi-tools/piqi_to_proto.ml similarity index 100% rename from piqi/piqi_to_proto.ml rename to piqi-tools/piqi_to_proto.ml diff --git a/piqi.org/Makefile b/piqi.org/Makefile index 04f04d70..39825b92 100644 --- a/piqi.org/Makefile +++ b/piqi.org/Makefile @@ -19,7 +19,7 @@ test: test_boot test_piqi test_piqi_piqi test_piqtype_core test_boot: set -e; \ - ln -sf ../piqi/piqobj.piqi piqobj.piqi; \ + ln -sf ../piqilib/piqobj.piqi piqobj.piqi; \ for i in $(BOOT_PIQI_FILES); \ do \ piqi check --noboot $$i; \ diff --git a/piqi/Makefile b/piqi/Makefile deleted file mode 100644 index 73dc056a..00000000 --- a/piqi/Makefile +++ /dev/null @@ -1,74 +0,0 @@ -include $(PIQI_ROOT)/make/Makefile.ocaml -include Makefile.piqi_extended - - -RESULT = piqi - -SOURCES += \ - piqi_json_common.ml \ - piqi_json_parser.mll \ - piqi_json_gen.ml \ - piqi_json.ml \ - \ - base64.ml \ - piqobj_to_json.ml \ - piqobj_of_json.ml \ - \ - piqi_pp.ml \ - piq.ml \ - \ - piqi_convert.ml \ - piqi_check.ml \ - piqi_expand.ml \ - piqi_light.ml \ - piqi_getopt.ml \ - \ - piqi_to_proto.ml \ - \ - descriptor.ml \ - piqi_of_proto.ml \ - \ - piqi_run.ml \ - - - -PIQI_MODULES = descriptor.piqi -PIQI_ML_SOURCES = $(PIQI_MODULES:%.piqi=%.ml) - - -PRE_TARGETS += $(PIQI_ML_SOURCES) - - -PIQIC = ../piqic/piqic -PIQIC_FLAGS = -I $(PIQI_ROOT) - -PIQICC = ../piqicc/piqicc -PIQICC_FLAGS = $(PIQIC_FLAGS) - - -#OCAMLBLDFLAGS = -custom - - -all: nc - - -debug: dc top - - -piqtype.ml: piqi-impl.piqi - # Note, using standard piqi.org/piqi-boot as boot module for piqicc - $(PIQICC) $(PIQICC_FLAGS) -o $@.tmp.ml \ - --boot piqi-boot.piqi --piqi piqi.piqi --impl piqi-impl.piqi - $(CAMLP4)o -o $@ $@.tmp.ml - - -$(PIQI_ML_SOURCES): %.ml: %.piqi - $(PIQIC) ocaml $(PIQIC_FLAGS) -o $@.tmp.ml $< - $(CAMLP4)o -o $@ $@.tmp.ml - - -clean:: - rm -f *.tmp.ml - - -include $(OCAMLMAKEFILE) diff --git a/piqic/Makefile b/piqic/Makefile index ddeacd9c..3d927ee8 100644 --- a/piqic/Makefile +++ b/piqic/Makefile @@ -1,5 +1,5 @@ include $(PIQI_ROOT)/make/Makefile.ocaml -include $(PIQI_ROOT)/piqi/Makefile.piqi_extended +include $(PIQI_ROOT)/piqilib/Makefile.piqi_extended RESULT = piqic diff --git a/piqic/piqic-impl.piqi b/piqic/piqic-impl.piqi index 33767eb5..bf0bb207 100644 --- a/piqic/piqic-impl.piqi +++ b/piqic/piqic-impl.piqi @@ -3,12 +3,12 @@ % % 1) The mandatory piqi specification ("piqic.piqi"). % -% 2) Implementation-specific extensions ("piqi/piqi-impl.piqi") utilized by -% common piqi library (../piqi). This way, piqic doesn't have any custom +% 2) Implementation-specific extensions ("piqilib/piqi-impl.piqi") used by +% common piqi library (../piqilib). This way, piqic doesn't have any custom % implementation-specific extensions -- only those required by common piqi % library. -.include [ .module piqi/piqi-impl ] +.include [ .module piqilib/piqi-impl ] .include [ .module piqic/piqic ] diff --git a/piqicc/Makefile.piqicc_common b/piqicc/Makefile.piqicc_common index 811bd587..f8be33bb 100644 --- a/piqicc/Makefile.piqicc_common +++ b/piqicc/Makefile.piqicc_common @@ -1,5 +1,5 @@ include $(PIQI_ROOT)/make/Makefile.ocaml -include $(PIQI_ROOT)/piqi/Makefile.piqi_common +include $(PIQI_ROOT)/piqilib/Makefile.piqi_common PIQIC_LIB_SOURCES = \ diff --git a/piqicc/piqicc-impl.piqi b/piqicc/piqicc-impl.piqi index 30d3acb5..c4a6158d 100644 --- a/piqicc/piqicc-impl.piqi +++ b/piqicc/piqicc-impl.piqi @@ -3,12 +3,12 @@ % % 1) The mandatory piqi specification ("piqicc.piqi"). % -% 2) Implementation-specific extensions ("piqi/piqi-impl.piqi") utilized by -% common piqi library (../piqi). This way, piqicc doesn't have any custom +% 2) Implementation-specific extensions ("piqilib/piqi-impl.piqi") used by +% common piqi library (../piqilib). This way, piqicc doesn't have any custom % implementation-specific extensions -- only those required by common piqi % library. -.include [ .module piqi/piqi-impl ] +.include [ .module piqilib/piqi-impl ] .include [ .module piqicc/piqicc ] diff --git a/piqi/.gitignore b/piqilib/.gitignore similarity index 53% rename from piqi/.gitignore rename to piqilib/.gitignore index ea93bfdf..5c6b17a1 100644 --- a/piqi/.gitignore +++ b/piqilib/.gitignore @@ -1,7 +1,8 @@ -piqi +piqilib.* piqtype.* piqobj.ml -descriptor.* piqi_version.ml piqi_json_parser.ml +dllpiqilib_stubs.so +libpiqilib_stubs.a diff --git a/piqilib/Makefile b/piqilib/Makefile new file mode 100644 index 00000000..52b04876 --- /dev/null +++ b/piqilib/Makefile @@ -0,0 +1,11 @@ +include $(PIQI_ROOT)/make/Makefile.ocaml +include Makefile.piqilib + + +RESULT = piqilib + + +all: bcl #ncl + + +include $(OCAMLMAKEFILE) diff --git a/piqi/Makefile.piqi_common b/piqilib/Makefile.piqi_common similarity index 92% rename from piqi/Makefile.piqi_common rename to piqilib/Makefile.piqi_common index 829e7f5b..76c46e2b 100644 --- a/piqi/Makefile.piqi_common +++ b/piqilib/Makefile.piqi_common @@ -62,13 +62,13 @@ INCDIRS = ../piqirun-ocaml # USE_CAMLP4 = yes -piqobj.ml: $(PIQI_ROOT)/piqi/piqobj.ml.m4 +piqobj.ml: $(PIQI_ROOT)/piqilib/piqobj.ml.m4 m4 $< >$@ piqi_version.ml: $(PIQI_ROOT)/VERSION echo "let version = \"`head -1 $<`\"" >$@ -$(PIQI_SOURCES): % : $(PIQI_ROOT)/piqi/% +$(PIQI_SOURCES): % : $(PIQI_ROOT)/piqilib/% ln -s $< $@ diff --git a/piqi/Makefile.piqi_extended b/piqilib/Makefile.piqi_extended similarity index 54% rename from piqi/Makefile.piqi_extended rename to piqilib/Makefile.piqi_extended index 7ab4d5f5..f816b54c 100644 --- a/piqi/Makefile.piqi_extended +++ b/piqilib/Makefile.piqi_extended @@ -1,4 +1,4 @@ -include $(PIQI_ROOT)/piqi/Makefile.piqi_common +include $(PIQI_ROOT)/piqilib/Makefile.piqi_common PIQI_SOURCES_EXTENDED = \ @@ -9,6 +9,6 @@ PIQI_SOURCES_EXTENDED = \ PIQI_SOURCES_TAIL += $(PIQI_SOURCES_EXTENDED) -$(PIQI_SOURCES_EXTENDED): % : $(PIQI_ROOT)/piqi/% +$(PIQI_SOURCES_EXTENDED): % : $(PIQI_ROOT)/piqilib/% ln -s $< $@ diff --git a/piqilib/Makefile.piqilib b/piqilib/Makefile.piqilib new file mode 100644 index 00000000..9279717b --- /dev/null +++ b/piqilib/Makefile.piqilib @@ -0,0 +1,44 @@ +include $(PIQI_ROOT)/piqilib/Makefile.piqi_extended + + +PIQI_SOURCES_LIB = \ + piqi_json_common.ml \ + piqi_json_parser.mll \ + piqi_json_gen.ml \ + piqi_json.ml \ + \ + piqi_base64.ml \ + piqobj_to_json.ml \ + piqobj_of_json.ml \ + \ + piqi_pp.ml \ + piq.ml \ + \ + piqi_light.ml \ + piqi_getopt.ml \ + + +PIQI_SOURCES_TAIL += $(PIQI_SOURCES_LIB) + + +PIQI_PIQI = piqi-boot.piqi piqi.piqi piqi-impl.piqi + + +$(PIQI_SOURCES_LIB) $(PIQI_PIQI): % : $(PIQI_ROOT)/piqilib/% + ln -s $< $@ + + +PIQICC = ../piqicc/piqicc +PIQICC_FLAGS = -I $(PIQI_ROOT) + + +piqtype.ml: $(PIQI_PIQI) + # Note, using standard piqi.org/piqi-boot as boot module for piqicc + $(PIQICC) $(PIQICC_FLAGS) -o $@.tmp.ml \ + --boot piqi-boot.piqi --piqi piqi.piqi --impl piqi-impl.piqi + $(CAMLP4)o -o $@ $@.tmp.ml + + +clean:: + rm -f piqtype.ml.tmp.ml + diff --git a/piqi/piq.ml b/piqilib/piq.ml similarity index 100% rename from piqi/piq.ml rename to piqilib/piq.ml diff --git a/piqi/piq_gen.ml b/piqilib/piq_gen.ml similarity index 100% rename from piqi/piq_gen.ml rename to piqilib/piq_gen.ml diff --git a/piqi/piq_lexer.ml b/piqilib/piq_lexer.ml similarity index 100% rename from piqi/piq_lexer.ml rename to piqilib/piq_lexer.ml diff --git a/piqi/piq_parser.ml b/piqilib/piq_parser.ml similarity index 100% rename from piqi/piq_parser.ml rename to piqilib/piq_parser.ml diff --git a/piqi/piqi-boot.piqi b/piqilib/piqi-boot.piqi similarity index 100% rename from piqi/piqi-boot.piqi rename to piqilib/piqi-boot.piqi diff --git a/piqi/piqi-impl.piqi b/piqilib/piqi-impl.piqi similarity index 100% rename from piqi/piqi-impl.piqi rename to piqilib/piqi-impl.piqi diff --git a/piqi/piqi.ml b/piqilib/piqi.ml similarity index 100% rename from piqi/piqi.ml rename to piqilib/piqi.ml diff --git a/piqi/piqi.piqi b/piqilib/piqi.piqi similarity index 100% rename from piqi/piqi.piqi rename to piqilib/piqi.piqi diff --git a/piqi/base64.ml b/piqilib/piqi_base64.ml similarity index 100% rename from piqi/base64.ml rename to piqilib/piqi_base64.ml diff --git a/piqi/piqi_c.idl b/piqilib/piqi_c.idl similarity index 100% rename from piqi/piqi_c.idl rename to piqilib/piqi_c.idl diff --git a/piqi/piqi_c.ml b/piqilib/piqi_c.ml similarity index 100% rename from piqi/piqi_c.ml rename to piqilib/piqi_c.ml diff --git a/piqi/piqi_c.mli b/piqilib/piqi_c.mli similarity index 100% rename from piqi/piqi_c.mli rename to piqilib/piqi_c.mli diff --git a/piqi/piqi_c_impl.c b/piqilib/piqi_c_impl.c similarity index 100% rename from piqi/piqi_c_impl.c rename to piqilib/piqi_c_impl.c diff --git a/piqi/piqi_common.ml b/piqilib/piqi_common.ml similarity index 100% rename from piqi/piqi_common.ml rename to piqilib/piqi_common.ml diff --git a/piqi/piqi_config.ml b/piqilib/piqi_config.ml similarity index 100% rename from piqi/piqi_config.ml rename to piqilib/piqi_config.ml diff --git a/piqi/piqi_db.ml b/piqilib/piqi_db.ml similarity index 100% rename from piqi/piqi_db.ml rename to piqilib/piqi_db.ml diff --git a/piqilib/piqi_ext.ml b/piqilib/piqi_ext.ml new file mode 100644 index 00000000..b5615afb --- /dev/null +++ b/piqilib/piqi_ext.ml @@ -0,0 +1,91 @@ +(*pp camlp4o -I $PIQI_ROOT/camlp4 pa_labelscope.cmo pa_openin.cmo *) +(* + Copyright 2009, 2010, 2011 Anton Lavrik + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*) + + +(* This module contains functionality that Piqi module would contain, but if it + * did, it would break Piqicc bootstrap that doesn't support advanced Piqi + * functionality such as functions. *) + +(* XXX: include this module from Piqi? *) + + +open Piqi_common +open Piqi + + +let get_functions modules = + flatmap (fun x -> x.P#func) modules + + +let get_resolved_functions modules = + flatmap (fun x -> x.P#resolved_func) modules + + +(* expand all includes and, optionally, extensions and produce an expanded + * version of the Piqi module *) +let expand_piqi ?(includes_only=false) piqi = + let open P in + let all_piqi = piqi.included_piqi in + let orig_piqi = some_of piqi.original_piqi in + + (* create a new piqi module from the original piqi module *) + let res_piqi = + { + T.default_piqi () with + + modname = orig_piqi.modname; + proto_package = orig_piqi.proto_package; + piqdef = []; + extend = []; + includ = []; + import = []; + func = []; + + custom_field = []; + + (* piqi-impl (implementation related) extensions *) + extended_piqdef = []; + resolved_piqdef = []; + imported_piqdef = []; + resolved_import = []; + resolved_func = []; + included_piqi = []; + original_piqi = None; + } + in + + (* copy all imports to the resulting module *) + let imports = get_imports all_piqi in + res_piqi.import <- imports; + + (* copy all definitions to the resulting module *) + res_piqi.piqdef <- + if includes_only + then get_piqdefs all_piqi + else piqi.extended_piqdef; + + (* copy all extensions to the resulting module *) + res_piqi.extend <- + if includes_only + then get_extensions all_piqi + else []; + + (* copy all functions to the resulting module *) + res_piqi.func <- get_functions all_piqi; + + res_piqi + diff --git a/piqi/piqi_file.ml b/piqilib/piqi_file.ml similarity index 100% rename from piqi/piqi_file.ml rename to piqilib/piqi_file.ml diff --git a/piqi/piqi_func.ml b/piqilib/piqi_func.ml similarity index 100% rename from piqi/piqi_func.ml rename to piqilib/piqi_func.ml diff --git a/piqi/piqi_getopt.ml b/piqilib/piqi_getopt.ml similarity index 59% rename from piqi/piqi_getopt.ml rename to piqilib/piqi_getopt.ml index 6a7ebf09..631f386f 100644 --- a/piqi/piqi_getopt.ml +++ b/piqilib/piqi_getopt.ml @@ -17,8 +17,7 @@ (* - * Command-line element parsing, pretty-printing and converting to various - * output formats. + * Interpreting command-line arguments as Piq data *) @@ -138,20 +137,8 @@ let parse_argv start = aux start -(* index of the "--" element in argv array *) -let argv_start_index = ref 0 - -(* command-line arguments *) -let output_encoding = ref "" -let typename = ref "" - - -module Main = Piqi_main -open Main - - -let getopt_piq () = - let tokens = parse_argv !argv_start_index in +let getopt_piq argv_start_index = + let tokens = parse_argv argv_start_index in let piq_parser = Piq_parser.init_from_token_list getopt_filename tokens in let piq_objects = Piq_parser.read_all piq_parser in let piq_ast = @@ -166,75 +153,3 @@ let getopt_piq () = in piq_ast - -let validate_options () = - if !typename = "" (* pretty-print mode *) - then ( - if !output_encoding <> "" - then piqi_error "option -t can not be used without --piqtype"; - ) - - -let getopt_command () = - validate_options (); - (* open output file *) - let och = Main.open_output !ofile in - (* interpret command-line arguments after "--" as Piq data *) - let piq_ast = getopt_piq () in - match piq_ast with - | None -> () (* no data *) - | Some ast when !typename = "" -> - (* with no --piqtype parameter given, just pretty-print the Piq AST *) - Piqi_pp.prettyprint_ast och ast; - output_char och '\n' - | Some ast -> - let writer = Piqi_convert.make_writer !output_encoding in - let piqtype = Piqi_convert.find_piqtype !typename in - (* parse the Piq AST according to "--piqtype" and convert to the output - * format according to "-t" *) - Piqobj_of_piq.resolve_defaults := !Piqi_convert.flag_add_defaults; - Piqobj_of_piq.parse_words_as_strings := true; - let piqobj = Piqobj_of_piq.parse_obj piqtype ast in - (* write the object *) - writer och (Piq.Typed_piqobj piqobj) - - -(* find the position of the first argument after "--" *) -let rest_fun arg = - if !argv_start_index = 0 (* first argument after first occurence of "--" *) - then argv_start_index := !Arg.current + 1 - else () - - -let usage = "Usage: piqi getopt [options] -- [] \nOptions:" - - -let speclist = Main.common_speclist @ - [ - arg_o; - - "-t", Arg.Set_string output_encoding, - "piq|wire|pb|json|piq-json output encoding (piq is used by default)"; - - "--piqtype", Arg.Set_string typename, - " type of the object represented by data arguments"; - - "--add-defaults", Arg.Set Piqi_convert.flag_add_defaults, - "add field default values while converting records"; - - "--", Arg.Rest rest_fun, - "separator between piqi command-line arguments adn data arguments"; - ] - - -let run () = - Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:0; - if !argv_start_index = 0 (* "--" is not present in the list of arguments *) - then argv_start_index := Array.length Sys.argv; - getopt_command () - - -let _ = - Main.register_command run "getopt" - "interpret command-line arguments as Piq data, pretty-print and convert to various encodings" - diff --git a/piqi/piqi_iolist.ml b/piqilib/piqi_iolist.ml similarity index 100% rename from piqi/piqi_iolist.ml rename to piqilib/piqi_iolist.ml diff --git a/piqi/piqi_json.ml b/piqilib/piqi_json.ml similarity index 79% rename from piqi/piqi_json.ml rename to piqilib/piqi_json.ml index 42536c6b..fc9c5fbc 100644 --- a/piqi/piqi_json.ml +++ b/piqilib/piqi_json.ml @@ -161,22 +161,7 @@ let init () = Piqi.register_processing_hook json_name_piqi -module Main = Piqi_main -open Main - - -let usage = "Usage: piqi json-pp [options] [<.json file>] [output file]\nOptions:" - - -let speclist = Main.common_speclist @ - [ - arg_o; - arg__; - ] - - -let prettyprint_json ch json = - Piqi_json_gen.pretty_to_channel ch json +(**) let open_json fname = @@ -189,33 +174,3 @@ let read_json_obj json_parser = let res = Piqi_json_parser.read_next json_parser in res - -let prettyprint_json ch json_parser = - let rec aux () = - match read_json_obj json_parser with - | None -> () - | Some json -> - (* reset location db to allow GC to collect previously read objects *) - Piqloc.reset (); - prettyprint_json ch json; - output_string ch "\n\n"; - aux () - in aux () - - -let prettyprint_file filename = - let ch = Main.open_output !ofile in - (* switch parser/generator to pretty-print mode *) - Config.pp_mode := true; - let json_parser = open_json filename in - prettyprint_json ch json_parser - - -let run () = - Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:2; - prettyprint_file !ifile - - -let _ = - Main.register_command run "json-pp" "pretty-print %.json" - diff --git a/piqi/piqi_json_common.ml b/piqilib/piqi_json_common.ml similarity index 100% rename from piqi/piqi_json_common.ml rename to piqilib/piqi_json_common.ml diff --git a/piqi/piqi_json_gen.ml b/piqilib/piqi_json_gen.ml similarity index 100% rename from piqi/piqi_json_gen.ml rename to piqilib/piqi_json_gen.ml diff --git a/piqi/piqi_json_parser.mll b/piqilib/piqi_json_parser.mll similarity index 100% rename from piqi/piqi_json_parser.mll rename to piqilib/piqi_json_parser.mll diff --git a/piqi/piqi_light.ml b/piqilib/piqi_light.ml similarity index 91% rename from piqi/piqi_light.ml rename to piqilib/piqi_light.ml index 16421ecb..d475ff60 100644 --- a/piqi/piqi_light.ml +++ b/piqilib/piqi_light.ml @@ -17,7 +17,7 @@ (* - * Light syntax for Piqi DDL: Piqi to Piqi-light pretty-printer + * Light syntax for Piqi DDL. *) @@ -285,30 +285,3 @@ let gen_piqi ch (piqi:T.piqi) = in Iolist.to_channel ch code - -let print_piqi_file ch filename = - let piqi = Piqi.load_piqi filename in - gen_piqi ch piqi - - -module Main = Piqi_main -open Main - -let usage = "Usage: piqi light [options] [<.piqi file>] [output-file]\nOptions:" - -let speclist = Main.common_speclist @ - [ - arg_o; - ] - - -let run () = - Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:2; - let ch = Main.open_output !ofile in - print_piqi_file ch !ifile - - -let _ = - Main.register_command run "light" - "pretty-print %.piqi using Piqi-light syntax" - diff --git a/piqi/piqi_main.ml b/piqilib/piqi_main.ml similarity index 100% rename from piqi/piqi_main.ml rename to piqilib/piqi_main.ml diff --git a/piqi/piqi_name.ml b/piqilib/piqi_name.ml similarity index 100% rename from piqi/piqi_name.ml rename to piqilib/piqi_name.ml diff --git a/piqi/piqi_pp.ml b/piqilib/piqi_pp.ml similarity index 70% rename from piqi/piqi_pp.ml rename to piqilib/piqi_pp.ml index e1d333d8..70326673 100644 --- a/piqi/piqi_pp.ml +++ b/piqilib/piqi_pp.ml @@ -19,51 +19,6 @@ open Piqi_common -let map_words (x:T.ast) f = - let rec aux = function - | `word s -> `word (f s) - | `name s -> `name (f s) - | `named ({T.Named.value = v} as x) -> - `named {x with T.Named.value = aux v} - | `typed ({T.Typed.value = v} as x) -> - let ast = some_of v.T.Any.ast in - let v = {v with T.Any.ast = Some (aux ast)} in - `typed {x with T.Typed.value = v} - | `list l -> `list (List.map aux l) - | `control l -> `control (List.map aux l) - | x -> x - in aux x - - -let normalize ast = - map_words ast Piqi_name.normalize_name - - -module Main = Piqi_main -open Main - - -let flag_normalize = ref false -let flag_expand_abbr = ref false - - -let usage = "Usage: piqi pp [options] [<.piqi|.piq file>] [output file]\nOptions:" - - -let speclist = Main.common_speclist @ - [ - arg_o; - - "--normalize-words", Arg.Set flag_normalize, - "normalize all words while pretty-printing (convert CamelCase to camel-case)"; - - "--expand-abbr", Arg.Set flag_expand_abbr, - "expand built-in syntax abbreviations"; - - arg__; - ] - - (* old method for pretty-printing: let prettyprint_ast ast = let code = Piq_gen.print_ast ast in @@ -231,58 +186,3 @@ let prettyprint_piqi ch (piqi:T.piqi) = let ast = piqi_to_ast piqi ~simplify:true in prettyprint_piqi_ast ch ast - -let transform_ast ast = - let ast = - if !flag_normalize - then normalize ast - else ast - in - let ast = - if !flag_expand_abbr - then Piq_parser.expand ast - else ast - in ast - - -let open_piq fname = - let ch = Piqi_main.open_input fname in - let piq_parser = Piq_parser.init_from_channel fname ch in - piq_parser - - -let read_piq_obj piq_parser = - let res = Piq_parser.read_next piq_parser in - (* reset location db to allow GC to collect previously read objects *) - Piqloc.reset (); - res - - -let prettyprint_piq ch piq_parser = - let rec aux () = - match read_piq_obj piq_parser with - | None -> () - | Some ast -> - let ast = transform_ast ast in - prettyprint_ast ch ast; - output_char ch '\n'; - aux () - in aux () - - -let prettyprint_file filename = - let ch = Main.open_output !ofile in - (* switch piq parser/generator to pretty-print mode *) - Config.pp_mode := true; - let piq_parser = open_piq filename in - prettyprint_piq ch piq_parser - - -let run () = - Main.parse_args () ~speclist ~usage ~min_arg_count:0 ~max_arg_count:2; - prettyprint_file !ifile - - -let _ = - Main.register_command run "pp" "pretty-print %.piqi or %.piq" - diff --git a/piqi/piqi_wire.ml b/piqilib/piqi_wire.ml similarity index 100% rename from piqi/piqi_wire.ml rename to piqilib/piqi_wire.ml diff --git a/piqi/piqloc.ml b/piqilib/piqloc.ml similarity index 100% rename from piqi/piqloc.ml rename to piqilib/piqloc.ml diff --git a/piqi/piqobj.ml.m4 b/piqilib/piqobj.ml.m4 similarity index 100% rename from piqi/piqobj.ml.m4 rename to piqilib/piqobj.ml.m4 diff --git a/piqi/piqobj.piqi b/piqilib/piqobj.piqi similarity index 100% rename from piqi/piqobj.piqi rename to piqilib/piqobj.piqi diff --git a/piqi/piqobj_common.ml b/piqilib/piqobj_common.ml similarity index 100% rename from piqi/piqobj_common.ml rename to piqilib/piqobj_common.ml diff --git a/piqi/piqobj_of_json.ml b/piqilib/piqobj_of_json.ml similarity index 99% rename from piqi/piqobj_of_json.ml rename to piqilib/piqobj_of_json.ml index b2dd8cfe..e391b95c 100644 --- a/piqi/piqobj_of_json.ml +++ b/piqilib/piqobj_of_json.ml @@ -74,7 +74,7 @@ let parse_string (x:json) = match x with let parse_binary (x:json) = match x with | `String x -> - (try Base64.decode x + (try Piqi_base64.decode x with Invalid_argument _ -> error x "invalid base64-encoded string" ) diff --git a/piqi/piqobj_of_piq.ml b/piqilib/piqobj_of_piq.ml similarity index 100% rename from piqi/piqobj_of_piq.ml rename to piqilib/piqobj_of_piq.ml diff --git a/piqi/piqobj_of_wire.ml b/piqilib/piqobj_of_wire.ml similarity index 100% rename from piqi/piqobj_of_wire.ml rename to piqilib/piqobj_of_wire.ml diff --git a/piqi/piqobj_to_json.ml b/piqilib/piqobj_to_json.ml similarity index 98% rename from piqi/piqobj_to_json.ml rename to piqilib/piqobj_to_json.ml index 5598b76b..3cacb1af 100644 --- a/piqi/piqobj_to_json.ml +++ b/piqilib/piqobj_to_json.ml @@ -53,7 +53,7 @@ let rec gen_obj (x:Piqobj.obj) :json = | `float x -> `Float x | `bool x -> `Bool x | `string x -> `String x - | `binary x -> `String (Base64.encode x) + | `binary x -> `String (Piqi_base64.encode x) | `word x -> `String x | `text x -> `String x | `any x -> gen_any x diff --git a/piqi/piqobj_to_piq.ml b/piqilib/piqobj_to_piq.ml similarity index 100% rename from piqi/piqobj_to_piq.ml rename to piqilib/piqobj_to_piq.ml diff --git a/piqi/piqobj_to_wire.ml b/piqilib/piqobj_to_wire.ml similarity index 100% rename from piqi/piqobj_to_wire.ml rename to piqilib/piqobj_to_wire.ml diff --git a/tests/erlang/Makefile b/tests/erlang/Makefile index 8cd04f42..d184e8ff 100644 --- a/tests/erlang/Makefile +++ b/tests/erlang/Makefile @@ -6,7 +6,7 @@ all: prep test prep: ln -sf ../../piqi.org/piqi.piqi . ln -sf ../../piqi.org/piqtype.piqi . - ln -sf ../../piqi/piqobj.piqi . + ln -sf ../../piqilib/piqobj.piqi . piqi expand piqi.piqi > piqi-expanded.piqi piqi convert --add-defaults -t pb -o piqi.piqi.pb piqi-expanded.piqi diff --git a/tests/ocaml/Makefile b/tests/ocaml/Makefile index 44ba2eae..3d5c048f 100644 --- a/tests/ocaml/Makefile +++ b/tests/ocaml/Makefile @@ -4,7 +4,7 @@ all: prep test prep: ln -sf ../../piqi.org/piqi.piqi . ln -sf ../../piqi.org/piqtype.piqi . - ln -sf ../../piqi/piqobj.piqi . + ln -sf ../../piqilib/piqobj.piqi . piqi expand piqi.piqi > piqi-expanded.piqi piqi convert --add-defaults -t pb -o piqi.piqi.pb piqi-expanded.piqi