From 03301049d9bf99408476b9f61d4498b63b287d7a Mon Sep 17 00:00:00 2001 From: Marek Kubica Date: Mon, 28 Feb 2022 15:17:25 +0100 Subject: [PATCH] Add `suf` separator to configure what terminates JSON values This is a generalization of `newline`, but it works for all writing functions and can be overridden. --- CHANGES.md | 7 ++-- lib/write.ml | 40 +++++++++++----------- lib/write.mli | 28 ++++++++++++---- test/test_write.ml | 82 ++++++++++++++++++++++++---------------------- 4 files changed, 88 insertions(+), 69 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3c041890..c435acec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,11 +23,14 @@ (@tmcgilchrist, #117) - Add a benchmark to judge the respective performance of providing a buffer vs letting Yojson create an internal (#134, @Leonidas-from-XIV) +- Add an optional `suf` keyword argument was added to functions that write + serialized JSON, thus allowing NDJSON output. Most functions default to not + adding any suffix except for `to_file` (#124, @panglesd) and functions + writing sequences of values where the default is `\n` (#135, + @Leonidas-from-XIV) ### Change -- The function `to_file` now adds a newline at the end of the generated file. An - optional argument allows to return to the original behaviour (#124, @panglesd) - The `stream_from_*` and `stream_to_*` functions now use a `Seq.t` instead of a `Stream.t`, and they are renamed into `seq_from_*` and `seq_to_*` (@gasche, #131). diff --git a/lib/write.ml b/lib/write.ml index 45a2dcb0..007e4e89 100644 --- a/lib/write.ml +++ b/lib/write.ml @@ -417,14 +417,14 @@ and write_std_variant ob s o = #endif -let to_buffer ?(std = false) ob x = +let to_buffer ?(suf = "") ?(std = false) ob x = if std then write_std_json ob x else - write_json ob x - + write_json ob x; + Buffer.add_string ob suf -let to_string ?buf ?(len = 256) ?std x = +let to_string ?buf ?(len = 256) ?(suf = "") ?std x = let ob = match buf with None -> Buffer.create len @@ -432,46 +432,44 @@ let to_string ?buf ?(len = 256) ?std x = Buffer.clear ob; ob in - to_buffer ?std ob x; + to_buffer ~suf ?std ob x; let s = Buffer.contents ob in Buffer.clear ob; s -let to_channel ?buf ?(len=4096) ?std oc x = +let to_channel ?buf ?(len=4096) ?(suf = "") ?std oc x = let ob = match buf with None -> Buffer.create len | Some ob -> Buffer.clear ob; ob in - to_buffer ?std ob x; + to_buffer ~suf ?std ob x; Buffer.output_buffer oc ob; Buffer.clear ob -let to_output ?buf ?(len=4096) ?std out x = +let to_output ?buf ?(len=4096) ?(suf = "") ?std out x = let ob = match buf with None -> Buffer.create len | Some ob -> Buffer.clear ob; ob in - to_buffer ?std ob x; + to_buffer ~suf ?std ob x; out#output (Buffer.contents ob) 0 (Buffer.length ob); Buffer.clear ob -let to_file ?len ?std ?(newline = true) file x = +let to_file ?len ?std ?(suf = "\n") file x = let oc = open_out file in try - to_channel ?len ?std oc x; - if newline then - output_string oc "\n"; + to_channel ?len ~suf ?std oc x; close_out oc with e -> close_out_noerr oc; raise e -let seq_to_buffer ?std ob st = - Seq.iter (to_buffer ?std ob) st +let seq_to_buffer ?(suf = "\n") ?std ob st = + Seq.iter (to_buffer ~suf ?std ob) st -let seq_to_string ?buf ?(len = 256) ?std st = +let seq_to_string ?buf ?(len = 256) ?(suf = "\n") ?std st = let ob = match buf with None -> Buffer.create len @@ -479,27 +477,27 @@ let seq_to_string ?buf ?(len = 256) ?std st = Buffer.clear ob; ob in - seq_to_buffer ?std ob st; + seq_to_buffer ~suf ?std ob st; let s = Buffer.contents ob in Buffer.clear ob; s -let seq_to_channel ?buf ?(len=2096) ?std oc seq = +let seq_to_channel ?buf ?(len=2096) ?(suf = "\n") ?std oc seq = let ob = match buf with None -> Buffer.create len | Some ob -> Buffer.clear ob; ob in Seq.iter (fun json -> - to_buffer ?std ob json; + to_buffer ~suf ?std ob json; Buffer.output_buffer oc ob; Buffer.clear ob; ) seq -let seq_to_file ?len ?std file st = +let seq_to_file ?len ?(suf = "\n") ?std file st = let oc = open_out file in try - seq_to_channel ?len ?std oc st; + seq_to_channel ?len ~suf ?std oc st; close_out oc with e -> close_out_noerr oc; diff --git a/lib/write.mli b/lib/write.mli index 37ddf8ba..03e1e705 100644 --- a/lib/write.mli +++ b/lib/write.mli @@ -3,6 +3,7 @@ val to_string : ?buf:Buffer.t -> ?len:int -> + ?suf:string -> ?std:bool -> t -> string (** Write a compact JSON value to a string. @@ -10,6 +11,8 @@ val to_string : [Buffer.create]. The buffer is cleared of all contents before starting and right before returning. @param len initial length of the output buffer. + @param suf appended to the output as a suffix, + defaults to empty string. @param std use only standard JSON syntax, i.e. convert tuples and variants into standard JSON (if applicable), refuse to print NaN and infinities, @@ -20,6 +23,7 @@ val to_string : val to_channel : ?buf:Buffer.t -> ?len:int -> + ?suf:string -> ?std:bool -> out_channel -> t -> unit (** Write a compact JSON value to a channel. @@ -30,6 +34,7 @@ val to_channel : val to_output : ?buf:Buffer.t -> ?len:int -> + ?suf:string -> ?std:bool -> < output : string -> int -> int -> int; .. > -> t -> unit (** Write a compact JSON value to an OO channel. @@ -39,14 +44,15 @@ val to_output : val to_file : ?len:int -> ?std:bool -> - ?newline:bool -> + ?suf:string -> string -> t -> unit (** Write a compact JSON value to a file. See [to_string] for the role of the optional arguments. - @param newline whether to end the content of the file with a new line. - Optional parameter with value [true] by default. *) + @param suf is a suffix appended to the output Newline by default + for POSIX compliance. *) val to_buffer : + ?suf:string -> ?std:bool -> Buffer.t -> t -> unit (** Write a compact JSON value to an existing buffer. @@ -55,35 +61,43 @@ val to_buffer : val seq_to_string : ?buf:Buffer.t -> ?len:int -> + ?suf:string -> ?std:bool -> t Seq.t -> string - (** Write a newline-separated sequence of compact one-line JSON values to + (** Write a sequence of [suf]-suffixed compact one-line JSON values to a string. + @param suf is the suffix ouf each value written. Newline by default. See [to_string] for the role of the optional arguments. *) val seq_to_channel : ?buf:Buffer.t -> ?len:int -> + ?suf:string -> ?std:bool -> out_channel -> t Seq.t -> unit - (** Write a newline-separated sequence of compact one-line JSON values to + (** Write a sequence of [suf]-suffixed compact one-line JSON values to a channel. + @param suf is the suffix of each value written. Newline by default. See [to_channel] for the role of the optional arguments. *) val seq_to_file : ?len:int -> + ?suf:string -> ?std:bool -> string -> t Seq.t -> unit - (** Write a newline-separated sequence of compact one-line JSON values to + (** Write a sequence of [suf]-suffixed compact one-line JSON values to a file. + @param suf is the suffix of each value written. Newline by default. See [to_string] for the role of the optional arguments. *) val seq_to_buffer : + ?suf:string -> ?std:bool -> Buffer.t -> t Seq.t -> unit - (** Write a newline-separated sequence of compact one-line JSON values to + (** Write a sequence of [suf]-suffixed compact one-line JSON values to an existing buffer. + @param suf is the suffix of each value written. Newline by default. See [to_string] for the role of the optional arguments. *) val write_t : Buffer.t -> t -> unit diff --git a/test/test_write.ml b/test/test_write.ml index db2c012b..d3b9211a 100644 --- a/test/test_write.ml +++ b/test/test_write.ml @@ -1,20 +1,17 @@ -let to_string () = - Alcotest.(check string) __LOC__ Fixtures.json_string (Yojson.Safe.to_string Fixtures.json_value) +let to_string_tests = + let test ?suf expected = + Alcotest.(check string) __LOC__ expected (Yojson.Safe.to_string ?suf Fixtures.json_value) + in + [ + "to_string with default settings", `Quick, (fun () -> test Fixtures.json_string); + "to_string with newline", `Quick, (fun () -> test ~suf:"\n" Fixtures.json_string_newline); + "to_string without newline", `Quick, (fun () -> test ~suf:"" Fixtures.json_string); + ] -let to_file () = - let test ?newline () = +let to_file_tests = + let test ?suf expected = let output_file = Filename.temp_file "test_yojson_to_file" ".json" in - let correction = match newline with - | None -> - Yojson.Safe.to_file output_file Fixtures.json_value; - Fixtures.json_string_newline - | Some newline -> - Yojson.Safe.to_file ~newline output_file Fixtures.json_value; - if newline then - Fixtures.json_string_newline - else - Fixtures.json_string - in + Yojson.Safe.to_file ?suf output_file Fixtures.json_value; let file_content = let ic = open_in output_file in let length = in_channel_length ic in @@ -22,36 +19,43 @@ let to_file () = close_in ic; s in - Alcotest.(check string) __LOC__ correction file_content; - Sys.remove output_file + Sys.remove output_file; + Alcotest.(check string) __LOC__ expected file_content in - test (); - test ~newline:true (); - test ~newline:false () + [ + "to_file with default settings", `Quick, (fun () -> test Fixtures.json_string_newline); + "to_file with newline", `Quick, (fun () -> test ~suf:"\n" Fixtures.json_string_newline); + "to_file without newline", `Quick, (fun () -> test ~suf:"" Fixtures.json_string); + ] (* List.to_seq is not available on old OCaml versions. *) let rec list_to_seq = function | [] -> (fun () -> Seq.Nil) | x :: xs -> (fun () -> Seq.Cons (x, list_to_seq xs)) -let seq_to_file () = - let output_file = Filename.temp_file "test_yojson_seq_to_file" ".json" in - let data = [`String "foo"; `String "bar"] in - Yojson.Safe.seq_to_file output_file (list_to_seq data); - let read_data = - let seq = Yojson.Safe.seq_from_file output_file in - let acc = ref [] in - Seq.iter (fun v -> acc := v :: !acc) seq; - List.rev !acc +let seq_to_file_tests = + let test ?suf () = + let output_file = Filename.temp_file "test_yojson_seq_to_file" ".json" in + let data = [`String "foo"; `String "bar"] in + Yojson.Safe.seq_to_file ?suf output_file (list_to_seq data); + let read_data = + let seq = Yojson.Safe.seq_from_file output_file in + let acc = ref [] in + Seq.iter (fun v -> acc := v :: !acc) seq; + List.rev !acc + in + Sys.remove output_file; + Alcotest.(check (list Testable.yojson)) "seq_{to,from}_file roundtrip" data read_data in - Sys.remove output_file; - if data <> read_data then - (* TODO: it would be nice to use Alcotest.check, - but we don't have a 'testable' instance for JSON values. *) - Alcotest.fail "seq_{to,from}_file roundtrip failure" + [ + "seq_to_file with default settings", `Quick, (fun () -> test ()); + "seq_to_file with newline", `Quick, (fun () -> test ~suf:"\n" ()); + "seq_to_file without newline", `Quick, (fun () -> test ~suf:"" ()); + ] -let single_json = [ - "to_string", `Quick, to_string; - "to_file", `Quick, to_file; - "seq_to_file", `Quick, seq_to_file; -] +let single_json = + List.flatten [ + to_file_tests; + to_string_tests; + seq_to_file_tests; + ]