diff --git a/CHANGES.md b/CHANGES.md index 031b07d63f18..9e70cd2c08ac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -48,6 +48,10 @@ Unreleased on UNIX-like systems as a unrelated `gmake` binary might exist on Windows. (#3853, @kit-ty-kate) +- Correctly call `git ls-tree` so unicode files are not quoted, this + fixes problems with `dune subst` in the presence of unicode + files. Fixes #3219 (#3879, @ejgallego) + 2.7.1 (2/09/2020) ----------------- diff --git a/src/dune_engine/process.ml b/src/dune_engine/process.ml index 19298d9d3d33..9967af95d054 100644 --- a/src/dune_engine/process.ml +++ b/src/dune_engine/process.ml @@ -602,6 +602,8 @@ let run_capture = run_capture_gen ~f:Stdune.Io.read_file let run_capture_lines = run_capture_gen ~f:Stdune.Io.lines_of_file +let run_capture_zero_separated = run_capture_gen ~f:Stdune.Io.zero_strings_of_file + let run_capture_line ?dir ?stderr_to ?stdin_from ?env ?(purpose = Internal_job) fail_mode prog args = run_capture_gen ?dir ?stderr_to ?stdin_from ?env ~purpose fail_mode prog args diff --git a/src/dune_engine/process.mli b/src/dune_engine/process.mli index 73fd4d377291..ba7e7b5731f0 100644 --- a/src/dune_engine/process.mli +++ b/src/dune_engine/process.mli @@ -98,3 +98,14 @@ val run_capture_lines : -> Path.t -> string list -> 'a Fiber.t + +val run_capture_zero_separated : + ?dir:Path.t + -> ?stderr_to:Io.output Io.t + -> ?stdin_from:Io.input Io.t + -> ?env:Env.t + -> ?purpose:purpose + -> (string list, 'a) failure_mode + -> Path.t + -> string list + -> 'a Fiber.t diff --git a/src/dune_engine/vcs.ml b/src/dune_engine/vcs.ml index 7d42caaf0a11..22719da041f4 100644 --- a/src/dune_engine/vcs.ml +++ b/src/dune_engine/vcs.ml @@ -81,6 +81,9 @@ let run t args = let run_lines t args = Process.run_capture_lines Strict (prog t) args ~dir:t.root ~env:Env.initial +let run_zero_separated t args = + Process.run_capture_zero_separated Strict (prog t) args ~dir:t.root ~env:Env.initial + let hg_describe t = let open Fiber.O in let* s = @@ -128,6 +131,11 @@ let commit_id = ~hg:(fun t -> run t [ "id"; "-i" ]) let files = + let f_zero args t = + let open Fiber.O in + let+ l = run_zero_separated t args in + List.map l ~f:Path.in_source + in let f args t = let open Fiber.O in let+ l = run_lines t args in @@ -142,5 +150,5 @@ let files = let to_dyn = Dyn.Encoder.list Path.to_dyn end )) - ~git:(f [ "ls-tree"; "-r"; "--name-only"; "HEAD" ]) + ~git:(f_zero [ "ls-tree"; "-z"; "-r"; "--name-only"; "HEAD" ]) ~hg:(f [ "files" ]) diff --git a/src/stdune/io.ml b/src/stdune/io.ml index ecfd63372b6f..58d7a2381fa6 100644 --- a/src/stdune/io.ml +++ b/src/stdune/io.ml @@ -17,6 +17,42 @@ let input_lines = in fun ic -> loop ic [] +let input_zero_from_buffer from buf = + match Bytes.index_from_opt buf from '\x00' with + | None -> + None, from + | Some eos -> + Some (Bytes.sub_string buf ~pos:from ~len:(eos-from)), eos+1 + +let input_zero_separated = + let buf_size = 65536 in + let buf = Bytes.create buf_size in + let rec process_buf buf pos acc = + match input_zero_from_buffer pos buf with + | Some s, eos -> + process_buf buf eos (s::acc) + | None, rest -> + acc, rest + in + let rec loop ic pos acc = + let res = input ic buf pos (buf_size - pos) in + if res = 0 then + if pos > 0 then + let last = Bytes.sub_string buf ~pos:0 ~len:pos in + List.rev (last :: acc) + else + List.rev acc + else + let len = res + pos in + let act_input = Bytes.sub buf ~pos:0 ~len in + let acc, rest = process_buf act_input 0 acc in + let rem = len - rest in + let () = Bytes.blit ~src:buf ~src_pos:rest ~dst:buf ~dst_pos:0 ~len:rem in + loop ic rem acc + in + fun ic -> + loop ic 0 [] + let copy_channels = let buf_len = 65536 in let buf = Bytes.create buf_len in @@ -137,6 +173,8 @@ struct let lines_of_file fn = with_file_in fn ~f:input_lines ~binary:false + let zero_strings_of_file fn = with_file_in fn ~f:input_zero_separated ~binary:true + let write_file ?binary fn data = with_file_out ?binary fn ~f:(fun oc -> output_string oc data) diff --git a/src/stdune/io_intf.ml b/src/stdune/io_intf.ml index 44e95fccf7fc..35bf651cb259 100644 --- a/src/stdune/io_intf.ml +++ b/src/stdune/io_intf.ml @@ -15,6 +15,9 @@ module type S = sig val lines_of_file : path -> string list + (** Reads zero-separated strings from a file *) + val zero_strings_of_file : path -> string list + val read_file : ?binary:bool -> path -> string val write_file : ?binary:bool -> path -> string -> unit