From 380262a7a65355c3ded16a8c24d5a396cb3cf0f1 Mon Sep 17 00:00:00 2001 From: Florian Verdonck Date: Sun, 3 May 2020 18:34:13 +0200 Subject: [PATCH] Respect byte-order mark. (#799) * Respect byte-order mark. Fixes 795 * Removed helper functions and expose runFantomasTool * Add remark about UTF-8 --- .../ByteOrderMarkTests.fs | 40 +++++++++++++++++++ .../Fantomas.CoreGlobalTool.Tests.fsproj | 1 + .../TestHelpers.fs | 20 ++++++++-- src/Fantomas.CoreGlobalTool/Program.fs | 29 +++++++++++++- 4 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 src/Fantomas.CoreGlobalTool.Tests/ByteOrderMarkTests.fs diff --git a/src/Fantomas.CoreGlobalTool.Tests/ByteOrderMarkTests.fs b/src/Fantomas.CoreGlobalTool.Tests/ByteOrderMarkTests.fs new file mode 100644 index 0000000000..01c5adea27 --- /dev/null +++ b/src/Fantomas.CoreGlobalTool.Tests/ByteOrderMarkTests.fs @@ -0,0 +1,40 @@ +module Fantomas.CoreGlobalTool.Tests.ByteOrderMarkTests + +open System.IO +open NUnit.Framework +open FsUnit +open Fantomas.CoreGlobalTool.Tests.TestHelpers +open System.Text + +[] +let Source = "namespace Company.Product.Feature" + +let private getInitialBytes file = + use file = new FileStream(file, FileMode.Open, FileAccess.Read) + let mutable bom = Array.zeroCreate 3 + file.Read(bom, 0, 3) |> ignore + bom + +[] +let ``byte-order mark should be preserved, 795``() = + use fileFixture = new TemporaryFileCodeSample(Source, true) + let exitCode = runFantomasTool fileFixture.Filename + exitCode |> should equal 0 + + let expectedPreamble = Encoding.UTF8.GetPreamble() + let actualPreamble = getInitialBytes fileFixture.Filename + expectedPreamble |> should equal actualPreamble + +[] +let ``preserve byte-order from original file`` () = + use inputFixture = new TemporaryFileCodeSample(Source, true) + use outputFixture = new OutputFile() + let exitCode = + sprintf "--out %s %s" outputFixture.Filename inputFixture.Filename + |> runFantomasTool + + exitCode |> should equal 0 + + let expectedPreamble = Encoding.UTF8.GetPreamble() + let actualPreamble = getInitialBytes outputFixture.Filename + expectedPreamble |> should equal actualPreamble diff --git a/src/Fantomas.CoreGlobalTool.Tests/Fantomas.CoreGlobalTool.Tests.fsproj b/src/Fantomas.CoreGlobalTool.Tests/Fantomas.CoreGlobalTool.Tests.fsproj index d4dd4064f7..061fec5b36 100644 --- a/src/Fantomas.CoreGlobalTool.Tests/Fantomas.CoreGlobalTool.Tests.fsproj +++ b/src/Fantomas.CoreGlobalTool.Tests/Fantomas.CoreGlobalTool.Tests.fsproj @@ -14,6 +14,7 @@ + \ No newline at end of file diff --git a/src/Fantomas.CoreGlobalTool.Tests/TestHelpers.fs b/src/Fantomas.CoreGlobalTool.Tests/TestHelpers.fs index a6c99a414e..6ed5a08900 100644 --- a/src/Fantomas.CoreGlobalTool.Tests/TestHelpers.fs +++ b/src/Fantomas.CoreGlobalTool.Tests/TestHelpers.fs @@ -3,16 +3,28 @@ module Fantomas.CoreGlobalTool.Tests.TestHelpers open System open System.Diagnostics open System.IO +open System.Text -type TemporaryFileCodeSample internal (codeSnippet: string) = +type TemporaryFileCodeSample internal (codeSnippet: string, ?hasByteOrderMark: bool) = + let hasByteOrderMark = defaultArg hasByteOrderMark false let filename = Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString() + ".fs") - do File.WriteAllText(filename, codeSnippet) + do (if hasByteOrderMark + then File.WriteAllText(filename, codeSnippet, Encoding.UTF8) + else File.WriteAllText(filename, codeSnippet)) member _.Filename: string = filename interface IDisposable with member this.Dispose(): unit = File.Delete(filename) -let private runFantomasTool arguments = +type OutputFile internal () = + let filename = Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString() + ".fs") + member _.Filename: string = filename + interface IDisposable with + member this.Dispose(): unit = + if File.Exists(filename) then + File.Delete(filename) + +let runFantomasTool arguments = let pwd = Path.GetDirectoryName(typeof.Assembly.Location) let configuration = #if DEBUG @@ -36,4 +48,4 @@ let private runFantomasTool arguments = let checkCode file = let arguments = sprintf "--check \"%s\"" file - runFantomasTool arguments \ No newline at end of file + runFantomasTool arguments diff --git a/src/Fantomas.CoreGlobalTool/Program.fs b/src/Fantomas.CoreGlobalTool/Program.fs index a47ab2feeb..bd8182d450 100644 --- a/src/Fantomas.CoreGlobalTool/Program.fs +++ b/src/Fantomas.CoreGlobalTool/Program.fs @@ -3,6 +3,7 @@ open System.IO open Fantomas open Fantomas.FormatConfig open Argu +open System.Text let extensions = set [| ".fs"; ".fsx"; ".fsi"; ".ml"; ".mli"; |] @@ -69,13 +70,29 @@ let rec allFiles isRec path = Directory.GetFiles(path, "*.*", searchOption) |> Seq.filter (fun f -> isFSharpFile f && not (isInExcludedDir f)) +/// Fantomas assumes the input files are UTF-8 +/// As is stated in F# language spec: https://fsharp.org/specs/language-spec/4.1/FSharpSpec-4.1-latest.pdf#page=25 +let private hasByteOrderMark file = + if File.Exists(file) then + let preamble = Encoding.UTF8.GetPreamble() + use file = new FileStream(file, FileMode.Open, FileAccess.Read) + let mutable bom = Array.zeroCreate 3 + file.Read(bom, 0, 3) |> ignore + bom = preamble + else + false + /// Format a source string using given config and write to a text writer let processSourceString isFsiFile s (tw : Choice) config = let fileName = if isFsiFile then "/tmp.fsi" else "/tmp.fsx" let writeResult (formatted: string) = match tw with | Choice1Of2 tw -> tw.Write(formatted) - | Choice2Of2 path -> File.WriteAllText(path, formatted) + | Choice2Of2 path -> + if hasByteOrderMark path then + File.WriteAllText(path, formatted, Encoding.UTF8) + else + File.WriteAllText(path, formatted) async { let! formatted = s |> FakeHelpers.formatContentAsync config fileName @@ -230,7 +247,15 @@ let main argv = let fileToFile (inFile : string) (outFile : string) config = try printfn "Processing %s" inFile - use buffer = new StreamWriter(outFile) + let hasByteOrderMark = hasByteOrderMark inFile + + use buffer = + if hasByteOrderMark then + new StreamWriter(new FileStream(outFile, FileMode.OpenOrCreate, FileAccess.ReadWrite) + ,Encoding.UTF8) + else + new StreamWriter(outFile) + if profile then File.ReadLines(inFile) |> Seq.length |> printfn "Line count: %i" time (fun () -> processSourceFile inFile buffer config)