Respect byte-order mark. (#799)

* Respect byte-order mark. Fixes 795

* Removed helper functions and expose runFantomasTool

* Add remark about UTF-8

src/Fantomas.CoreGlobalTool.Tests/ByteOrderMarkTests.fs

@@ -0,0 +1,40 @@
+module Fantomas.CoreGlobalTool.Tests.ByteOrderMarkTests
+
+open System.IO
+open NUnit.Framework
+open FsUnit
+open Fantomas.CoreGlobalTool.Tests.TestHelpers
+open System.Text
+
+[<Literal>]
+let Source = "namespace Company.Product.Feature"
+
+let private getInitialBytes file =
+        use file = new FileStream(file, FileMode.Open, FileAccess.Read)
+        let mutable bom = Array.zeroCreate 3
+        file.Read(bom, 0, 3) |> ignore
+        bom
+
+[<Test>]
+let ``byte-order mark should be preserved, 795``() =
+    use fileFixture = new TemporaryFileCodeSample(Source, true)
+    let exitCode = runFantomasTool fileFixture.Filename
+    exitCode |> should equal 0
+
+    let expectedPreamble = Encoding.UTF8.GetPreamble()
+    let actualPreamble = getInitialBytes fileFixture.Filename
+    expectedPreamble |> should equal actualPreamble
+
+[<Test>]
+let ``preserve byte-order from original file`` () =
+    use inputFixture = new TemporaryFileCodeSample(Source, true)
+    use outputFixture = new OutputFile()
+    let exitCode =
+        sprintf "--out %s %s" outputFixture.Filename inputFixture.Filename
+        |> runFantomasTool
+
+    exitCode |> should equal 0
+
+    let expectedPreamble = Encoding.UTF8.GetPreamble()
+    let actualPreamble = getInitialBytes outputFixture.Filename
+    expectedPreamble |> should equal actualPreamble

src/Fantomas.CoreGlobalTool.Tests/Fantomas.CoreGlobalTool.Tests.fsproj

@@ -14,6 +14,7 @@
     <None Include="paket.references" />
     <Compile Include="TestHelpers.fs" />
     <Compile Include="CheckTests.fs" />
+    <Compile Include="ByteOrderMarkTests.fs" />
   </ItemGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />
 </Project>

src/Fantomas.CoreGlobalTool.Tests/TestHelpers.fs

@@ -3,16 +3,28 @@ module Fantomas.CoreGlobalTool.Tests.TestHelpers
 open System
 open System.Diagnostics
 open System.IO
+open System.Text
 
-type TemporaryFileCodeSample internal (codeSnippet: string) =
+type TemporaryFileCodeSample internal (codeSnippet: string, ?hasByteOrderMark: bool) =
+    let hasByteOrderMark = defaultArg hasByteOrderMark false
     let filename = Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString() + ".fs")
-    do File.WriteAllText(filename, codeSnippet)
+    do (if hasByteOrderMark
+        then File.WriteAllText(filename, codeSnippet, Encoding.UTF8)
+        else File.WriteAllText(filename, codeSnippet))
 
     member _.Filename: string = filename
     interface IDisposable with
         member this.Dispose(): unit = File.Delete(filename)
 
-let private runFantomasTool arguments =
+type OutputFile internal () =
+    let filename = Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString() + ".fs")
+    member _.Filename: string = filename
+    interface IDisposable with
+        member this.Dispose(): unit =
+            if File.Exists(filename) then
+                File.Delete(filename)
+
+let runFantomasTool arguments =
     let pwd = Path.GetDirectoryName(typeof<TemporaryFileCodeSample>.Assembly.Location)
     let configuration =
         #if DEBUG
@@ -36,4 +48,4 @@ let private runFantomasTool arguments =
 
 let checkCode file =
     let arguments = sprintf "--check \"%s\"" file
-    runFantomasTool arguments
+    runFantomasTool arguments

src/Fantomas.CoreGlobalTool/Program.fs

@@ -3,6 +3,7 @@ open System.IO
 open Fantomas
 open Fantomas.FormatConfig
 open Argu
+open System.Text
 
 let extensions = set [| ".fs"; ".fsx"; ".fsi"; ".ml"; ".mli"; |]
 
@@ -69,13 +70,29 @@ let rec allFiles isRec path =
     Directory.GetFiles(path, "*.*", searchOption)
     |> Seq.filter (fun f -> isFSharpFile f && not (isInExcludedDir f))
 
+/// Fantomas assumes the input files are UTF-8
+/// As is stated in F# language spec: https://fsharp.org/specs/language-spec/4.1/FSharpSpec-4.1-latest.pdf#page=25
+let private hasByteOrderMark file =
+    if File.Exists(file) then
+        let preamble = Encoding.UTF8.GetPreamble()
+        use file = new FileStream(file, FileMode.Open, FileAccess.Read)
+        let mutable bom = Array.zeroCreate 3
+        file.Read(bom, 0, 3) |> ignore
+        bom = preamble
+    else
+        false
+
 /// Format a source string using given config and write to a text writer
 let processSourceString isFsiFile s (tw : Choice<TextWriter, string>) config =
     let fileName = if isFsiFile then "/tmp.fsi" else "/tmp.fsx"
     let writeResult (formatted: string) =
         match tw with
         | Choice1Of2 tw -> tw.Write(formatted)
-        | Choice2Of2 path -> File.WriteAllText(path, formatted)
+        | Choice2Of2 path ->
+            if hasByteOrderMark path then
+                File.WriteAllText(path, formatted, Encoding.UTF8)
+            else
+                File.WriteAllText(path, formatted)
 
     async {
         let! formatted = s |> FakeHelpers.formatContentAsync config fileName
@@ -230,7 +247,15 @@ let main argv =
         let fileToFile (inFile : string) (outFile : string) config =
             try
                 printfn "Processing %s" inFile
-                use buffer = new StreamWriter(outFile)
+                let hasByteOrderMark = hasByteOrderMark inFile
+
+                use buffer =
+                    if hasByteOrderMark then
+                        new StreamWriter(new FileStream(outFile, FileMode.OpenOrCreate, FileAccess.ReadWrite)
+                                         ,Encoding.UTF8)
+                    else
+                        new StreamWriter(outFile)
+
                 if profile then
                     File.ReadLines(inFile) |> Seq.length |> printfn "Line count: %i"
                     time (fun () -> processSourceFile inFile buffer config)