diff --git a/.gitattributes b/.gitattributes index fc41139c4..1b382a5b0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,5 @@ # Auto detect text files and perform LF normalization -* text=auto +* text=auto eol=lf # Custom for Visual Studio *.cs text diff=csharp @@ -22,6 +22,8 @@ *.RTF diff=astextplain *.sh text eol=lf +*.cmd text eol=crlf +*.bat text eol=crlf *.png binary *.exe binary diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 29e551280..62a9917a7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ Type providers consist of two components: (that are mapped to runtime components by the compiler). We need a _runtime component_ for .NET Standard 2.0 (netstandard2.0). We also need a _design time_ -component for each, to be able to to host the type provider in .NET Core-based tooling. +component for each, to be able to host the type provider in .NET Core-based tooling. The _runtime_ components are in the following project: @@ -68,7 +68,7 @@ of files, typically like this: the common API in `StructureInference.fs`. * `JsonGenerator.fs` - implements code that generates provided types, adds properties - and methods etc. This uses the information infered by inference and it generates + and methods etc. This uses the information inferred by inference and it generates calls to the runtime components. * `JsonProvider.fs` - entry point that defines static properties of the type provider, @@ -79,7 +79,11 @@ between _runtime_ and _design-time_ components, so you'll find at least two file ### Debugging -To debug the type generation, the best way is to change `FSharp.Data.DesignTime` project to a Console application, rename `Test.fsx` to `Test.fs` and hit the Run command in the IDE, setting the breakpoints where you need them. This will invoke all the type providers manually without locking the files in Visual Studio / Xamarin Studio. You'll also see in the console output the complete dump of the generated types and expressions. This is also the process used for the signature tests. +To debug the type generation, the best way is to change `FSharp.Data.DesignTime` project to a Console application, +rename `Test.fsx` to `Test.fs` and hit the Run command in the IDE, setting the breakpoints where you need them. +This will invoke all the type providers manually without locking the files in Visual Studio / Xamarin Studio. +You'll also see in the console output the complete dump of the generated types and expressions. +This is also the process used for the signature tests. ## Documentation diff --git a/src/CommonProviderImplementation/Helpers.fs b/src/CommonProviderImplementation/Helpers.fs index 74cdc49d6..72a61745a 100644 --- a/src/CommonProviderImplementation/Helpers.fs +++ b/src/CommonProviderImplementation/Helpers.fs @@ -338,7 +338,7 @@ module internal ProviderHelpers = let private providedTypesCache = createInMemoryCache (TimeSpan.FromSeconds 30.0) let private activeDisposeActions = HashSet<_>() - // Cache generated types for a short time, since VS invokes the TP multiple tiems + // Cache generated types for a short time, since VS invokes the TP multiple times // Also cache temporarily during partial invalidation since the invalidation of one TP always causes invalidation of all TPs let internal getOrCreateProvidedType (cfg: TypeProviderConfig) diff --git a/src/CommonRuntime/StructuralInference.fs b/src/CommonRuntime/StructuralInference.fs index 0891afe89..a75300c3e 100644 --- a/src/CommonRuntime/StructuralInference.fs +++ b/src/CommonRuntime/StructuralInference.fs @@ -190,7 +190,7 @@ let rec subtypeInfered allowEmptyValues ot1 ot2 = | InferedType.Heterogeneous h, other | other, InferedType.Heterogeneous h -> // Add the other type as another option. We should never add - // heterogenous type as an option of other heterogeneous type. + // heterogeneous type as an option of other heterogeneous type. assert (typeTag other <> InferedTypeTag.Heterogeneous) InferedType.Heterogeneous(unionHeterogeneousTypes allowEmptyValues h (Map.ofSeq [ typeTag other, other ])) diff --git a/src/CommonRuntime/StructuralTypes.fs b/src/CommonRuntime/StructuralTypes.fs index 733560bf1..7263c2976 100644 --- a/src/CommonRuntime/StructuralTypes.fs +++ b/src/CommonRuntime/StructuralTypes.fs @@ -34,6 +34,7 @@ type InferedTypeTag = | Number | Boolean | String + /// Allow for support of embedded json in e.g. xml documents | Json | DateTime | TimeSpan diff --git a/src/CommonRuntime/TextConversions.fs b/src/CommonRuntime/TextConversions.fs index f6dede45c..90e896f3a 100644 --- a/src/CommonRuntime/TextConversions.fs +++ b/src/CommonRuntime/TextConversions.fs @@ -1,4 +1,4 @@ -// -------------------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------- // Helper operations for converting converting string values to other types // -------------------------------------------------------------------------------------- @@ -86,8 +86,8 @@ type TextConversions private () = static member val private DefaultRemovableAdornerCharacters = Set.union TextConversions.DefaultNonCurrencyAdorners TextConversions.DefaultCurrencyAdorners - //This removes any adorners that might otherwise casue the inference to infer string. A notable a change is - //Currency Symbols are now treated as an Adorner like a '%' sign thus are now independant + //This removes any adorners that might otherwise cause the inference to infer string. A notable a change is + //Currency Symbols are now treated as an Adorner like a '%' sign thus are now independent //of the culture. Which is probably better since we have lots of scenarios where we want to //consume values prefixed with € or $ but in a different culture. static member private RemoveAdorners(value: string) = @@ -157,7 +157,7 @@ type TextConversions private () = | x -> x static member AsDateTimeOffset cultureInfo (text: string) = - // get TimeSpan presentation from 4-digt integers like 0000 or -0600 + // get TimeSpan presentation from 4-digit integers like 0000 or -0600 let getTimeSpanFromHourMin (hourMin: int) = let hr = (hourMin / 100) |> float |> TimeSpan.FromHours let min = (hourMin % 100) |> float |> TimeSpan.FromMinutes @@ -202,7 +202,7 @@ module internal UnicodeHelper = // used http://en.wikipedia.org/wiki/UTF-16#Code_points_U.2B010000_to_U.2B10FFFF as a guide below let getUnicodeSurrogatePair num = // only code points U+010000 to U+10FFFF supported - // for coversion to UTF16 surrogate pair + // for conversion to UTF16 surrogate pair let codePoint = num - 0x010000u let HIGH_TEN_BIT_MASK = 0xFFC00u // 1111|1111|1100|0000|0000 let LOW_TEN_BIT_MASK = 0x003FFu // 0000|0000|0011|1111|1111 diff --git a/src/Csv/CsvInference.fs b/src/Csv/CsvInference.fs index 055116a42..afc089381 100644 --- a/src/Csv/CsvInference.fs +++ b/src/Csv/CsvInference.fs @@ -111,7 +111,7 @@ let private parseTypeAndUnit unitsOfMeasureProvider str = /// Parse schema specification for column. This can either be a name /// with type or just type: name (typeInfo)|typeInfo. -/// If forSchemaOverride is set to true, only Full or Name is returne +/// If forSchemaOverride is set to true, only Full or Name is returned /// (if we succeed we override the inferred schema, otherwise, we just /// override the header name) let private parseSchemaItem unitsOfMeasureProvider str forSchemaOverride = @@ -424,7 +424,7 @@ type CsvFile with /// Infers the types of the columns of a CSV file /// /// - Number of rows to use for inference. If this is zero, all rows are used - /// - The set of strings recogized as missing values + /// - The set of strings recognized as missing values /// - The culture used for parsing numbers and dates /// - Optional column types, in a comma separated list. Valid types are "int", "int64", "bool", "float", "decimal", "date", "timespan", "guid", "string", "int?", "int64?", "bool?", "float?", "decimal?", "date?", "guid?", "int option", "int64 option", "bool option", "float option", "decimal option", "date option", "guid option" and "string option". You can also specify a unit and the name of the column like this: Name (type<unit>). You can also override only the name. If you don't want to specify all the columns, you can specify by name like this: 'ColumnName=type' /// - Assumes all columns can have missing values diff --git a/src/Csv/CsvProvider.fs b/src/Csv/CsvProvider.fs index 9d3324c3e..6482608df 100644 --- a/src/Csv/CsvProvider.fs +++ b/src/Csv/CsvProvider.fs @@ -73,7 +73,7 @@ type public CsvProvider(cfg: TypeProviderConfig) as this = let value = if sample = "" then - // synthetize sample from the schema + // synthesize sample from the schema use reader = new StringReader(value) let schemaStr = @@ -235,7 +235,7 @@ type public CsvProvider(cfg: TypeProviderConfig) as this = When set to true, the type provider will assume all columns can have missing values, even if in the provided sample all values are present. Defaults to false. When set to true, inference will prefer to use the option type instead of nullable types, double.NaN or "" for missing values. Defaults to false. The quotation mark (for surrounding values containing the delimiter). Defaults to ". - The set of strings recogized as missing values specified as a comma-separated string (e.g., "NA,N/A"). Defaults to """ + The set of strings recognized as missing values specified as a comma-separated string (e.g., "NA,N/A"). Defaults to """ + String.Join(",", TextConversions.DefaultMissingValues) + """. Whether the rows should be caches so they can be iterated multiple times. Defaults to true. Disable for large datasets. diff --git a/src/Html/HtmlProvider.fs b/src/Html/HtmlProvider.fs index a2ea01dfe..6363b83ef 100644 --- a/src/Html/HtmlProvider.fs +++ b/src/Html/HtmlProvider.fs @@ -86,7 +86,7 @@ type public HtmlProvider(cfg: TypeProviderConfig) as this = Location of an HTML sample file or a string containing a sample HTML document. When set to true, inference will prefer to use the option type instead of nullable types, double.NaN or "" for missing values. Defaults to false. Includes tables that are potentially layout tables (with cellpadding=0 and cellspacing=0 attributes) - The set of strings recogized as missing values. Defaults to """ + The set of strings recognized as missing values. Defaults to """ + String.Join(",", TextConversions.DefaultMissingValues) + """. The culture used for parsing numbers and dates. Defaults to the invariant culture. diff --git a/src/Json/JsonInference.fs b/src/Json/JsonInference.fs index 308a798d6..d2d3303c6 100644 --- a/src/Json/JsonInference.fs +++ b/src/Json/JsonInference.fs @@ -9,7 +9,7 @@ open FSharp.Data open FSharp.Data.Runtime open FSharp.Data.Runtime.StructuralTypes -/// Infer type of a JSON value - this is simple function because most of the +/// Infer type of a JSON value - this is a simple function because most of the /// functionality is handled in `StructureInference` (most notably, by /// `inferCollectionType` and various functions to find common subtype), so /// here we just need to infer types of primitive JSON values. @@ -20,7 +20,7 @@ let rec inferType inferTypesFromValues cultureInfo parentName json = let inline isIntegerFloat (v: float) : bool = Math.Round v = v match json with - // Null and primitives without subtyping hiearchies + // Null and primitives without subtyping hierarchies | JsonValue.Null -> InferedType.Null | JsonValue.Boolean _ -> InferedType.Primitive(typeof, None, false) | JsonValue.String s when inferTypesFromValues -> StructuralInference.getInferedTypeFromString cultureInfo s None diff --git a/src/Json/JsonProvider.fs b/src/Json/JsonProvider.fs index 8dc171e7d..acfc8ea78 100644 --- a/src/Json/JsonProvider.fs +++ b/src/Json/JsonProvider.fs @@ -117,7 +117,7 @@ type public JsonProvider(cfg: TypeProviderConfig) as this = (e.g. 'MyCompany.MyAssembly, resource_name.json'). This is useful when exposing types generated by the type provider. If true, turns on additional type inference from values. (e.g. type inference infers string values such as "123" as ints and values constrained to 0 and 1 as booleans.) - If true, json record is considered as a dictionary, if the names of all the its fields are infered (by type inference rules) into the same non-string primitive type.""" + If true, json records are interpreted as dictionaries when the names of all the fields are infered (by type inference rules) into the same non-string primitive type.""" do jsonProvTy.AddXmlDoc helpText do jsonProvTy.DefineStaticParameters(parameters, buildTypes) diff --git a/src/Json/JsonValue.fs b/src/Json/JsonValue.fs index 557381c48..708ab1fb1 100644 --- a/src/Json/JsonValue.fs +++ b/src/Json/JsonValue.fs @@ -164,9 +164,6 @@ type private JsonParser(jsonText: string) = let buf = StringBuilder() // pre-allocate buffers for strings // Helper functions - let skipWhitespace () = - while i < s.Length && Char.IsWhiteSpace s.[i] do - i <- i + 1 let isNumChar c = Char.IsDigit c @@ -191,9 +188,50 @@ type private JsonParser(jsonText: string) = let ensure cond = if not cond then throw () + + let rec skipCommentsAndWhitespace () = + let skipComment () = + // Supported comment syntax: + // - // ...{newLine} + // - /* ... */ + if i < s.Length && s.[i] = '/' then + i <- i + 1 + + if i < s.Length && s.[i] = '/' then + i <- i + 1 + + while i < s.Length && (s.[i] <> '\r' && s.[i] <> '\n') do + i <- i + 1 + else if i < s.Length && s.[i] = '*' then + i <- i + 1 + + while i + 1 < s.Length + && s.[i] <> '*' + && s.[i + 1] <> '/' do + i <- i + 1 + + ensure (i + 1 < s.Length && s.[i] = '*' && s.[i + 1] = '/') + i <- i + 2 + + true + + else + false + + let skipWhitespace () = + let initialI = i + + while i < s.Length && Char.IsWhiteSpace s.[i] do + i <- i + 1 + + initialI <> i // return true if some whitespace was skipped + + if skipWhitespace () || skipComment () then + skipCommentsAndWhitespace () + // Recursive descent parser for JSON that uses global mutable index let rec parseValue cont = - skipWhitespace () + skipCommentsAndWhitespace () ensure (i < s.Length) match s.[i] with @@ -291,16 +329,16 @@ type private JsonParser(jsonText: string) = and parsePair cont = let key = parseString () - skipWhitespace () + skipCommentsAndWhitespace () ensure (i < s.Length && s.[i] = ':') i <- i + 1 - skipWhitespace () + skipCommentsAndWhitespace () parseValue (fun v -> cont (key, v)) and parseObject cont = ensure (i < s.Length && s.[i] = '{') i <- i + 1 - skipWhitespace () + skipCommentsAndWhitespace () let pairs = ResizeArray<_>() let parseObjectEnd () = @@ -312,16 +350,16 @@ type private JsonParser(jsonText: string) = if i < s.Length && s.[i] = '"' then parsePair (fun p -> pairs.Add(p) - skipWhitespace () + skipCommentsAndWhitespace () let rec parsePairItem () = if i < s.Length && s.[i] = ',' then i <- i + 1 - skipWhitespace () + skipCommentsAndWhitespace () parsePair (fun p -> pairs.Add(p) - skipWhitespace () + skipCommentsAndWhitespace () parsePairItem ()) else parseObjectEnd () @@ -333,7 +371,7 @@ type private JsonParser(jsonText: string) = and parseArray cont = ensure (i < s.Length && s.[i] = '[') i <- i + 1 - skipWhitespace () + skipCommentsAndWhitespace () let vals = ResizeArray<_>() let parseArrayEnd () = @@ -345,16 +383,16 @@ type private JsonParser(jsonText: string) = if i < s.Length && s.[i] <> ']' then parseValue (fun v -> vals.Add(v) - skipWhitespace () + skipCommentsAndWhitespace () let rec parseArrayItem () = if i < s.Length && s.[i] = ',' then i <- i + 1 - skipWhitespace () + skipCommentsAndWhitespace () parseValue (fun v -> vals.Add(v) - skipWhitespace () + skipCommentsAndWhitespace () parseArrayItem ()) else parseArrayEnd () @@ -375,7 +413,7 @@ type private JsonParser(jsonText: string) = // Start by parsing the top-level value member x.Parse() = let value = parseValue id - skipWhitespace () + skipCommentsAndWhitespace () if i <> s.Length then throw () value @@ -383,7 +421,7 @@ type private JsonParser(jsonText: string) = seq { while i <> s.Length do yield parseValue id - skipWhitespace () + skipCommentsAndWhitespace () } type JsonValue with diff --git a/src/Xml/XmlInference.fs b/src/Xml/XmlInference.fs index 1d278fa1f..16cbd4964 100644 --- a/src/Xml/XmlInference.fs +++ b/src/Xml/XmlInference.fs @@ -60,7 +60,7 @@ let getInferedTypeFromValue inferTypesFromValues cultureInfo (element: XElement) InferedType.Primitive(typeof, None, false) /// Infers type for the element, unifying nodes of the same name -/// accross the entire document (we first get information based +/// across the entire document (we first get information based /// on just attributes and then use a fixed point) let inferGlobalType inferTypesFromValues cultureInfo allowEmptyValues (elements: XElement[]) = diff --git a/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj b/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj index 47c25d563..cbecb419b 100644 --- a/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj +++ b/tests/FSharp.Data.DesignTime.Tests/FSharp.Data.DesignTime.Tests.fsproj @@ -14,6 +14,7 @@ + diff --git a/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config b/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config index 47a3ae15b..cb4260fed 100644 --- a/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config +++ b/tests/FSharp.Data.DesignTime.Tests/SignatureTestCases.config @@ -1,7 +1,7 @@ Csv,SmallTest.csv,,,true,false,false,,, Csv,MSFT.csv,,,true,false,false,,, Csv,AirQuality.csv,;,,true,false,false,,, -Csv,DnbHistoriskeKurser.csv,,,true,false,false,,fr-FR, +Csv,DnbHistoriskeKurser.csv,,,true,false,false,,nb-NO, Csv,file with spaces.csv,,,true,false,false,,, Csv,LastFM.tsv,,,false,false,false,,, Csv,Titanic.csv,,,true,false,false,,, diff --git a/tests/FSharp.Data.DesignTime.Tests/expected/Csv,DnbHistoriskeKurser.csv,,,True,False,False,,fr-FR,.expected b/tests/FSharp.Data.DesignTime.Tests/expected/Csv,DnbHistoriskeKurser.csv,,,True,False,False,,nb-NO,.expected similarity index 97% rename from tests/FSharp.Data.DesignTime.Tests/expected/Csv,DnbHistoriskeKurser.csv,,,True,False,False,,fr-FR,.expected rename to tests/FSharp.Data.DesignTime.Tests/expected/Csv,DnbHistoriskeKurser.csv,,,True,False,False,,nb-NO,.expected index e721d4b15..0f5081482 100644 --- a/tests/FSharp.Data.DesignTime.Tests/expected/Csv,DnbHistoriskeKurser.csv,,,True,False,False,,fr-FR,.expected +++ b/tests/FSharp.Data.DesignTime.Tests/expected/Csv,DnbHistoriskeKurser.csv,,,True,False,False,,nb-NO,.expected @@ -1,7 +1,7 @@ class CsvProvider : FDR.CsvFile new : rows:CsvProvider+Row seq -> CsvProvider let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -27,7 +27,7 @@ class CsvProvider : FDR.CsvFile new : () -> CsvProvider let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) @@ -49,7 +49,7 @@ class CsvProvider : FDR.CsvFile let value = TextConversions.AsString(row.[10]) TextRuntime.GetNonOptionalValue("AUD", TextRuntime.ConvertString(value), value)) let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -66,7 +66,7 @@ class CsvProvider : FDR.CsvFile let f = new Func<_,_>(fun (t:TextReader) -> let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) @@ -88,7 +88,7 @@ class CsvProvider : FDR.CsvFile let value = TextConversions.AsString(row.[10]) TextRuntime.GetNonOptionalValue("AUD", TextRuntime.ConvertString(value), value)) let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -106,7 +106,7 @@ class CsvProvider : FDR.CsvFile let f = new Func<_,_>(fun (t:TextReader) -> let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) @@ -128,7 +128,7 @@ class CsvProvider : FDR.CsvFile let value = TextConversions.AsString(row.[10]) TextRuntime.GetNonOptionalValue("AUD", TextRuntime.ConvertString(value), value)) let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -145,7 +145,7 @@ class CsvProvider : FDR.CsvFile static member GetSample: () -> CsvProvider let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) @@ -167,7 +167,7 @@ class CsvProvider : FDR.CsvFile let value = TextConversions.AsString(row.[10]) TextRuntime.GetNonOptionalValue("AUD", TextRuntime.ConvertString(value), value)) let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -183,7 +183,7 @@ class CsvProvider : FDR.CsvFile static member Load: stream:System.IO.Stream -> CsvProvider let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) @@ -205,7 +205,7 @@ class CsvProvider : FDR.CsvFile let value = TextConversions.AsString(row.[10]) TextRuntime.GetNonOptionalValue("AUD", TextRuntime.ConvertString(value), value)) let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -221,7 +221,7 @@ class CsvProvider : FDR.CsvFile static member Load: reader:System.IO.TextReader -> CsvProvider let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) @@ -243,7 +243,7 @@ class CsvProvider : FDR.CsvFile let value = TextConversions.AsString(row.[10]) TextRuntime.GetNonOptionalValue("AUD", TextRuntime.ConvertString(value), value)) let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -259,7 +259,7 @@ class CsvProvider : FDR.CsvFile static member Load: uri:string -> CsvProvider let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) @@ -281,7 +281,7 @@ class CsvProvider : FDR.CsvFile let value = TextConversions.AsString(row.[10]) TextRuntime.GetNonOptionalValue("AUD", TextRuntime.ConvertString(value), value)) let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -297,7 +297,7 @@ class CsvProvider : FDR.CsvFile static member Parse: text:string -> CsvProvider let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) @@ -319,7 +319,7 @@ class CsvProvider : FDR.CsvFile let value = TextConversions.AsString(row.[10]) TextRuntime.GetNonOptionalValue("AUD", TextRuntime.ConvertString(value), value)) let rowToStringArray = new Func<_,_>(fun (row:_ * _ * _ * _ * _ * _ * _ * _) -> - [| TextRuntime.ConvertDateTimeBack("fr-FR", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) + [| TextRuntime.ConvertDateTimeBack("nb-NO", Some (let t1,_,_,_,_,_,_,_,_,_,_ = row in t1)) TextRuntime.ConvertStringBack(Some (let _,t2,_,_,_,_,_,_,_,_,_ = row in t2)) TextRuntime.ConvertStringBack(Some (let _,_,t3,_,_,_,_,_,_,_,_ = row in t3)) TextRuntime.ConvertStringBack(Some (let _,_,_,t4,_,_,_,_,_,_,_ = row in t4)) @@ -335,7 +335,7 @@ class CsvProvider : FDR.CsvFile static member ParseRows: text:string -> CsvProvider+CsvProvider+Row[] let stringArrayToRow = new Func<_,_,_>(fun (parent:obj) (row:string[]) -> let value = TextConversions.AsString(row.[0]) - TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("fr-FR", value), value), + TextRuntime.GetNonOptionalValue("Dato", TextRuntime.ConvertDateTime("nb-NO", value), value), let value = TextConversions.AsString(row.[1]) TextRuntime.GetNonOptionalValue("USD", TextRuntime.ConvertString(value), value), let value = TextConversions.AsString(row.[2]) diff --git a/tests/FSharp.Data.Tests/CsvProvider.fs b/tests/FSharp.Data.Tests/CsvProvider.fs index 2f729c07c..16c5c6571 100644 --- a/tests/FSharp.Data.Tests/CsvProvider.fs +++ b/tests/FSharp.Data.Tests/CsvProvider.fs @@ -7,6 +7,7 @@ open System.IO open FSharp.Data.UnitSystems.SI.UnitNames open FSharp.Data open FSharp.Data.Runtime.CsvInference +open System.Globalization let [] simpleCsv = """ Column1,Column2,Column3 @@ -141,9 +142,17 @@ let ``Infers type of an emtpy CSV file`` () = let expected : string array = [||] actual |> should equal expected +[] +let norwayCultureName = "nb-NO" + [] let ``Does not treat invariant culture number such as 3.14 as a date in cultures using 3,14`` () = - let csv = CsvProvider<"Data/DnbHistoriskeKurser.csv", ",", 10, Culture="fr-FR">.GetSample() + let targetCulture = CultureInfo(norwayCultureName) + // Make sure assumptions about the culture hold: + targetCulture.DateTimeFormat.DateSeparator |> should equal "." + targetCulture.DateTimeFormat.TimeSeparator |> should equal ":" // See https://github.com/fsprojects/FSharp.Data/issues/767 + targetCulture.NumberFormat.NumberDecimalSeparator |> should equal "," + let csv = CsvProvider<"Data/DnbHistoriskeKurser.csv", ",", 10, Culture=norwayCultureName>.GetSample() let row = csv.Rows |> Seq.head (row.Dato, row.USD) |> should equal (DateTime(2013, 2, 7), "5.4970") diff --git a/tests/FSharp.Data.Tests/JsonParserProperties.fs b/tests/FSharp.Data.Tests/JsonParserProperties.fs index 41e703f6a..c50b003b6 100644 --- a/tests/FSharp.Data.Tests/JsonParserProperties.fs +++ b/tests/FSharp.Data.Tests/JsonParserProperties.fs @@ -160,3 +160,30 @@ let ``Stringifing parsed string returns the same string (UNICODE)`` () = let jsonConverted = jsonValue.ToString(JsonSaveOptions.DisableFormatting) Assert.AreNotEqual(input, jsonConverted) // this now has the escaped value Assert.AreEqual(unescape input, jsonConverted) + +[] +let ``Single line is skipped`` () = + let input = + """// + { + "test": true // x + } + //""" + let inputWithoutComment = """{"test": true}""" + let jsonValue = JsonValue.Parse input + let jsonValueWithoutComment = JsonValue.Parse inputWithoutComment + Assert.AreEqual(jsonValueWithoutComment, jsonValue) + +let ``Multiline comment is skipped`` () = + let input = + """/**/ + {"test": true} + /**//**/ /**/ // + /* { + "test": true + } + */""" + let inputWithoutComment = """{"test": true}""" + let jsonValue = JsonValue.Parse input + let jsonValueWithoutComment = JsonValue.Parse inputWithoutComment + Assert.AreEqual(jsonValueWithoutComment, jsonValue)