diff --git a/.gitignore b/.gitignore index fd5204b5..88719159 100644 --- a/.gitignore +++ b/.gitignore @@ -98,6 +98,7 @@ _TeamCity* # NCrunch _NCrunch_* .*crunch*.local.xml +*.ncrunch* # MightyMoose *.mm.* diff --git a/assets/CommonAssemblyInfo.cs b/assets/CommonAssemblyInfo.cs index f96e2b68..dfa1dd33 100644 --- a/assets/CommonAssemblyInfo.cs +++ b/assets/CommonAssemblyInfo.cs @@ -1,5 +1,5 @@ using System.Reflection; [assembly: AssemblyVersion("0.0.0")] -[assembly: AssemblyFileVersion("0..0")] +[assembly: AssemblyFileVersion("0.0.0")] [assembly: AssemblyInformationalVersion("0.")] diff --git a/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs b/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs index 4bb4d77b..89671054 100644 --- a/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs +++ b/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs @@ -15,179 +15,44 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Net.Configuration; using Elasticsearch.Net.Connection; using Elasticsearch.Net.Serialization; using Serilog.Configuration; using Serilog.Core; using Serilog.Events; -using Serilog.Sinks.ElasticSearch; +using Serilog.Sinks.Elasticsearch; namespace Serilog { /// - /// Adds the WriteTo.ElasticSearch() extension method to . + /// Adds the WriteTo.Elasticsearch() extension method to . /// - public static class LoggerConfigurationElasticSearchExtensions + public static class LoggerConfigurationElasticsearchExtensions { - /// - /// Adds a sink that writes log events as documents to an ElasticSearch index. - /// This works great with the Kibana web interface when using the default settings. - /// Make sure to add a template to ElasticSearch like the one found here: - /// https://gist.github.com/mivano/9688328 - /// - /// The logger configuration. - /// The index format where the events are send to. It defaults to the logstash index per day format. It uses a String.Format using the DateTime.UtcNow parameter. - /// The URI to the node where ElasticSearch is running. When null, will fall back to http://localhost:9200 - /// The connection time out in milliseconds. Default value is 5000. - /// The minimum log event level required in order to write an event to the sink. - /// The maximum number of events to post in a single batch. - /// The time to wait between checking for event batches. - /// Supplies culture-specific formatting information, or null. - /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation - /// - /// Logger configuration, allowing configuration to continue. - /// - /// loggerConfiguration - /// A required parameter is null. - [EditorBrowsable(EditorBrowsableState.Never)] - [Obsolete("Please use Elasticsearch(ElasticsearchSinkOptions options), this method might not expose all options and should be removed in the next Serilog major release")] - public static LoggerConfiguration ElasticSearch( - this LoggerSinkConfiguration loggerConfiguration, - string indexFormat = ElasticsearchSink.DefaultIndexFormat, - Uri node = null, - int connectionTimeOutInMilliseconds = ElasticsearchSink.DefaultConnectionTimeout, - LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, - int batchPostingLimit = ElasticsearchSink.DefaultBatchPostingLimit, - TimeSpan? period = null, - IFormatProvider formatProvider = null, - IElasticsearchSerializer serializer = null - ) - { - if (node == null) - node = new Uri("http://localhost:9200"); - - return Elasticsearch(loggerConfiguration, new ElasticsearchSinkOptions(new [] { node }) - { - Serializer = serializer, - FormatProvider = formatProvider, - IndexFormat = indexFormat, - ModifyConnectionSetttings = s => s.SetTimeout(connectionTimeOutInMilliseconds), - BatchPostingLimit = batchPostingLimit, - Period = period, - MinimumLogEventLevel = restrictedToMinimumLevel - }); - } - - /// - /// Adds a sink that writes log events as documents to an ElasticSearch index. - /// This works great with the Kibana web interface when using the default settings. - /// Make sure to add a template to ElasticSearch like the one found here: - /// https://gist.github.com/mivano/9688328 - /// - /// The logger configuration. - /// The node URIs of the Elasticsearch cluster. - /// The index format where the events are send to. It defaults to the logstash index per day format. It uses a String.Format using the DateTime.UtcNow parameter. - /// The connection time out in milliseconds. Default value is 5000. - /// The minimum log event level required in order to write an event to the sink. - /// The maximum number of events to post in a single batch. - /// The time to wait between checking for event batches. - /// Supplies culture-specific formatting information, or null. - /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation - /// - /// Logger configuration, allowing configuration to continue. - /// - /// loggerConfiguration - /// A required parameter is null. - [EditorBrowsable(EditorBrowsableState.Never)] - [Obsolete("Please use Elasticsearch(ElasticsearchSinkOptions options), this method might not expose all options and should be removed in the next Serilog major release")] - public static LoggerConfiguration ElasticSearch( - this LoggerSinkConfiguration loggerConfiguration, - IEnumerable nodes, - string indexFormat = ElasticsearchSink.DefaultIndexFormat, - int connectionTimeOutInMilliseconds = ElasticsearchSink.DefaultConnectionTimeout, - LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, - int batchPostingLimit = ElasticsearchSink.DefaultBatchPostingLimit, - TimeSpan? period = null, - IFormatProvider formatProvider = null, - IElasticsearchSerializer serializer = null - ) - { - return Elasticsearch(loggerConfiguration, new ElasticsearchSinkOptions(nodes) - { - Serializer = serializer, - FormatProvider = formatProvider, - IndexFormat = indexFormat, - ModifyConnectionSetttings = s => s.SetTimeout(connectionTimeOutInMilliseconds), - BatchPostingLimit = batchPostingLimit, - Period = period, - MinimumLogEventLevel = restrictedToMinimumLevel - }); - } /// - /// Adds a sink that writes log events as documents to an ElasticSearch index. + /// Adds a sink that writes log events as documents to an Elasticsearch index. /// This works great with the Kibana web interface when using the default settings. - /// Make sure to add a template to ElasticSearch like the one found here: - /// https://gist.github.com/mivano/9688328 - /// - /// The logger configuration. - /// The configuration to use for connecting to the Elasticsearch cluster. - /// The index format where the events are send to. It defaults to the logstash index per day format. It uses a String.Format using the DateTime.UtcNow parameter. - /// The minimum log event level required in order to write an event to the sink. - /// The maximum number of events to post in a single batch. - /// The time to wait between checking for event batches. - /// Supplies culture-specific formatting information, or null. - /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation - /// - /// Logger configuration, allowing configuration to continue. - /// - /// loggerConfiguration - /// A required parameter is null. - [EditorBrowsable(EditorBrowsableState.Never)] - [Obsolete("Please use Elasticsearch(ElasticsearchSinkOptions options), this method might not expose all options and should be removed in the next Serilog major release")] - public static LoggerConfiguration ElasticSearch( - this LoggerSinkConfiguration loggerConfiguration, - ConnectionConfiguration connectionConfiguration, - string indexFormat = ElasticsearchSink.DefaultIndexFormat, - LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, - int batchPostingLimit = ElasticsearchSink.DefaultBatchPostingLimit, - TimeSpan? period = null, - IFormatProvider formatProvider = null, - IElasticsearchSerializer serializer = null - ) - { - if (connectionConfiguration == null) - throw new ArgumentNullException("connectionConfiguration"); - IConnectionConfigurationValues values = connectionConfiguration; - return Elasticsearch(loggerConfiguration, new ElasticsearchSinkOptions(values.ConnectionPool) - { - Serializer = serializer, - FormatProvider = formatProvider, - IndexFormat = indexFormat, - ModifyConnectionSetttings = s => connectionConfiguration, - BatchPostingLimit = batchPostingLimit, - Period = period, - MinimumLogEventLevel = restrictedToMinimumLevel - }); - } - - /// - /// Adds a sink that writes log events as documents to an ElasticSearch index. - /// This works great with the Kibana web interface when using the default settings. - /// Make sure to add a template to ElasticSearch like the one found here: + /// Make sure to add a template to Elasticsearch like the one found here: /// https://gist.github.com/mivano/9688328 /// /// /// Provides options specific to the Elasticsearch sink /// - public static LoggerConfiguration Elasticsearch(this LoggerSinkConfiguration loggerSinkConfiguration, ElasticsearchSinkOptions options = null) + public static LoggerConfiguration Elasticsearch( + this LoggerSinkConfiguration loggerSinkConfiguration, + ElasticsearchSinkOptions options = null) { + //TODO make sure we do not kill appdata injection + //TODO handle bulk errors and write to self log, what does logstash do in this case? + //TODO NEST trace logging ID's to corrolate requests to eachother + //Deal with positional formatting in fields property (default to scalar string in mapping) options = options ?? new ElasticsearchSinkOptions(new [] { new Uri("http://localhost:9200") }); - var sink = string.IsNullOrWhiteSpace(options.BufferBaseFilename) + var sink = string.IsNullOrWhiteSpace(options.BufferBaseFilename) ? (ILogEventSink) new ElasticsearchSink(options) - : new DurableElasticSearchSink(options); - + : new DurableElasticsearchSink(options); return loggerSinkConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum); } } diff --git a/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs.orig b/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs.orig deleted file mode 100644 index 29696b6f..00000000 --- a/src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs.orig +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2014 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using Elasticsearch.Net.Connection; -using Elasticsearch.Net.Serialization; -using Serilog.Configuration; -using Serilog.Core; -using Serilog.Events; -using Serilog.Sinks.ElasticSearch; - -namespace Serilog -{ - /// - /// Adds the WriteTo.ElasticSearch() extension method to . - /// - public static class LoggerConfigurationElasticSearchExtensions - { - /// - /// Adds a sink that writes log events as documents to an ElasticSearch index. - /// This works great with the Kibana web interface when using the default settings. - /// Make sure to add a template to ElasticSearch like the one found here: - /// https://gist.github.com/mivano/9688328 - /// - /// The logger configuration. - /// The index format where the events are send to. It defaults to the logstash index per day format. It uses a String.Format using the DateTime.UtcNow parameter. - /// The URI to the node where ElasticSearch is running. When null, will fall back to http://localhost:9200 - /// The connection time out in milliseconds. Default value is 5000. - /// The minimum log event level required in order to write an event to the sink. - /// The maximum number of events to post in a single batch. - /// The time to wait between checking for event batches. - /// Supplies culture-specific formatting information, or null. - /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation - /// - /// Logger configuration, allowing configuration to continue. - /// - /// loggerConfiguration - /// A required parameter is null. - [EditorBrowsable(EditorBrowsableState.Never)] - [Obsolete("Please use Elasticsearch(ElasticsearchSinkOptions options), this method might not expose all options and should be removed in the next Serilog major release")] - public static LoggerConfiguration ElasticSearch( - this LoggerSinkConfiguration loggerConfiguration, - string indexFormat = ElasticsearchSink.DefaultIndexFormat, - Uri node = null, - int connectionTimeOutInMilliseconds = ElasticsearchSink.DefaultConnectionTimeout, - LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, - int batchPostingLimit = ElasticsearchSink.DefaultBatchPostingLimit, - TimeSpan? period = null, - IFormatProvider formatProvider = null, - IElasticsearchSerializer serializer = null - ) - { - if (node == null) - node = new Uri("http://localhost:9200"); - - return Elasticsearch(loggerConfiguration, new ElasticsearchSinkOptions(new [] { node }) - { - Serializer = serializer, - FormatProvider = formatProvider, - IndexFormat = indexFormat, - ModifyConnectionSetttings = s => s.SetTimeout(connectionTimeOutInMilliseconds), - BatchPostingLimit = batchPostingLimit, - Period = period, - MinimumLogEventLevel = restrictedToMinimumLevel - }); - } - - /// - /// Adds a sink that writes log events as documents to an ElasticSearch index. - /// This works great with the Kibana web interface when using the default settings. - /// Make sure to add a template to ElasticSearch like the one found here: - /// https://gist.github.com/mivano/9688328 - /// - /// The logger configuration. - /// The node URIs of the Elasticsearch cluster. - /// The index format where the events are send to. It defaults to the logstash index per day format. It uses a String.Format using the DateTime.UtcNow parameter. - /// The connection time out in milliseconds. Default value is 5000. - /// The minimum log event level required in order to write an event to the sink. - /// The maximum number of events to post in a single batch. - /// The time to wait between checking for event batches. - /// Supplies culture-specific formatting information, or null. - /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation - /// - /// Logger configuration, allowing configuration to continue. - /// - /// loggerConfiguration - /// A required parameter is null. - [EditorBrowsable(EditorBrowsableState.Never)] - [Obsolete("Please use Elasticsearch(ElasticsearchSinkOptions options), this method might not expose all options and should be removed in the next Serilog major release")] - public static LoggerConfiguration ElasticSearch( - this LoggerSinkConfiguration loggerConfiguration, - IEnumerable nodes, - string indexFormat = ElasticsearchSink.DefaultIndexFormat, - int connectionTimeOutInMilliseconds = ElasticsearchSink.DefaultConnectionTimeout, - LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, - int batchPostingLimit = ElasticsearchSink.DefaultBatchPostingLimit, - TimeSpan? period = null, - IFormatProvider formatProvider = null, - IElasticsearchSerializer serializer = null - ) - { - return Elasticsearch(loggerConfiguration, new ElasticsearchSinkOptions(nodes) - { - Serializer = serializer, - FormatProvider = formatProvider, - IndexFormat = indexFormat, - ModifyConnectionSetttings = s => s.SetTimeout(connectionTimeOutInMilliseconds), - BatchPostingLimit = batchPostingLimit, - Period = period, - MinimumLogEventLevel = restrictedToMinimumLevel - }); - } - - /// - /// Adds a sink that writes log events as documents to an ElasticSearch index. - /// This works great with the Kibana web interface when using the default settings. - /// Make sure to add a template to ElasticSearch like the one found here: - /// https://gist.github.com/mivano/9688328 - /// - /// The logger configuration. - /// The configuration to use for connecting to the Elasticsearch cluster. - /// The index format where the events are send to. It defaults to the logstash index per day format. It uses a String.Format using the DateTime.UtcNow parameter. - /// The minimum log event level required in order to write an event to the sink. - /// The maximum number of events to post in a single batch. - /// The time to wait between checking for event batches. - /// Supplies culture-specific formatting information, or null. - /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation - /// - /// Logger configuration, allowing configuration to continue. - /// - /// loggerConfiguration - /// A required parameter is null. - [EditorBrowsable(EditorBrowsableState.Never)] - [Obsolete("Please use Elasticsearch(ElasticsearchSinkOptions options), this method might not expose all options and should be removed in the next Serilog major release")] - public static LoggerConfiguration ElasticSearch( - this LoggerSinkConfiguration loggerConfiguration, - ConnectionConfiguration connectionConfiguration, - string indexFormat = ElasticsearchSink.DefaultIndexFormat, - LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, - int batchPostingLimit = ElasticsearchSink.DefaultBatchPostingLimit, - TimeSpan? period = null, - IFormatProvider formatProvider = null, - IElasticsearchSerializer serializer = null - ) - { - if (connectionConfiguration == null) - throw new ArgumentNullException("connectionConfiguration"); - IConnectionConfigurationValues values = connectionConfiguration; - return Elasticsearch(loggerConfiguration, new ElasticsearchSinkOptions(values.ConnectionPool) - { - Serializer = serializer, - FormatProvider = formatProvider, - IndexFormat = indexFormat, - ModifyConnectionSetttings = s => connectionConfiguration, - BatchPostingLimit = batchPostingLimit, - Period = period, - MinimumLogEventLevel = restrictedToMinimumLevel - }); - } - - /// - /// Adds a sink that writes log events as documents to an ElasticSearch index. - /// This works great with the Kibana web interface when using the default settings. - /// Make sure to add a template to ElasticSearch like the one found here: - /// https://gist.github.com/mivano/9688328 - /// - /// - /// Provides options specific to the Elasticsearch sink - /// - public static LoggerConfiguration Elasticsearch(this LoggerSinkConfiguration loggerSinkConfiguration, ElasticsearchSinkOptions options = null) - { -<<<<<<< HEAD - options = options ?? new ElasticsearchSinkOptions(new [] { new Uri("http://localhost:9200") }); - var sink = new ElasticsearchSink(options); -======= - options = options ?? new ElasticsearchSinkOptions(new [] { new Uri("http://locahost:9200") }); - - var sink = string.IsNullOrWhiteSpace(options.BufferBaseFilename) - ? (ILogEventSink) new ElasticsearchSink(options) - : new DurableElasticSearchSink(options); - ->>>>>>> 35384861bb26488009b884f4a919d50a2970d5b9 - return loggerSinkConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum); - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs b/src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs index 467866ce..932d9e5a 100644 --- a/src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs +++ b/src/Serilog.Sinks.Elasticsearch/Properties/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; using System.Runtime.CompilerServices; -[assembly: AssemblyTitle("Serilog.Sinks.ElasticSearch")] -[assembly: AssemblyDescription("Serilog sink for ElasticSearch")] +[assembly: AssemblyTitle("Serilog.Sinks.Elasticsearch")] +[assembly: AssemblyDescription("Serilog sink for Elasticsearch")] [assembly: AssemblyCopyright("Copyright © Serilog Contributors 2014")] [assembly: InternalsVisibleTo("Serilog.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100fb8d13fd344a1c" + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.csproj.orig b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.csproj.orig deleted file mode 100644 index 2f765fcc..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.csproj.orig +++ /dev/null @@ -1,113 +0,0 @@ - - - - - Debug - AnyCPU - {E12881F7-B522-4E42-BCCC-4A81F42F8D8B} - Library - Properties - Serilog - Serilog.Sinks.ElasticSearch - v4.5 - 512 - ..\..\ - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - true - bin\Debug\Serilog.Sinks.ElasticSearch.xml - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - true - bin\Release\Serilog.Sinks.ElasticSearch.xml - - - - - - true - - - ..\..\assets\Serilog.snk - - - - False - ..\..\packages\Elasticsearch.Net.1.1.2\lib\Elasticsearch.Net.dll - - - - - - - - - - - - - Properties\CommonAssemblyInfo.cs - -<<<<<<< HEAD - -======= - - ->>>>>>> 35384861bb26488009b884f4a919d50a2970d5b9 - - - - - Serilog.snk - - - - - - - - False - Microsoft .NET Framework 4.5 %28x86 and x64%29 - true - - - False - .NET Framework 3.5 SP1 Client Profile - false - - - False - .NET Framework 3.5 SP1 - false - - - - - {7a9e1095-167d-402a-b43d-b36b97ff183d} - Serilog.FullNetFx - - - {0915dbd9-0f7c-4439-8d9e-74c3d579b219} - Serilog - - - - - copy "$(TargetDir)\*.dll" "C:\Projects\Felix\lib\serilog.sinks.elasticsearch" - - \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec index 307d4df6..cde4b175 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.ElasticSearch.nuspec @@ -1,14 +1,14 @@  - Serilog.Sinks.ElasticSearch + Serilog.Sinks.Elasticsearch $version$ Michiel van Oudheusden - The perfect way for .NET apps to write structured log events to ElasticSearch. + The perfect way for .NET apps to write structured log events to Elasticsearch. en-US http://serilog.net http://www.apache.org/licenses/LICENSE-2.0 - http://serilog.net/images/serilog-sink-nuget.png + http://serilog.net/images/serilog-nuget.png serilog logging elasticsearch @@ -16,7 +16,7 @@ - - + + diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch-net40.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch-net40.csproj index a1703df1..f8c17b6e 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch-net40.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch-net40.csproj @@ -49,14 +49,6 @@ ..\..\packages\Elasticsearch.Net.1.1.2\lib\Elasticsearch.Net.dll - - False - ..\..\packages\Serilog.1.4.196\lib\net40\Serilog.dll - - - False - ..\..\packages\Serilog.1.4.196\lib\net40\Serilog.FullNetFx.dll - @@ -68,9 +60,7 @@ Properties\CommonAssemblyInfo.cs - - @@ -99,5 +89,15 @@ false + + + {7a9e1095-167d-402a-b43d-b36b97ff183d} + Serilog.FullNetFx-net40 + + + {0915dbd9-0f7c-4439-8d9e-74c3d579b219} + Serilog-net40 + + \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj index 9a3d9199..8b94b874 100644 --- a/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj +++ b/src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj @@ -8,7 +8,7 @@ Library Properties Serilog - Serilog.Sinks.ElasticSearch + Serilog.Sinks.Elasticsearch v4.5 512 ..\..\ @@ -23,7 +23,7 @@ prompt 4 true - bin\Debug\Serilog.Sinks.ElasticSearch.xml + bin\Debug\Serilog.Sinks.Elasticsearch.xml AnyCPU @@ -34,7 +34,7 @@ prompt 4 true - bin\Release\Serilog.Sinks.ElasticSearch.xml + bin\Release\Serilog.Sinks.Elasticsearch.xml @@ -50,13 +50,11 @@ False ..\..\packages\Elasticsearch.Net.1.1.2\lib\Elasticsearch.Net.dll - - False + ..\..\packages\Serilog.1.4.196\lib\net45\Serilog.dll ..\..\packages\Serilog.1.4.196\lib\net45\Serilog.FullNetFx.dll - True @@ -66,14 +64,15 @@ - - + + Properties\CommonAssemblyInfo.cs - - - + + + + @@ -81,7 +80,7 @@ - + @@ -101,8 +100,4 @@ - - - - \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DurableElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DurableElasticSearchSink.cs index 10a1a670..07b1cd2d 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DurableElasticSearchSink.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/DurableElasticSearchSink.cs @@ -17,40 +17,33 @@ using Serilog.Events; using Serilog.Sinks.RollingFile; -namespace Serilog.Sinks.ElasticSearch +namespace Serilog.Sinks.Elasticsearch { - class DurableElasticSearchSink : ILogEventSink, IDisposable + class DurableElasticsearchSink : ILogEventSink, IDisposable { // we rely on the date in the filename later! const string FileNameSuffix = "-{Date}.json"; readonly RollingFileSink _sink; - readonly ElasticSearchLogShipper _shipper; + readonly ElasticsearchLogShipper _shipper; + private readonly ElasticsearchSinkState _state; - public DurableElasticSearchSink(ElasticsearchSinkOptions options) + public DurableElasticsearchSink(ElasticsearchSinkOptions options) { - if (options == null) throw new ArgumentNullException("options"); + _state = ElasticsearchSinkState.Create(options); if (string.IsNullOrWhiteSpace(options.BufferBaseFilename)) { throw new ArgumentException("Cannot create the durable ElasticSearch sink without a buffer base file name!"); } - var formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter( - formatProvider: options.FormatProvider, - renderMessage: true, - closingDelimiter: Environment.NewLine, - serializer: options.Serializer, - inlineFields: options.InlineFields - ); - _sink = new RollingFileSink( options.BufferBaseFilename + FileNameSuffix, - formatter, + _state.Formatter, options.BufferFileSizeLimitBytes, null); - _shipper = new ElasticSearchLogShipper(options); + _shipper = new ElasticsearchLogShipper(_state); } public void Emit(LogEvent logEvent) diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchLogShipper.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchLogShipper.cs index 6fa364b6..76f20cd7 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchLogShipper.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchLogShipper.cs @@ -25,11 +25,11 @@ using Elasticsearch.Net.Serialization; using Serilog.Debugging; -namespace Serilog.Sinks.ElasticSearch +namespace Serilog.Sinks.Elasticsearch { - class ElasticSearchLogShipper : IDisposable + class ElasticsearchLogShipper : IDisposable { - readonly ElasticsearchClient _client; + private readonly ElasticsearchSinkState _state; readonly int _batchPostingLimit; readonly Timer _timer; @@ -39,27 +39,15 @@ class ElasticSearchLogShipper : IDisposable readonly string _bookmarkFilename; readonly string _logFolder; readonly string _candidateSearchPath; - readonly string _typeName; - readonly string _indexFormat; - public ElasticSearchLogShipper(ElasticsearchSinkOptions options) + internal ElasticsearchLogShipper(ElasticsearchSinkState state) { - var configuration = new ConnectionConfiguration(options.ConnectionPool) - .SetTimeout(ElasticsearchSink.DefaultConnectionTimeout) - .SetMaximumAsyncConnections(20); - - _period = options.BufferLogShippingInterval ?? TimeSpan.FromSeconds(5); - - _indexFormat = !string.IsNullOrWhiteSpace(options.IndexFormat) ? options.IndexFormat : ElasticsearchSink.DefaultIndexFormat; - _typeName = !string.IsNullOrWhiteSpace(options.TypeName) ? options.TypeName : ElasticsearchSink.DefaultTypeName; - - _client = new ElasticsearchClient(configuration, connection: options.Connection, serializer: options.Serializer); - - _batchPostingLimit = options.BatchPostingLimit ?? 100; - - _bookmarkFilename = Path.GetFullPath(options.BufferBaseFilename + ".bookmark"); + _state = state; + _period = _state.Options.BufferLogShippingInterval ?? TimeSpan.FromSeconds(5); + _batchPostingLimit = _state.Options.BatchPostingLimit; + _bookmarkFilename = Path.GetFullPath(_state.Options.BufferBaseFilename + ".bookmark"); _logFolder = Path.GetDirectoryName(_bookmarkFilename); - _candidateSearchPath = Path.GetFileName(options.BufferBaseFilename) + "*.json"; + _candidateSearchPath = Path.GetFileName(_state.Options.BufferBaseFilename) + "*.json"; _timer = new Timer(s => OnTick()); @@ -173,11 +161,10 @@ void OnTick() string nextLine; while (count < _batchPostingLimit && TryReadLine(current, ref nextLineBeginsAtOffset, out nextLine)) { - var indexName = string.Format(_indexFormat, date); - var action = new { index = new { _index = indexName, _type = _typeName } }; - var actionJson = _client.Serializer.Serialize(action, SerializationFormatting.None); - - payload.Add(Encoding.UTF8.GetString(actionJson)); + var indexName = string.Format(_state.Options.IndexFormat, date); + var action = new { index = new { _index = indexName, _type = _state.Options.TypeName } }; + var actionJson = _state.Serialize(action); + payload.Add(actionJson); payload.Add(nextLine); ++count; } @@ -185,7 +172,7 @@ void OnTick() if (count > 0) { - var response = _client.Bulk(payload); + var response = _state.Client.Bulk(payload); if (response.Success) { diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs index 6d76bcfa..bd8c2922 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs @@ -22,74 +22,27 @@ using Serilog.Events; using Serilog.Sinks.PeriodicBatching; using System.Text; +using System.Text.RegularExpressions; using Serilog.Formatting; -namespace Serilog.Sinks.ElasticSearch +namespace Serilog.Sinks.Elasticsearch { /// /// Writes log events as documents to ElasticSearch. /// public class ElasticsearchSink : PeriodicBatchingSink { - readonly ITextFormatter _formatter; - readonly string _typeName; - readonly ElasticsearchClient _client; - readonly Func _indexDecider; - /// - /// A reasonable default for the number of events posted in each batch. - /// - public const int DefaultBatchPostingLimit = 50; - - /// - /// A reasonable default time to wait between checking for event batches. - /// - public static readonly TimeSpan DefaultPeriod = TimeSpan.FromSeconds(2); - - /// - /// Default to the Logstash index name format - /// - public const string DefaultIndexFormat = "logstash-{0:yyyy.MM.dd}"; - - /// - /// Defaults to the type of logevent - /// - public const string DefaultTypeName = "logevent"; - - /// - /// Default connection timeout in milliseconds - /// - public const int DefaultConnectionTimeout = 5000; + private readonly ElasticsearchSinkState _state; /// /// Creates a new ElasticsearchSink instance with the provided options /// - /// Options configuring how the sink behaves + /// Options configuring how the sink behaves, may NOT be null public ElasticsearchSink(ElasticsearchSinkOptions options) - : base(options.BatchPostingLimit ?? DefaultBatchPostingLimit, options.Period ?? DefaultPeriod) + : base(options.BatchPostingLimit, options.Period) { - _indexDecider = options.IndexDecider ?? DefaultIndexDecider(options.IndexFormat); - _typeName = !string.IsNullOrWhiteSpace(options.TypeName) ? options.TypeName : DefaultTypeName; - var configuration = new ConnectionConfiguration(options.ConnectionPool) - .SetTimeout(DefaultConnectionTimeout) - .SetMaximumAsyncConnections(20); - if (options.ModifyConnectionSetttings != null) - configuration = options.ModifyConnectionSetttings(configuration); - _client = new ElasticsearchClient(configuration, connection: options.Connection, serializer: options.Serializer); - - _formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter( - formatProvider: options.FormatProvider, - renderMessage: true, - closingDelimiter: string.Empty, - serializer: options.Serializer, - inlineFields: options.InlineFields - ); - } - - Func DefaultIndexDecider(string indexFormat) - { - var closedIndexFormat = !string.IsNullOrWhiteSpace(indexFormat) ? indexFormat : DefaultIndexFormat; - return (@event, offset) => string.Format(closedIndexFormat, offset); + _state = ElasticsearchSinkState.Create(options); } /// @@ -104,23 +57,21 @@ Func DefaultIndexDecider(string indexFormat) protected override void EmitBatch(IEnumerable events) { // ReSharper disable PossibleMultipleEnumeration - if (!events.Any()) + if (events == null || !events.Any()) return; var payload = new List(); - foreach (var e in events) { - var indexName = _indexDecider(e, e.Timestamp.ToUniversalTime()); - var action = new { index = new { _index = indexName, _type = _typeName } }; - var actionJson = _client.Serializer.Serialize(action, SerializationFormatting.None); - payload.Add(Encoding.UTF8.GetString(actionJson)); + var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime()); + var action = new { index = new { _index = indexName, _type = _state.Options.TypeName } }; + var actionJson = _state.Serialize(action); + payload.Add(actionJson); var sw = new StringWriter(); - _formatter.Format(e, sw); + _state.Formatter.Format(e, sw); payload.Add(sw.ToString()); } - - _client.Bulk(payload); + _state.Client.Bulk(payload); } } } diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs.orig b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs.orig deleted file mode 100644 index c529b658..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticSearchSink.cs.orig +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2014 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Elasticsearch.Net; -using Elasticsearch.Net.Connection; -using Elasticsearch.Net.Serialization; -using Serilog.Events; -using Serilog.Sinks.PeriodicBatching; -using System.Text; -using Serilog.Formatting; - -namespace Serilog.Sinks.ElasticSearch -{ - /// - /// Writes log events as documents to ElasticSearch. - /// - public class ElasticsearchSink : PeriodicBatchingSink - { -<<<<<<< HEAD - readonly ElasticsearchJsonFormatter _formatter; -======= - readonly ITextFormatter _formatter; - readonly string _indexFormat; ->>>>>>> 35384861bb26488009b884f4a919d50a2970d5b9 - readonly string _typeName; - readonly ElasticsearchClient _client; - readonly Func _indexDecider; - - /// - /// A reasonable default for the number of events posted in each batch. - /// - public const int DefaultBatchPostingLimit = 50; - - /// - /// A reasonable default time to wait between checking for event batches. - /// - public static readonly TimeSpan DefaultPeriod = TimeSpan.FromSeconds(2); - - /// - /// Default to the Logstash index name format - /// - public const string DefaultIndexFormat = "logstash-{0:yyyy.MM.dd}"; - - /// - /// Defaults to the type of logevent - /// - public const string DefaultTypeName = "logevent"; - - /// - /// Default connection timeout in milliseconds - /// - public const int DefaultConnectionTimeout = 5000; - - /// - /// Creates a new ElasticsearchSink instance with the provided options - /// - /// Options configuring how the sink behaves - public ElasticsearchSink(ElasticsearchSinkOptions options) - : base(options.BatchPostingLimit ?? DefaultBatchPostingLimit, options.Period ?? DefaultPeriod) - { - _indexDecider = options.IndexDecider ?? DefaultIndexDecider(options.IndexFormat); - _typeName = !string.IsNullOrWhiteSpace(options.TypeName) ? options.TypeName : DefaultTypeName; - var configuration = new ConnectionConfiguration(options.ConnectionPool) - .SetTimeout(DefaultConnectionTimeout) - .SetMaximumAsyncConnections(20); - if (options.ModifyConnectionSetttings != null) - configuration = options.ModifyConnectionSetttings(configuration); - _client = new ElasticsearchClient(configuration, connection: options.Connection, serializer: options.Serializer); - - _formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter( - formatProvider: options.FormatProvider, - renderMessage: true, - closingDelimiter: string.Empty, - serializer: options.Serializer, - inlineFields: options.InlineFields - ); - } - - Func DefaultIndexDecider(string indexFormat) - { - var closedIndexFormat = !string.IsNullOrWhiteSpace(indexFormat) ? indexFormat : DefaultIndexFormat; - return (@event, offset) => string.Format(closedIndexFormat, offset); - } - - /// - /// Emit a batch of log events, running to completion synchronously. - /// - /// The events to emit. - /// - /// Override either - /// or , - /// not both. - /// - protected override void EmitBatch(IEnumerable events) - { - // ReSharper disable PossibleMultipleEnumeration - if (!events.Any()) - return; - - var payload = new List(); - - foreach (var e in events) - { - var indexName = _indexDecider(e, e.Timestamp.ToUniversalTime()); - var action = new { index = new { _index = indexName, _type = _typeName } }; - var actionJson = _client.Serializer.Serialize(action, SerializationFormatting.None); - payload.Add(Encoding.UTF8.GetString(actionJson)); - var sw = new StringWriter(); - _formatter.Format(e, sw); - payload.Add(sw.ToString()); - } - - _client.Bulk(payload); - } - } -} diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchJsonFormatter.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchJsonFormatter.cs index fc0a869e..b96c6b93 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchJsonFormatter.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchJsonFormatter.cs @@ -1,31 +1,24 @@ -// Copyright 2014 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; +using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Contracts; +using System.Globalization; using System.IO; using System.Linq; +using System.Reflection; +using System.Runtime.Serialization; using System.Text; using Elasticsearch.Net.Serialization; using Serilog.Events; using Serilog.Formatting.Json; using Serilog.Parsing; -namespace Serilog.Sinks.ElasticSearch +namespace Serilog.Sinks.Elasticsearch { /// /// Custom Json formatter that respects the configured property name handling and forces 'Timestamp' to @timestamp + /// h + /// /// public class ElasticsearchJsonFormatter : JsonFormatter { @@ -67,7 +60,7 @@ protected override void WriteRenderings(IGrouping[] token WriteRenderingsValues(tokensWithFormat, properties, output); output.Write("}"); } - + /// /// Writes out the attached properties /// @@ -75,9 +68,9 @@ protected override void WriteProperties(IReadOnlyDictionary protected override void WriteException(Exception exception, ref string delim, TextWriter output) { - WriteJsonProperty("exception", exception, ref delim, output); + output.Write(delim); + output.Write("\""); + output.Write("exceptions"); + output.Write("\":["); + + delim = ""; + this.WriteExceptionSerializationInfo(exception, ref delim, output, depth: 0); + output.Write("]"); + } + + private void WriteExceptionSerializationInfo(Exception exception, ref string delim, TextWriter output, int depth) + { + + var si = new SerializationInfo(exception.GetType(), new FormatterConverter()); + var sc = new StreamingContext(); + exception.GetObjectData(si, sc); + + var helpUrl = si.GetString("HelpURL"); + var stackTrace = si.GetString("StackTraceString"); + var remoteStackTrace = si.GetString("RemoteStackTraceString"); + var remoteStackIndex = si.GetInt32("RemoteStackIndex"); + var exceptionMethod = si.GetString("ExceptionMethod"); + var hresult = si.GetInt32("HResult"); + var source = si.GetString("Source"); + var className = si.GetString("ClassName"); + var watsonBuckets = si.GetValue("WatsonBuckets", typeof(byte[])) as byte[]; + + //TODO Loop over ISerializable data + + output.Write(delim); + output.Write("{"); + delim = ""; + this.WriteJsonProperty("Depth", depth, ref delim, output); + this.WriteJsonProperty("ClassName", className, ref delim, output); + this.WriteJsonProperty("Message", exception.Message, ref delim, output); + this.WriteJsonProperty("Source", source, ref delim, output); + this.WriteJsonProperty("StackTraceString", stackTrace, ref delim, output); + this.WriteJsonProperty("RemoteStackTraceString", remoteStackTrace, ref delim, output); + this.WriteJsonProperty("RemoteStackIndex", remoteStackIndex, ref delim, output); + this.WriteStructuredExceptionMethod(exceptionMethod, ref delim, output); + this.WriteJsonProperty("HResult", hresult, ref delim, output); + this.WriteJsonProperty("HelpURL", helpUrl, ref delim, output); + + //writing byte[] will fall back to serializer and they differ in output + //JsonNET assumes string, simplejson writes array of numerics. + //Skip for now + //this.WriteJsonProperty("WatsonBuckets", watsonBuckets, ref delim, output); + + output.Write("}"); + delim = ","; + if (exception.InnerException != null && depth < 20) + this.WriteExceptionSerializationInfo(exception.InnerException, ref delim, output, ++depth); } - + + private void WriteStructuredExceptionMethod(string exceptionMethodString, ref string delim, TextWriter output) + { + if (string.IsNullOrWhiteSpace(exceptionMethodString)) return; + + var args = exceptionMethodString.Split('\0', '\n'); + + if (args.Length!=5) return; + + var memberType = Int32.Parse(args[0], CultureInfo.InvariantCulture); + var name = args[1]; + var assemblyName = args[2]; + var className = args[3]; + var signature = args[4]; + var an = new AssemblyName(assemblyName); + output.Write(delim); + output.Write("\""); + output.Write("ExceptionMethod"); + output.Write("\":{"); + delim = ""; + this.WriteJsonProperty("Name", name, ref delim, output); + this.WriteJsonProperty("AssemblyName", an.Name, ref delim, output); + this.WriteJsonProperty("AssemblyVersion", an.Version.ToString(), ref delim, output); + this.WriteJsonProperty("AssemblyCulture", an.CultureName, ref delim, output); + this.WriteJsonProperty("ClassName", className, ref delim, output); + this.WriteJsonProperty("Signature", signature, ref delim, output); + this.WriteJsonProperty("MemberType", memberType, ref delim, output); + + output.Write("}"); + delim = ","; + } + + /// /// (Optionally) writes out the rendered message /// @@ -99,7 +175,7 @@ protected override void WriteRenderedMessage(string message, ref string delim, T { WriteJsonProperty("message", message, ref delim, output); } - + /// /// Writes out the message template for the logevent. /// @@ -107,7 +183,7 @@ protected override void WriteMessageTemplate(string template, ref string delim, { WriteJsonProperty("messageTemplate", template, ref delim, output); } - + /// /// Writes out the log level /// @@ -116,7 +192,7 @@ protected override void WriteLevel(LogEventLevel level, ref string delim, TextWr var stringLevel = Enum.GetName(typeof(LogEventLevel), level); WriteJsonProperty("level", stringLevel, ref delim, output); } - + /// /// Writes out the log timestamp /// diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs index 3b1b9f32..f6294a3c 100644 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs @@ -1,142 +1,164 @@ -// Copyright 2014 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - using System; using System.Collections.Generic; using System.Linq; +using System.Net.Configuration; using Elasticsearch.Net.Connection; using Elasticsearch.Net.ConnectionPool; using Elasticsearch.Net.Serialization; using Serilog.Events; using Serilog.Formatting; -namespace Serilog.Sinks.ElasticSearch +namespace Serilog.Sinks.Elasticsearch { - /// - /// Provides ElasticsearchSink with configurable options - /// - public class ElasticsearchSinkOptions - { - /// - /// Connection configuration to use for connecting to the cluster. - /// - public Func ModifyConnectionSetttings { get; set; } - - /// - /// The index name formatter. A string.Format using the DateTimeOffset of the event is run over this string. - /// defaults to "logstash-{0:yyyy.MM.dd}" - /// - public string IndexFormat { get; set; } - - /// - /// The default elasticsearch type name to use for the log events defaults to: logevent - /// - public string TypeName { get; set; } - - /// - /// The maximum number of events to post in a single batch. - /// - public int? BatchPostingLimit { get; set; } - - /// - /// The time to wait between checking for event batches. - /// - public TimeSpan? Period { get; set; } - - /// - /// Supplies culture-specific formatting information, or null. - /// - public IFormatProvider FormatProvider { get; set; } - - /// - /// Allows you to override the connection used to communicate with elasticsearch - /// - public IConnection Connection { get; set; } - - /// - /// When true fields will be written at the root of the json document - /// - public bool InlineFields { get; set; } - - /// - /// The minimum log event level required in order to write an event to the sink. - /// - public LogEventLevel? MinimumLogEventLevel { get; set; } - - /// - /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation - /// - public IElasticsearchSerializer Serializer { get; set; } - - /// - /// The connectionpool describing the cluster to write event to - /// - public IConnectionPool ConnectionPool { get; private set; } - - /// - /// Optional path to directory that can be used as a log shipping buffer for increasing the reliability of the log forwarding. - /// - public string BufferBaseFilename { get; set; } - - /// - /// The maximum size, in bytes, to which the buffer log file for a specific date will be allowed to grow. By default no limit will be applied. - /// - public long? BufferFileSizeLimitBytes { get; set; } - - /// - /// The interval between checking the buffer files - /// - public TimeSpan? BufferLogShippingInterval { get; set; } - - /// - /// Customizes the formatter used when converting log events into ElasticSearch documents. Please note that the formatter output must be valid JSON :) - /// - public ITextFormatter CustomFormatter { get; set; } - - /// - /// Function to decide which index to write the LogEvent to - /// - public Func IndexDecider { get; set; } - - /// - /// Configures the elasticsearch sink - /// - /// The connectionpool to use to write events to - public ElasticsearchSinkOptions(IConnectionPool connectionPool) - { - ConnectionPool = connectionPool; - } - - /// - /// Configures the elasticsearch sink - /// - /// The nodes to write to - public ElasticsearchSinkOptions(IEnumerable nodes) - { - nodes = nodes != null && nodes.Any(n=>n != null) - ? nodes.Where(n=>n != null) - : new[] { new Uri("http://localhost:9200") }; - if (nodes.Count() == 1) - ConnectionPool = new SingleNodeConnectionPool(nodes.First()); - else - ConnectionPool = new StaticConnectionPool(nodes); - } - - /// - /// Configures the elasticsearch sink - /// - /// The node to write to - public ElasticsearchSinkOptions(Uri node) : this(new [] {node}) { } - } + /// + /// Provides ElasticsearchSink with configurable options + /// + public class ElasticsearchSinkOptions + { + + /// + /// When set to true the sink will register an index template for the logs in elasticsearch. + /// This template is optimized to deal with serilog events + /// + public bool AutoRegisterTemplate { get; set; } + + /// + /// When using the feature this allows you to override the default template name. + /// Defaults to: serilog-events-template + /// + public string TemplateName { get; set; } + + /// + /// Connection configuration to use for connecting to the cluster. + /// + public Func ModifyConnectionSetttings { get; set; } + + /// + /// The index name formatter. A string.Format using the DateTimeOffset of the event is run over this string. + /// defaults to "logstash-{0:yyyy.MM.dd}" + /// + public string IndexFormat { get; set; } + + /// + /// The default elasticsearch type name to use for the log events defaults to: logevent + /// + public string TypeName { get; set; } + + /// + /// The maximum number of events to post in a single batch. + /// + public int BatchPostingLimit { get; set; } + + /// + /// The time to wait between checking for event batches. Defaults to 2 seconds. + /// + public TimeSpan Period { get; set; } + + /// + /// Supplies culture-specific formatting information, or null. + /// + public IFormatProvider FormatProvider { get; set; } + + /// + /// Allows you to override the connection used to communicate with elasticsearch + /// + public IConnection Connection { get; set; } + + /// + /// The connection timeout (in milliseconds) when sending bulk operations to elasticsearch (defaults to 5000) + /// + public int ConnectionTimeout { get; set; } + + /// + /// When true fields will be written at the root of the json document + /// + public bool InlineFields { get; set; } + + /// + /// The minimum log event level required in order to write an event to the sink. + /// + public LogEventLevel? MinimumLogEventLevel { get; set; } + + /// + /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation + /// + public IElasticsearchSerializer Serializer { get; set; } + + /// + /// The connectionpool describing the cluster to write event to + /// + public IConnectionPool ConnectionPool { get; private set; } + + /// + /// Function to decide which index to write the LogEvent to + /// + public Func IndexDecider { get; set; } + + + /// + /// Optional path to directory that can be used as a log shipping buffer for increasing the reliability of the log forwarding. + /// + public string BufferBaseFilename { get; set; } + + /// + /// The maximum size, in bytes, to which the buffer log file for a specific date will be allowed to grow. By default no limit will be applied. + /// + public long? BufferFileSizeLimitBytes { get; set; } + + /// + /// The interval between checking the buffer files + /// + public TimeSpan? BufferLogShippingInterval { get; set; } + + /// + /// Customizes the formatter used when converting log events into ElasticSearch documents. Please note that the formatter output must be valid JSON :) + /// + public ITextFormatter CustomFormatter { get; set; } + + + /// + /// Configures the elasticsearch sink defaults + /// + protected ElasticsearchSinkOptions() + { + this.IndexFormat = "logstash-{0:yyyy.MM.dd}"; + this.TypeName = "logevent"; + this.Period = TimeSpan.FromSeconds(2); + this.BatchPostingLimit = 50; + this.TemplateName = "serilog-events-template"; + this.ConnectionTimeout = 5000; + } + + /// + /// Configures the elasticsearch sink + /// + /// The connectionpool to use to write events to + public ElasticsearchSinkOptions(IConnectionPool connectionPool) + : this() + { + ConnectionPool = connectionPool; + } + + /// + /// Configures the elasticsearch sink + /// + /// The nodes to write to + public ElasticsearchSinkOptions(IEnumerable nodes) + : this() + { + nodes = nodes != null && nodes.Any(n => n != null) + ? nodes.Where(n => n != null) + : new[] { new Uri("http://localhost:9200") }; + if (nodes.Count() == 1) + ConnectionPool = new SingleNodeConnectionPool(nodes.First()); + else + ConnectionPool = new StaticConnectionPool(nodes); + } + + /// + /// Configures the elasticsearch sink + /// + /// The node to write to + public ElasticsearchSinkOptions(Uri node) : this(new[] { node }) { } + } } \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs.orig b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs.orig deleted file mode 100644 index d59e4a1d..00000000 --- a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkOptions.cs.orig +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2014 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.Linq; -using Elasticsearch.Net.Connection; -using Elasticsearch.Net.ConnectionPool; -using Elasticsearch.Net.Serialization; -using Serilog.Events; -using Serilog.Formatting; - -namespace Serilog.Sinks.ElasticSearch -{ - /// - /// Provides ElasticsearchSink with configurable options - /// - public class ElasticsearchSinkOptions - { - /// - /// Connection configuration to use for connecting to the cluster. - /// - public Func ModifyConnectionSetttings { get; set; } - - /// - /// The index name formatter. A string.Format using the DateTimeOffset of the event is run over this string. - /// defaults to "logstash-{0:yyyy.MM.dd}" - /// - public string IndexFormat { get; set; } - - /// - /// The default elasticsearch type name to use for the log events defaults to: logevent - /// - public string TypeName { get; set; } - - /// - /// The maximum number of events to post in a single batch. - /// - public int? BatchPostingLimit { get; set; } - - /// - /// The time to wait between checking for event batches. - /// - public TimeSpan? Period { get; set; } - - /// - /// Supplies culture-specific formatting information, or null. - /// - public IFormatProvider FormatProvider { get; set; } - - /// - /// Allows you to override the connection used to communicate with elasticsearch - /// - public IConnection Connection { get; set; } - - /// - /// When true fields will be written at the root of the json document - /// - public bool InlineFields { get; set; } - - /// - /// The minimum log event level required in order to write an event to the sink. - /// - public LogEventLevel? MinimumLogEventLevel { get; set; } - - /// - /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation - /// - public IElasticsearchSerializer Serializer { get; set; } - - /// - /// The connectionpool describing the cluster to write event to - /// - public IConnectionPool ConnectionPool { get; private set; } - - /// -<<<<<<< HEAD - /// Function to decide which index to write the LogEvent to - /// - public Func IndexDecider { get; set; } -======= - /// Optional path to directory that can be used as a log shipping buffer for increasing the reliability of the log forwarding. - /// - public string BufferBaseFilename { get; set; } - - /// - /// The maximum size, in bytes, to which the buffer log file for a specific date will be allowed to grow. By default no limit will be applied. - /// - public long? BufferFileSizeLimitBytes { get; set; } - - /// - /// The interval between checking the buffer files - /// - public TimeSpan? BufferLogShippingInterval { get; set; } - - /// - /// Customizes the formatter used when converting log events into ElasticSearch documents. Please note that the formatter output must be valid JSON :) - /// - public ITextFormatter CustomFormatter { get; set; } ->>>>>>> 35384861bb26488009b884f4a919d50a2970d5b9 - - /// - /// Configures the elasticsearch sink - /// - /// The connectionpool to use to write events to - public ElasticsearchSinkOptions(IConnectionPool connectionPool) - { - ConnectionPool = connectionPool; - } - - /// - /// Configures the elasticsearch sink - /// - /// The nodes to write to - public ElasticsearchSinkOptions(IEnumerable nodes) - { - nodes = nodes != null && nodes.Any(n=>n != null) - ? nodes.Where(n=>n != null) - : new[] { new Uri("http://localhost:9200") }; - if (nodes.Count() == 1) - ConnectionPool = new SingleNodeConnectionPool(nodes.First()); - else - ConnectionPool = new StaticConnectionPool(nodes); - } - - /// - /// Configures the elasticsearch sink - /// - /// The node to write to - public ElasticsearchSinkOptions(Uri node) : this(new [] {node}) { } - } -} \ No newline at end of file diff --git a/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs new file mode 100644 index 00000000..984981fb --- /dev/null +++ b/src/Serilog.Sinks.Elasticsearch/Sinks/ElasticSearch/ElasticsearchSinkState.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Elasticsearch.Net; +using Elasticsearch.Net.Connection; +using Elasticsearch.Net.Serialization; +using Serilog.Events; +using Serilog.Formatting; + +namespace Serilog.Sinks.Elasticsearch +{ + internal class ElasticsearchSinkState + { + public static ElasticsearchSinkState Create(ElasticsearchSinkOptions options) + { + if (options == null) throw new ArgumentNullException("options"); + var state = new ElasticsearchSinkState(options); + if (state.Options.AutoRegisterTemplate) + state.RegisterTemplateIfNeeded(); + return state; + } + + private readonly ElasticsearchSinkOptions _options; + readonly Func _indexDecider; + + private readonly ITextFormatter _formatter; + private readonly ElasticsearchClient _client; + + readonly string _typeName; + private readonly bool _registerTemplateOnStartup; + private readonly string _templateName; + private readonly string _templateMatchString; + private static readonly Regex IndexFormatRegex = new Regex(@"^(.*)(?:\{0\:.+\})(.*)$"); + + public ElasticsearchSinkOptions Options { get { return this._options; }} + public IElasticsearchClient Client { get { return this._client; }} + public ITextFormatter Formatter { get { return this._formatter; }} + + + private ElasticsearchSinkState(ElasticsearchSinkOptions options) + { + if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat"); + if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); + if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName"); + + this._templateName = options.TemplateName; + this._templateMatchString = IndexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); + + _indexDecider = options.IndexDecider ?? ((@event, offset) => string.Format(options.IndexFormat, offset)); + + _typeName = options.TypeName; + _options = options; + + var configuration = new ConnectionConfiguration(options.ConnectionPool) + .SetTimeout(options.ConnectionTimeout) + .SetMaximumAsyncConnections(20); + + if (options.ModifyConnectionSetttings != null) + configuration = options.ModifyConnectionSetttings(configuration); + + _client = new ElasticsearchClient(configuration, connection: options.Connection, serializer: options.Serializer); + _formatter = options.CustomFormatter ?? new ElasticsearchJsonFormatter( + formatProvider: options.FormatProvider, + renderMessage: true, + closingDelimiter: string.Empty, + serializer: options.Serializer, + inlineFields: options.InlineFields + ); + + this._registerTemplateOnStartup = options.AutoRegisterTemplate; + } + + + public string Serialize(object o) + { + var bytes = _client.Serializer.Serialize(o, SerializationFormatting.None); + return Encoding.UTF8.GetString(bytes); + } + + public string GetIndexForEvent(LogEvent e, DateTimeOffset offset) + { + return this._indexDecider(e, offset); + } + + /// + /// Register the elasticsearch index template if the provided options mandate it. + /// + public void RegisterTemplateIfNeeded() + { + if (!this._registerTemplateOnStartup) return; + var result = this._client.IndicesPutTemplateForAll(this._templateName, new + { + template = this._templateMatchString, + settings = new Dictionary + { + {"index.refresh_interval", "5s"} + }, + mappings = new + { + _default_ = new + { + _all = new { enabled = true }, + dynamic_templates = new[] + { + new + { + string_fields = new + { + match = "*", + match_mapping_type = "string", + mapping = new + { + type = "string", index = "analyzed", omit_norms = true, + fields = new + { + raw = new + { + type= "string", index = "not_analyzed", ignore_above = 256 + } + } + } + } + } + }, + properties = new Dictionary + { + { "message", new { type = "string", index = "analyzed" } }, + { "exceptions", new + { + type = "nested", properties = new Dictionary + { + { "Depth", new { type = "integer" } }, + { "RemoteStackIndex", new { type = "integer" } }, + { "HResult", new { type = "integer" } }, + { "StackTraceString", new { type = "string", index = "analyzed" } }, + { "RemoteStackTraceString", new { type = "string", index = "analyzed" } }, + { "ExceptionMessage", new + { + type = "object", properties = new Dictionary + { + { "MemberType", new { type = "integer" } }, + } + }} + } + } } + } + } + } + }); + } + + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs index fad3130f..825085b0 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/CustomIndexTypeNameTests.cs @@ -4,7 +4,7 @@ using FluentAssertions; using Serilog.Events; using Serilog.Parsing; -using Serilog.Sinks.ElasticSearch; +using Serilog.Sinks.Elasticsearch; using NUnit.Framework; namespace Serilog.Sinks.Elasticsearch.Tests diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchDefaultSerializerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchDefaultSerializerTests.cs new file mode 100644 index 00000000..5030fd25 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchDefaultSerializerTests.cs @@ -0,0 +1,19 @@ +using System.Linq; +using Elasticsearch.Net.Serialization; +using NUnit.Framework; + +namespace Serilog.Sinks.Elasticsearch.Tests.Discrepancies +{ + [TestFixture] + public class ElasticsearchDefaultSerializerTests : ElasticsearchSinkUniformityTestsBase + { + public ElasticsearchDefaultSerializerTests() : base(new ElasticsearchDefaultSerializer()) { } + + [Test] + public void Should_SerializeToExpandedExceptionObjectWhenExceptionIsSet() + { + this.ThrowAndLogAndCatchBulkOutput("test_with_default_serializer"); + } + } + +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchSinkUniformityTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchSinkUniformityTestsBase.cs new file mode 100644 index 00000000..81c7877e --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/ElasticsearchSinkUniformityTestsBase.cs @@ -0,0 +1,104 @@ +using System; +using System.Runtime.Serialization; +using Elasticsearch.Net.Serialization; +using FluentAssertions; + +namespace Serilog.Sinks.Elasticsearch.Tests.Discrepancies +{ + public class ElasticsearchSinkUniformityTestsBase : ElasticsearchSinkTestsBase + { + public ElasticsearchSinkUniformityTestsBase(IElasticsearchSerializer serializer) + { + _options.Serializer = serializer; + } + + public void ThrowAndLogAndCatchBulkOutput(string exceptionMessage) + { + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + try + { + try + { + throw new Exception("inner most exception"); + } + catch (Exception e) + { + var innerException = new NastyException("nasty inner exception", e); + throw new Exception(exceptionMessage, innerException); + } + } + catch (Exception e) + { + logger.Error(e, "Test exception. Should contain an embedded exception object."); + } + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + var postedEvents = this.GetPostedLogEvents(expectedCount: 2); + Console.WriteLine("BULK OUTPUT BEGIN =========="); + foreach (var post in _seenHttpPosts) + Console.WriteLine(post); + Console.WriteLine("BULK OUTPUT END ============"); + + var firstEvent = postedEvents[0]; + firstEvent.Exceptions.Should().NotBeNull().And.HaveCount(3); + firstEvent.Exceptions[0].Message.Should().NotBeNullOrWhiteSpace() + .And.Be(exceptionMessage); + var realException = firstEvent.Exceptions[0]; + realException.ExceptionMethod.Should().NotBeNull(); + realException.ExceptionMethod.Name.Should().NotBeNullOrWhiteSpace(); + realException.ExceptionMethod.AssemblyName.Should().NotBeNullOrWhiteSpace(); + realException.ExceptionMethod.AssemblyVersion.Should().NotBeNullOrWhiteSpace(); + realException.ExceptionMethod.ClassName.Should().NotBeNullOrWhiteSpace(); + realException.ExceptionMethod.Signature.Should().NotBeNullOrWhiteSpace(); + realException.ExceptionMethod.MemberType.Should().BeGreaterThan(0); + + var nastyException = firstEvent.Exceptions[1]; + nastyException.Depth.Should().Be(1); + nastyException.Message.Should().Be("nasty inner exception"); + nastyException.HelpUrl.Should().Be("help url"); + nastyException.StackTraceString.Should().Be("stack trace string"); + nastyException.RemoteStackTraceString.Should().Be("remote stack trace string"); + nastyException.RemoteStackIndex.Should().Be(1); + nastyException.HResult.Should().Be(123123); + nastyException.Source.Should().Be("source"); + nastyException.ClassName.Should().Be("classname nasty exception"); + //nastyException.WatsonBuckets.Should().BeEquivalentTo(new byte[] {1,2,3}); + + + var secondEvent = postedEvents[1]; + secondEvent.Exceptions.Should().BeNullOrEmpty(); + } + } + + /// + /// Exception that forces often empty serializationinfo values to have a value + /// + public class NastyException : Exception + { + public NastyException(string message) : base(message) { } + public NastyException(string message, Exception innerException) : base(message, innerException) { } + + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + info.AddValue("Message", this.Message); + info.AddValue("HelpURL", "help url"); + info.AddValue("StackTraceString", "stack trace string"); + info.AddValue("RemoteStackTraceString", "remote stack trace string"); + info.AddValue("RemoteStackIndex", 1); + info.AddValue("ExceptionMethod", "exception method"); + info.AddValue("HResult", 123123); + info.AddValue("Source", "source"); + info.AddValue("ClassName", "classname nasty exception"); + info.AddValue("WatsonBuckets", new byte[] { 1, 2, 3 }, typeof(byte[])); + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/JsonNetSerializerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/JsonNetSerializerTests.cs new file mode 100644 index 00000000..4a863468 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/JsonNetSerializerTests.cs @@ -0,0 +1,22 @@ +using System; +using System.Linq; +using Elasticsearch.Net.JsonNet; +using Elasticsearch.Net.Serialization; +using FluentAssertions; +using NUnit.Framework; + +namespace Serilog.Sinks.Elasticsearch.Tests.Discrepancies +{ + [TestFixture] + public class JsonNetSerializerTests : ElasticsearchSinkUniformityTestsBase + { + public JsonNetSerializerTests() : base(new ElasticsearchJsonNetSerializer()) { } + + [Test] + public void Should_SerializeToExpandedExceptionObjectWhenExceptionIsSet() + { + this.ThrowAndLogAndCatchBulkOutput("test_with_jsonnet_serializer"); + } + } + +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/NoSerializerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/NoSerializerTests.cs new file mode 100644 index 00000000..a6c590ca --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Discrepancies/NoSerializerTests.cs @@ -0,0 +1,20 @@ +using System; +using System.Linq; +using FluentAssertions; +using NUnit.Framework; + +namespace Serilog.Sinks.Elasticsearch.Tests.Discrepancies +{ + [TestFixture] + public class NoSerializerTests : ElasticsearchSinkUniformityTestsBase + { + public NoSerializerTests() : base(null) {} + + [Test] + public void Should_SerializeToExpandedExceptionObjectWhenExceptionIsSet() + { + this.ThrowAndLogAndCatchBulkOutput("test_with_no_serializer"); + } + } + +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Domain/BulkAction.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Domain/BulkAction.cs new file mode 100644 index 00000000..8ab550ef --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Domain/BulkAction.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Serilog.Events; + +namespace Serilog.Sinks.Elasticsearch.Tests.Domain +{ + /// + /// Elasticsearch _bulk follows a specific pattern: + /// {operation}\n + /// {operationmetadata}\n + /// This provides a marker interface for both + /// + interface IBulkData { } + + public class BulkOperation : IBulkData + { + [JsonProperty("index")] + public IndexAction IndexAction { get; set; } + } + + public class IndexAction + { + [JsonProperty("_index")] + public string Index { get; set; } + [JsonProperty("_type")] + public string Type { get; set; } + } + + public class SerilogElasticsearchEvent : IBulkData + { + [JsonProperty("@timestamp")] + public DateTime Timestamp { get; set; } + + [JsonProperty("level")] + [JsonConverter(typeof(StringEnumConverter))] + public LogEventLevel Level { get; set; } + + [JsonProperty("messageTemplate")] + public string MessageTemplate { get; set; } + + [JsonProperty("message")] + public string Message { get; set; } + + [JsonProperty("exceptions")] + public List Exceptions { get; set; } + } + + public class SerilogElasticsearchExceptionInfo + { + public int Depth { get; set; } + public string ClassName { get; set; } + public string Message { get; set; } + public string Source { get; set; } + public string StackTraceString { get; set; } + public string RemoteStackTraceString { get; set; } + public int RemoteStackIndex { get; set; } + public SerilogExceptionMethodInfo ExceptionMethod { get; set; } + public int HResult { get; set; } + public string HelpUrl { get; set; } + + //writing byte[] will fall back to serializer and they differ in output + //JsonNET assumes string, simplejson writes array of numerics. + //Skip for now + + //public byte[] WatsonBuckets { get; set; } + } + + public class SerilogExceptionMethodInfo + { + public string Name { get; set; } + public string AssemblyName { get; set; } + public string AssemblyVersion { get; set; } + public string AssemblyCulture { get; set; } + public string ClassName { get; set; } + public string Signature { get; set; } + public int MemberType { get; set; } + } +} diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs index 87b03361..128695fc 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/ElasticsearchSinkTestsBase.cs @@ -1,12 +1,15 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using Elasticsearch.Net; using Elasticsearch.Net.Connection; using Elasticsearch.Net.Connection.Configuration; +using Elasticsearch.Net.JsonNet; using FakeItEasy; -using Serilog.Sinks.ElasticSearch; +using FluentAssertions; +using Serilog.Sinks.Elasticsearch.Tests.Domain; namespace Serilog.Sinks.Elasticsearch.Tests { @@ -16,9 +19,13 @@ public abstract class ElasticsearchSinkTestsBase protected readonly IConnection _connection; protected readonly ElasticsearchSinkOptions _options; protected readonly List _seenHttpPosts = new List(); + protected readonly List> _seenHttpPuts = new List>(); + private ElasticsearchJsonNetSerializer _serializer; protected ElasticsearchSinkTestsBase() { + Serilog.Debugging.SelfLog.Out = Console.Out; + _serializer = new ElasticsearchJsonNetSerializer(); _connection = A.Fake(); _options = new ElasticsearchSinkOptions(new Uri("http://localhost:9200")) { @@ -26,13 +33,67 @@ protected ElasticsearchSinkTestsBase() Period = TinyWait, Connection = _connection }; - var fixedRespone = new MemoryStream(Encoding.UTF8.GetBytes(@"{ ""ok"": true }")); + A.CallTo(() => _connection.PutSync(A._, A._, A._)) + .ReturnsLazily((Uri uri, byte[] postData, IRequestConfiguration requestConfiguration) => + { + var fixedRespone = new MemoryStream(Encoding.UTF8.GetBytes(@"{ ""ok"": true }")); + _seenHttpPuts.Add(Tuple.Create(uri, Encoding.UTF8.GetString(postData))); + return ElasticsearchResponse.Create(new ConnectionConfiguration(), 200, "PUT", "/", postData, fixedRespone); + }); A.CallTo(() => _connection.PostSync(A._, A._, A._)) .ReturnsLazily((Uri uri, byte[] postData, IRequestConfiguration requestConfiguration) => { + var fixedRespone = new MemoryStream(Encoding.UTF8.GetBytes(@"{ ""ok"": true }")); _seenHttpPosts.Add(Encoding.UTF8.GetString(postData)); return ElasticsearchResponse.Create(new ConnectionConfiguration(), 200, "POST", "/", postData, fixedRespone); }); } + + /// + /// Returns the posted serilog messages and validates the entire bulk in the process + /// + /// + /// + protected IList GetPostedLogEvents(int expectedCount) + { + this._seenHttpPosts.Should().NotBeNullOrEmpty(); + var totalBulks = this._seenHttpPosts.SelectMany(p=>p.Split(new []{"\n"}, StringSplitOptions.RemoveEmptyEntries)).ToList(); + totalBulks.Should().NotBeNullOrEmpty().And.HaveCount(expectedCount*2); + + var bulkActions = new List(); + for (var i = 0; i < totalBulks.Count; i += 2) + { + BulkOperation action; + try + { + action = this.Deserialize(totalBulks[i]); + } + catch (Exception e) + { + throw new Exception(string.Format("Can not deserialize into BulkOperation \r\n:{0}", totalBulks[i]), e); + } + action.IndexAction.Should().NotBeNull(); + action.IndexAction.Index.Should().NotBeNullOrEmpty().And.StartWith("logstash-"); + action.IndexAction.Type.Should().NotBeNullOrEmpty().And.Be("logevent"); + + SerilogElasticsearchEvent actionMetaData; + try + { + actionMetaData = this.Deserialize(totalBulks[i + 1]); + } + catch (Exception e) + { + throw new Exception(string.Format("Can not deserialize into SerilogElasticsearchMessage \r\n:{0}", totalBulks[i + 1]), e); + } + actionMetaData.Should().NotBeNull(); + bulkActions.Add(actionMetaData); + } + return bulkActions; + } + + protected T Deserialize(string json) + { + return this._serializer.Deserialize(new MemoryStream(Encoding.UTF8.GetBytes(json))); + } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/IndexDeciderTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/IndexDeciderTests.cs index c058f7f5..9ba040ef 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/IndexDeciderTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/IndexDeciderTests.cs @@ -5,7 +5,6 @@ using NUnit.Framework; using Serilog.Events; using Serilog.Parsing; -using Serilog.Sinks.ElasticSearch; namespace Serilog.Sinks.Elasticsearch.Tests { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/InlineFieldsTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/InlineFieldsTests.cs index dd576a2b..c5e9ef67 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/InlineFieldsTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/InlineFieldsTests.cs @@ -6,7 +6,6 @@ using FluentAssertions; using Serilog.Events; using Serilog.Parsing; -using Serilog.Sinks.ElasticSearch; using NUnit.Framework; namespace Serilog.Sinks.Elasticsearch.Tests diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/PropertyNameTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/PropertyNameTests.cs index f75c3ab8..d59edac7 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/PropertyNameTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/PropertyNameTests.cs @@ -1,20 +1,12 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Net.Http; -using System.Text; using System.Threading.Tasks; -using Elasticsearch.Net; -using Elasticsearch.Net.Connection; -using Elasticsearch.Net.Connection.Configuration; -using Elasticsearch.Net.JsonNet; -using FakeItEasy; using FluentAssertions; +using NUnit.Framework; using Serilog.Events; using Serilog.Parsing; -using Serilog.Sinks.ElasticSearch; -using NUnit.Framework; namespace Serilog.Sinks.Elasticsearch.Tests { diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionNoSerializerTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionNoSerializerTests.cs index 54c9725a..bb822cbc 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionNoSerializerTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionNoSerializerTests.cs @@ -1,19 +1,12 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Net.Http; -using System.Text; using System.Threading.Tasks; -using Elasticsearch.Net; -using Elasticsearch.Net.Connection; -using Elasticsearch.Net.Connection.Configuration; -using FakeItEasy; using FluentAssertions; +using NUnit.Framework; using Serilog.Events; using Serilog.Parsing; -using Serilog.Sinks.ElasticSearch; -using NUnit.Framework; namespace Serilog.Sinks.Elasticsearch.Tests { @@ -54,9 +47,8 @@ public async Task WhenPassingASerializer_ShouldExpandToJson() bulkJsonPieces[1].Should().NotContain("Properties\""); bulkJsonPieces[2].Should().Contain(@"""_index"":""logstash-2013.05.30"); - //We have no serializer associated with the sink so we expect the forced ToString() of scalar values bulkJsonPieces[3].Should().Contain("Complex\":\"{"); - bulkJsonPieces[3].Should().Contain("exception\":\"System.Net.Http.HttpRequestException: An error"); + bulkJsonPieces[3].Should().Contain("exceptions\":[{\"Depth\":0"); } } } diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionTests.cs index 0b93ea74..8cf702ce 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionTests.cs +++ b/test/Serilog.Sinks.Elasticsearch.Tests/RealExceptionTests.cs @@ -1,21 +1,13 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Net.Http; -using System.Text; using System.Threading.Tasks; -using Elasticsearch.Net; -using Elasticsearch.Net.Connection; -using Elasticsearch.Net.Connection.Configuration; using Elasticsearch.Net.JsonNet; -using FakeItEasy; using FluentAssertions; -using Newtonsoft.Json; +using NUnit.Framework; using Serilog.Events; using Serilog.Parsing; -using Serilog.Sinks.ElasticSearch; -using NUnit.Framework; namespace Serilog.Sinks.Elasticsearch.Tests { @@ -60,8 +52,7 @@ public async Task WhenPassingASerializer_ShouldExpandToJson() //tostring implemenation //DO NOTE that you cant send objects as scalar values through Logger.*("{Scalar}", {}); bulkJsonPieces[3].Should().Contain("Complex\":{"); - //Since we are passing a ISerializer the exception should be be logged as object and not string - bulkJsonPieces[3].Should().Contain("exception\":{"); + bulkJsonPieces[3].Should().Contain("exceptions\":[{"); } } } diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj index c92fb3ad..ee6390b7 100644 --- a/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Serilog.Sinks.Elasticsearch.Tests.csproj @@ -70,7 +70,12 @@ + + + + + @@ -78,9 +83,14 @@ + + + + Always + diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateTests.cs new file mode 100644 index 00000000..d031442e --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/SendsTemplateTests.cs @@ -0,0 +1,75 @@ +using System; +using System.IO; +using System.Reflection; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + [TestFixture] + public class SendsTemplateTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public SendsTemplateTests() + { + _options.AutoRegisterTemplate = true; + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + + } + + + [Test] + public void ShouldRegisterTheCorrectTemplateOnRegistration() + { + this.JsonEquals(this._templatePut.Item2, MethodBase.GetCurrentMethod(), "template"); + } + + [Test] + public void TemplatePutToCorrectUrl() + { + var uri = this._templatePut.Item1; + uri.AbsolutePath.Should().Be("/_template/serilog-events-template"); + } + + protected void JsonEquals(string json, MethodBase method, string fileName = null) + { + var file = this.GetFileFromMethod(method, fileName); + var exists = File.Exists(file); + exists.Should().BeTrue(file + "does not exist"); + + var expected = File.ReadAllText(file); + var nJson = JObject.Parse(json); + var nOtherJson = JObject.Parse(expected); + var equals = JToken.DeepEquals(nJson, nOtherJson); + if (equals) return; + expected.Should().BeEquivalentTo(json); + + } + protected string GetFileFromMethod(MethodBase method, string fileName) + { + var type = method.DeclaringType; + var @namespace = method.DeclaringType.Namespace; + var folderSep = Path.DirectorySeparatorChar.ToString(); + var folder = @namespace.Replace("Serilog.Sinks.Elasticsearch.Tests.", "").Replace(".", folderSep); + var file = Path.Combine(folder, (fileName ?? method.Name).Replace(@"\", folderSep) + ".json"); + file = Path.Combine(Environment.CurrentDirectory.Replace("bin" + folderSep + "Debug", "").Replace("bin" + folderSep + "Release", ""), file); + return file; + } + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/TemplateMatchTests.cs b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/TemplateMatchTests.cs new file mode 100644 index 00000000..5554df01 --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/TemplateMatchTests.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; +using System.Reflection; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +namespace Serilog.Sinks.Elasticsearch.Tests.Templating +{ + [TestFixture] + public class TemplateMatchTests : ElasticsearchSinkTestsBase + { + private readonly Tuple _templatePut; + + public TemplateMatchTests() + { + _options.AutoRegisterTemplate = true; + _options.IndexFormat = "dailyindex-{0:yyyy.MM.dd}-mycompany"; + _options.TemplateName = "dailyindex-logs-template"; + var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug() + .Enrich.WithMachineName() + .WriteTo.ColoredConsole() + .WriteTo.Elasticsearch(_options); + + var logger = loggerConfig.CreateLogger(); + using (logger as IDisposable) + { + logger.Error("Test exception. Should not contain an embedded exception object."); + } + + this._seenHttpPosts.Should().NotBeNullOrEmpty().And.HaveCount(1); + this._seenHttpPuts.Should().NotBeNullOrEmpty().And.HaveCount(1); + _templatePut = this._seenHttpPuts[0]; + + } + + [Test] + public void TemplatePutToCorrectUrl() + { + var uri = this._templatePut.Item1; + uri.AbsolutePath.Should().Be("/_template/dailyindex-logs-template"); + } + + [Test] + public void TemplateMatchShouldReflectConfiguredIndexFormat() + { + var json = this._templatePut.Item2; + json.Should().Contain(@"""template"":""dailyindex-*-mycompany"""); + } + + } +} \ No newline at end of file diff --git a/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template.json b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template.json new file mode 100644 index 00000000..86e3a3bb --- /dev/null +++ b/test/Serilog.Sinks.Elasticsearch.Tests/Templating/template.json @@ -0,0 +1,69 @@ +{ + "template": "logstash-*", + "settings": { + "index.refresh_interval": "5s" + }, + "mappings": { + "_default_": { + "_all": { + "enabled": true + }, + "dynamic_templates": [ + { + "string_fields": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "string", + "index": "analyzed", + "omit_norms": true, + "fields": { + "raw": { + "type": "string", + "index": "not_analyzed", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "message": { + "type": "string", + "index": "analyzed" + }, + "exceptions": { + "type": "nested", + "properties": { + "Depth": { + "type": "integer" + }, + "RemoteStackIndex": { + "type": "integer" + }, + "HResult": { + "type": "integer" + }, + "StackTraceString": { + "type": "string", + "index": "analyzed" + }, + "RemoteStackTraceString": { + "type": "string", + "index": "analyzed" + }, + "ExceptionMessage": { + "type": "object", + "properties": { + "MemberType": { + "type": "integer" + } + } + } + } + } + } + } + } +} \ No newline at end of file