From 0bbdb25f98764c1665acc3e4f924038ed87e11b7 Mon Sep 17 00:00:00 2001 From: Jan Trejbal Date: Fri, 24 Mar 2023 17:04:21 +0100 Subject: [PATCH 1/2] [Extensions] Add LogToActivityEventConversionOptions.Filter callback (#1059) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Piotr Kiełkowicz --- .../.publicApi/net462/PublicAPI.Unshipped.txt | 2 + .../.publicApi/net6.0/PublicAPI.Unshipped.txt | 2 + .../netstandard2.0/PublicAPI.Unshipped.txt | 2 + src/OpenTelemetry.Extensions/CHANGELOG.md | 3 + .../ActivityEventAttachingLogProcessor.cs | 15 ++++ .../OpenTelemetryExtensionsEventSource.cs | 15 ++++ .../LogToActivityEventConversionOptions.cs | 13 +++ ...ActivityEventAttachingLogProcessorTests.cs | 79 ++++++++++++++++++- 8 files changed, 129 insertions(+), 2 deletions(-) diff --git a/src/OpenTelemetry.Extensions/.publicApi/net462/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Extensions/.publicApi/net462/PublicAPI.Unshipped.txt index 0caf94ce2c..02a765cd0b 100644 --- a/src/OpenTelemetry.Extensions/.publicApi/net462/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Extensions/.publicApi/net462/PublicAPI.Unshipped.txt @@ -1,6 +1,8 @@ #nullable enable Microsoft.Extensions.Logging.OpenTelemetryLoggingExtensions OpenTelemetry.Logs.LogToActivityEventConversionOptions +OpenTelemetry.Logs.LogToActivityEventConversionOptions.Filter.get -> System.Func? +OpenTelemetry.Logs.LogToActivityEventConversionOptions.Filter.set -> void OpenTelemetry.Logs.LogToActivityEventConversionOptions.LogToActivityEventConversionOptions() -> void OpenTelemetry.Logs.LogToActivityEventConversionOptions.ScopeConverter.get -> System.Action! OpenTelemetry.Logs.LogToActivityEventConversionOptions.ScopeConverter.set -> void diff --git a/src/OpenTelemetry.Extensions/.publicApi/net6.0/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Extensions/.publicApi/net6.0/PublicAPI.Unshipped.txt index 0caf94ce2c..02a765cd0b 100644 --- a/src/OpenTelemetry.Extensions/.publicApi/net6.0/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Extensions/.publicApi/net6.0/PublicAPI.Unshipped.txt @@ -1,6 +1,8 @@ #nullable enable Microsoft.Extensions.Logging.OpenTelemetryLoggingExtensions OpenTelemetry.Logs.LogToActivityEventConversionOptions +OpenTelemetry.Logs.LogToActivityEventConversionOptions.Filter.get -> System.Func? +OpenTelemetry.Logs.LogToActivityEventConversionOptions.Filter.set -> void OpenTelemetry.Logs.LogToActivityEventConversionOptions.LogToActivityEventConversionOptions() -> void OpenTelemetry.Logs.LogToActivityEventConversionOptions.ScopeConverter.get -> System.Action! OpenTelemetry.Logs.LogToActivityEventConversionOptions.ScopeConverter.set -> void diff --git a/src/OpenTelemetry.Extensions/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Extensions/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt index 0caf94ce2c..02a765cd0b 100644 --- a/src/OpenTelemetry.Extensions/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Extensions/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt @@ -1,6 +1,8 @@ #nullable enable Microsoft.Extensions.Logging.OpenTelemetryLoggingExtensions OpenTelemetry.Logs.LogToActivityEventConversionOptions +OpenTelemetry.Logs.LogToActivityEventConversionOptions.Filter.get -> System.Func? +OpenTelemetry.Logs.LogToActivityEventConversionOptions.Filter.set -> void OpenTelemetry.Logs.LogToActivityEventConversionOptions.LogToActivityEventConversionOptions() -> void OpenTelemetry.Logs.LogToActivityEventConversionOptions.ScopeConverter.get -> System.Action! OpenTelemetry.Logs.LogToActivityEventConversionOptions.ScopeConverter.set -> void diff --git a/src/OpenTelemetry.Extensions/CHANGELOG.md b/src/OpenTelemetry.Extensions/CHANGELOG.md index f0c228edd6..0e618d7a3d 100644 --- a/src/OpenTelemetry.Extensions/CHANGELOG.md +++ b/src/OpenTelemetry.Extensions/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +* Add LogToActivityEventConversionOptions.Filter callback + ([#1059](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/1059)) + ## 1.0.0-beta.4 Released 2023-Feb-27 diff --git a/src/OpenTelemetry.Extensions/Internal/ActivityEventAttachingLogProcessor.cs b/src/OpenTelemetry.Extensions/Internal/ActivityEventAttachingLogProcessor.cs index eb38ff899b..33e8b2f887 100644 --- a/src/OpenTelemetry.Extensions/Internal/ActivityEventAttachingLogProcessor.cs +++ b/src/OpenTelemetry.Extensions/Internal/ActivityEventAttachingLogProcessor.cs @@ -50,6 +50,21 @@ public override void OnEnd(LogRecord data) if (activity?.IsAllDataRequested == true) { + try + { + if (this.options.Filter?.Invoke(data) == false) + { + return; + } + } +#pragma warning disable CA1031 // Do not catch general exception types + catch (Exception ex) +#pragma warning restore CA1031 // Do not catch general exception types + { + OpenTelemetryExtensionsEventSource.Log.LogRecordFilterException(data.CategoryName, ex); + return; + } + var tags = new ActivityTagsCollection { { nameof(data.CategoryName), data.CategoryName }, diff --git a/src/OpenTelemetry.Extensions/Internal/OpenTelemetryExtensionsEventSource.cs b/src/OpenTelemetry.Extensions/Internal/OpenTelemetryExtensionsEventSource.cs index 5a5d446831..1b199c2832 100644 --- a/src/OpenTelemetry.Extensions/Internal/OpenTelemetryExtensionsEventSource.cs +++ b/src/OpenTelemetry.Extensions/Internal/OpenTelemetryExtensionsEventSource.cs @@ -42,4 +42,19 @@ public void LogProcessorException(string @event, string exception) { this.WriteEvent(1, @event, exception); } + + [NonEvent] + public void LogRecordFilterException(string? categoryName, Exception ex) + { + if (this.IsEnabled(EventLevel.Warning, (EventKeywords)(-1))) + { + this.LogRecordFilterException(categoryName, ex.ToInvariantString()); + } + } + + [Event(2, Message = "Filter threw an exception, log record will not be attached to an activity, the log record would flow to its pipeline unaffected. CategoryName: '{0}', Exception: {1}.", Level = EventLevel.Warning)] + public void LogRecordFilterException(string? categoryName, string exception) + { + this.WriteEvent(2, categoryName, exception); + } } diff --git a/src/OpenTelemetry.Extensions/Logs/LogToActivityEventConversionOptions.cs b/src/OpenTelemetry.Extensions/Logs/LogToActivityEventConversionOptions.cs index 3451f0775c..fa79214378 100644 --- a/src/OpenTelemetry.Extensions/Logs/LogToActivityEventConversionOptions.cs +++ b/src/OpenTelemetry.Extensions/Logs/LogToActivityEventConversionOptions.cs @@ -34,4 +34,17 @@ public class LogToActivityEventConversionOptions /// Gets or sets the callback action used to convert log scopes into tags. /// public Action ScopeConverter { get; set; } = DefaultLogStateConverter.ConvertScope; + + /// + /// Gets or sets the callback method allowing to filter out particular . + /// + /// + /// The filter callback receives the for the + /// processed logRecord and should return a boolean. + /// + /// If filter returns the event is collected. + /// If filter returns or throws an exception the event is filtered out (NOT collected). + /// + /// + public Func? Filter { get; set; } } diff --git a/test/OpenTelemetry.Extensions.Tests/ActivityEventAttachingLogProcessorTests.cs b/test/OpenTelemetry.Extensions.Tests/ActivityEventAttachingLogProcessorTests.cs index ee0e28f688..e0a787ba73 100644 --- a/test/OpenTelemetry.Extensions.Tests/ActivityEventAttachingLogProcessorTests.cs +++ b/test/OpenTelemetry.Extensions.Tests/ActivityEventAttachingLogProcessorTests.cs @@ -57,13 +57,16 @@ public void Dispose() [InlineData(false)] [InlineData(true, 18, true, true, true)] [InlineData(true, 0, false, false, true, true)] + [InlineData(true, 18, true, true, true, false, true)] + [InlineData(true, 0, false, false, true, true, true)] public void AttachLogsToActivityEventTest( bool sampled, int eventId = 0, bool includeFormattedMessage = false, bool parseStateValues = false, bool includeScopes = false, - bool recordException = false) + bool recordException = false, + bool? filter = null) { this.sampled = sampled; @@ -74,7 +77,15 @@ public void AttachLogsToActivityEventTest( options.IncludeScopes = includeScopes; options.IncludeFormattedMessage = includeFormattedMessage; options.ParseStateValues = parseStateValues; - options.AttachLogsToActivityEvent(); + options.AttachLogsToActivityEvent(x => + { + x.Filter = filter switch + { + true => _ => true, + false => _ => false, + null => null, + }; + }); }); builder.AddFilter(typeof(ActivityEventAttachingLogProcessorTests).FullName, LogLevel.Trace); }); @@ -180,4 +191,68 @@ public void AttachLogsToActivityEventTest( Assert.Empty(activity.Events); } } + + [Theory] + [InlineData(true, true)] + [InlineData(false, true)] + [InlineData(true, true, 18, true, true, true)] + [InlineData(true, true, 0, false, false, true, true)] + [InlineData(true, false)] + [InlineData(false, false)] + [InlineData(true, false, 18, true, true, true)] + [InlineData(true, false, 0, false, false, true, true)] + public void AttachLogsToActivityEventTest_Filter( + bool sampled, + bool filterThrows, + int eventId = 0, + bool includeFormattedMessage = false, + bool parseStateValues = false, + bool includeScopes = false, + bool recordException = false) + { + this.sampled = sampled; + + using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => + { + builder.AddOpenTelemetry(options => + { + options.IncludeScopes = includeScopes; + options.IncludeFormattedMessage = includeFormattedMessage; + options.ParseStateValues = parseStateValues; + options.AttachLogsToActivityEvent(x => x.Filter = _ => filterThrows + ? throw new Exception() + : false); + }); + builder.AddFilter(typeof(ActivityEventAttachingLogProcessorTests).FullName, LogLevel.Trace); + }); + + ILogger logger = loggerFactory.CreateLogger(); + Activity activity = this.activitySource.StartActivity("Test"); + + using IDisposable scope = logger.BeginScope("{NodeId}", 99); + + logger.LogInformation(eventId, "Hello OpenTelemetry {UserId}!", 8); + + if (recordException) + { + var innerActivity = this.activitySource.StartActivity("InnerTest"); + + using IDisposable innerScope = logger.BeginScope("{RequestId}", "1234"); + + logger.LogError(new InvalidOperationException("Goodbye OpenTelemetry."), "Exception event."); + + innerActivity.Dispose(); + } + + activity.Dispose(); + + if (sampled) + { + Assert.DoesNotContain(activity.Events, x => x.Name == "log"); + } + else + { + Assert.Empty(activity.Events); + } + } } From 595a9a34e3e0f10f25df64153a83889ad386a3fc Mon Sep 17 00:00:00 2001 From: Utkarsh Umesan Pillai Date: Fri, 24 Mar 2023 09:26:42 -0700 Subject: [PATCH 2/2] [Exporter.Geneva] Refactor GenevaMetricExporter (#1102) Co-authored-by: Cijo Thomas --- .../Metrics/GenevaMetricExporter.cs | 555 ++++++++---------- 1 file changed, 238 insertions(+), 317 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs index 4128c03bb3..72819a4f56 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Metrics/GenevaMetricExporter.cs @@ -508,143 +508,17 @@ internal unsafe ushort SerializeMetricWithTLV( // Leave enough space for the header var bufferIndex = sizeof(BinaryHeader); - // Serialize metric name - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.MetricName); - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(metricName)); + SerializeMetricName(metricName, this.buffer, ref bufferIndex); - #region Serialize metric data + SerializeNonHistogramMetricData(eventType, value, timestamp, this.buffer, ref bufferIndex); - var payloadType = eventType == MetricEventType.ULongMetric ? PayloadType.ULongMetric : PayloadType.DoubleMetric; - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)payloadType); + SerializeMetricDimensions(tags, this.prepopulatedDimensionsCount, this.serializedPrepopulatedDimensionsKeys, this.serializedPrepopulatedDimensionsValues, this.buffer, ref bufferIndex); - // Get a placeholder to add the payloadType length - int payloadTypeStartIndex = bufferIndex; - bufferIndex += 2; - - MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, (ulong)timestamp); // timestamp - - if (payloadType == PayloadType.ULongMetric) - { - MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, value.UInt64Value); - } - else - { - MetricSerializer.SerializeFloat64(this.buffer, ref bufferIndex, value.DoubleValue); - } - - var payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); - MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); - - #endregion - - #region Serialize metric dimensions - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Dimensions); - - // Get a placeholder to add the payloadType length - payloadTypeStartIndex = bufferIndex; - bufferIndex += 2; - - // Get a placeholder to add dimensions count later - var bufferIndexForDimensionsCount = bufferIndex; - bufferIndex += 2; - - ushort dimensionsWritten = 0; - - // Serialize PrepopulatedDimensions keys - for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) - { - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, this.serializedPrepopulatedDimensionsKeys[i]); - } - - if (this.prepopulatedDimensionsCount > 0) - { - dimensionsWritten += this.prepopulatedDimensionsCount; - } - - // Serialize MetricPoint Dimension keys - foreach (var tag in tags) - { - if (tag.Key.Length > MaxDimensionNameSize) - { - // TODO: Data Validation - } - - MetricSerializer.SerializeString(this.buffer, ref bufferIndex, tag.Key); - } - - dimensionsWritten += (ushort)tags.Count; - - // Serialize PrepopulatedDimensions values - for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) - { - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, this.serializedPrepopulatedDimensionsValues[i]); - } - - // Serialize MetricPoint Dimension values - foreach (var tag in tags) - { - var dimensionValue = Convert.ToString(tag.Value, CultureInfo.InvariantCulture); - if (dimensionValue.Length > MaxDimensionValueSize) - { - // TODO: Data Validation - } - - MetricSerializer.SerializeString(this.buffer, ref bufferIndex, dimensionValue); - } - - // Backfill the number of dimensions written - MetricSerializer.SerializeUInt16(this.buffer, ref bufferIndexForDimensionsCount, dimensionsWritten); - - payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); - MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); - - #endregion - - #region Serialize exemplars - - if (exemplars.Length > 0) - { - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Exemplars); - - // Get a placeholder to add the payloadType length - payloadTypeStartIndex = bufferIndex; - bufferIndex += 2; - - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, 0); // version - - // TODO: Avoid this additional enumeration - var exemplarsCount = 0; - foreach (var exemplar in exemplars) - { - if (exemplar.Timestamp != default) - { - exemplarsCount++; - } - } - - MetricSerializer.SerializeInt32AsBase128(this.buffer, ref bufferIndex, exemplarsCount); - - foreach (var exemplar in exemplars) - { - if (exemplar.Timestamp != default) - { - this.SerializeExemplar(exemplar, ref bufferIndex); - } - } + SerializeExemplars(exemplars, this.buffer, ref bufferIndex); - payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); - MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); - } + SerializeMonitoringAccount(this.monitoringAccount, this.buffer, ref bufferIndex); - #endregion - - // Serialize monitoring account - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.AccountName); - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(this.monitoringAccount)); - - // Serialize metric namespace - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Namespace); - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(this.metricNamespace)); + SerializeMetricNamespace(this.metricNamespace, this.buffer, ref bufferIndex); // Write the final size of the payload bodyLength = (ushort)(bufferIndex - this.fixedPayloadStartIndex); @@ -685,227 +559,163 @@ internal unsafe ushort SerializeHistogramMetricWithTLV( // Leave enough space for the header var bufferIndex = sizeof(BinaryHeader); - // Serialize metric name - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.MetricName); - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(metricName)); - - #region Serialize histogram metric data - - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.ExternallyAggregatedULongDistributionMetric); + SerializeMetricName(metricName, this.buffer, ref bufferIndex); - // Get a placeholder to add the payloadType length - int payloadTypeStartIndex = bufferIndex; - bufferIndex += 2; + SerializeHistogramMetricData(buckets, sum, count, min, max, timestamp, this.buffer, ref bufferIndex); - // Serialize sum, count, min, and max - MetricSerializer.SerializeUInt32(this.buffer, ref bufferIndex, count); // histogram count - MetricSerializer.SerializeUInt32(this.buffer, ref bufferIndex, 0); // padding - MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, (ulong)timestamp); // timestamp - MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(sum)); // histogram sum - MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(min)); // histogram min - MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(max)); // histogram max + SerializeMetricDimensions(tags, this.prepopulatedDimensionsCount, this.serializedPrepopulatedDimensionsKeys, this.serializedPrepopulatedDimensionsValues, this.buffer, ref bufferIndex); - var payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); - MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + SerializeExemplars(exemplars, this.buffer, ref bufferIndex); - // Serialize histogram buckets as value-count pairs - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.HistogramULongValueCountPairs); + SerializeMonitoringAccount(this.monitoringAccount, this.buffer, ref bufferIndex); - // Get a placeholder to add the payloadType length - payloadTypeStartIndex = bufferIndex; - bufferIndex += 2; + SerializeMetricNamespace(this.metricNamespace, this.buffer, ref bufferIndex); - // Get a placeholder to add the number of value-count pairs added - // with value being the bucket boundary and count being the respective count - - var itemsWrittenIndex = bufferIndex; - MetricSerializer.SerializeUInt16(this.buffer, ref bufferIndex, 0); + // Write the final size of the payload + bodyLength = (ushort)(bufferIndex - this.fixedPayloadStartIndex); - // Bucket values - ushort bucketCount = 0; - double lastExplicitBound = default; - foreach (var bucket in buckets) + // Copy in the final structures to the front + fixed (byte* bufferBytes = this.buffer) { - if (bucket.BucketCount > 0) - { - this.SerializeHistogramBucketWithTLV(bucket, ref bufferIndex, lastExplicitBound); - bucketCount++; - } - - lastExplicitBound = bucket.ExplicitBound; + var ptr = (BinaryHeader*)bufferBytes; + ptr->EventId = (ushort)MetricEventType.TLV; + ptr->BodyLength = bodyLength; } + } + finally + { + } - // Write the number of items in distribution emitted and reset back to end. - MetricSerializer.SerializeUInt16(this.buffer, ref itemsWrittenIndex, bucketCount); + return bodyLength; + } - payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); - MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SerializeMetricName(string metricName, byte[] buffer, ref int bufferIndex) + { + MetricSerializer.SerializeByte(buffer, ref bufferIndex, (byte)PayloadType.MetricName); + MetricSerializer.SerializeEncodedString(buffer, ref bufferIndex, Encoding.UTF8.GetBytes(metricName)); + } - #endregion + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SerializeMetricNamespace(string metricNamespace, byte[] buffer, ref int bufferIndex) + { + MetricSerializer.SerializeByte(buffer, ref bufferIndex, (byte)PayloadType.Namespace); + MetricSerializer.SerializeEncodedString(buffer, ref bufferIndex, Encoding.UTF8.GetBytes(metricNamespace)); + } - #region Serialize metric dimensions - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Dimensions); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SerializeMonitoringAccount(string monitoringAccount, byte[] buffer, ref int bufferIndex) + { + MetricSerializer.SerializeByte(buffer, ref bufferIndex, (byte)PayloadType.AccountName); + MetricSerializer.SerializeEncodedString(buffer, ref bufferIndex, Encoding.UTF8.GetBytes(monitoringAccount)); + } - // Get a placeholder to add the payloadType length - payloadTypeStartIndex = bufferIndex; - bufferIndex += 2; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SerializeMetricDimensions(in ReadOnlyTagCollection tags, ushort prepopulatedDimensionsCount, List serializedPrepopulatedDimensionsKeys, List serializedPrepopulatedDimensionsValues, byte[] buffer, ref int bufferIndex) + { + MetricSerializer.SerializeByte(buffer, ref bufferIndex, (byte)PayloadType.Dimensions); - // Get a placeholder to add dimensions count later - var bufferIndexForDimensionsCount = bufferIndex; - bufferIndex += 2; + // Get a placeholder to add the payloadType length + var payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; - ushort dimensionsWritten = 0; + // Get a placeholder to add dimensions count later + var bufferIndexForDimensionsCount = bufferIndex; + bufferIndex += 2; - // Serialize PrepopulatedDimensions keys - for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) - { - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, this.serializedPrepopulatedDimensionsKeys[i]); - } + ushort dimensionsWritten = 0; - if (this.prepopulatedDimensionsCount > 0) + // Serialize PrepopulatedDimensions keys + for (ushort i = 0; i < prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(buffer, ref bufferIndex, serializedPrepopulatedDimensionsKeys[i]); + } + + if (prepopulatedDimensionsCount > 0) + { + dimensionsWritten += prepopulatedDimensionsCount; + } + + // Serialize MetricPoint Dimension keys + foreach (var tag in tags) + { + if (tag.Key.Length > MaxDimensionNameSize) { - dimensionsWritten += this.prepopulatedDimensionsCount; + // TODO: Data Validation } - // Serialize MetricPoint Dimension keys - foreach (var tag in tags) - { - if (tag.Key.Length > MaxDimensionNameSize) - { - // TODO: Data Validation - } + MetricSerializer.SerializeString(buffer, ref bufferIndex, tag.Key); + } - MetricSerializer.SerializeString(this.buffer, ref bufferIndex, tag.Key); - } + dimensionsWritten += (ushort)tags.Count; - dimensionsWritten += (ushort)tags.Count; + // Serialize PrepopulatedDimensions values + for (ushort i = 0; i < prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(buffer, ref bufferIndex, serializedPrepopulatedDimensionsValues[i]); + } - // Serialize PrepopulatedDimensions values - for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + // Serialize MetricPoint Dimension values + foreach (var tag in tags) + { + var dimensionValue = Convert.ToString(tag.Value, CultureInfo.InvariantCulture); + if (dimensionValue.Length > MaxDimensionValueSize) { - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, this.serializedPrepopulatedDimensionsValues[i]); + // TODO: Data Validation } - // Serialize MetricPoint Dimension values - foreach (var tag in tags) - { - var dimensionValue = Convert.ToString(tag.Value, CultureInfo.InvariantCulture); - if (dimensionValue.Length > MaxDimensionValueSize) - { - // TODO: Data Validation - } + MetricSerializer.SerializeString(buffer, ref bufferIndex, dimensionValue); + } - MetricSerializer.SerializeString(this.buffer, ref bufferIndex, dimensionValue); - } + // Backfill the number of dimensions written + MetricSerializer.SerializeUInt16(buffer, ref bufferIndexForDimensionsCount, dimensionsWritten); - // Backfill the number of dimensions written - MetricSerializer.SerializeUInt16(this.buffer, ref bufferIndexForDimensionsCount, dimensionsWritten); + var payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(buffer, ref payloadTypeStartIndex, payloadTypeLength); + } - payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); - MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SerializeExemplars(Exemplar[] exemplars, byte[] buffer, ref int bufferIndex) + { + if (exemplars.Length > 0) + { + MetricSerializer.SerializeByte(buffer, ref bufferIndex, (byte)PayloadType.Exemplars); - #endregion + // Get a placeholder to add the payloadType length + var payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; - #region Serialize exemplars + MetricSerializer.SerializeByte(buffer, ref bufferIndex, 0); // version - if (exemplars.Length > 0) + // TODO: Avoid this additional enumeration + var exemplarsCount = 0; + foreach (var exemplar in exemplars) { - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Exemplars); - - // Get a placeholder to add the payloadType length - payloadTypeStartIndex = bufferIndex; - bufferIndex += 2; - - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, 0); // version - - // TODO: Avoid this additional enumeration - var exemplarsCount = 0; - foreach (var exemplar in exemplars) - { - if (exemplar.Timestamp != default) - { - exemplarsCount++; - } - } - - MetricSerializer.SerializeInt32AsBase128(this.buffer, ref bufferIndex, exemplarsCount); - - foreach (var exemplar in exemplars) + if (exemplar.Timestamp != default) { - if (exemplar.Timestamp != default) - { - this.SerializeExemplar(exemplar, ref bufferIndex); - } + exemplarsCount++; } - - payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); - MetricSerializer.SerializeUInt16(this.buffer, ref payloadTypeStartIndex, payloadTypeLength); } - #endregion - - // Serialize monitoring account - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.AccountName); - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(this.monitoringAccount)); - - // Serialize metric namespace - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, (byte)PayloadType.Namespace); - MetricSerializer.SerializeEncodedString(this.buffer, ref bufferIndex, Encoding.UTF8.GetBytes(this.metricNamespace)); + MetricSerializer.SerializeInt32AsBase128(buffer, ref bufferIndex, exemplarsCount); - // Write the final size of the payload - bodyLength = (ushort)(bufferIndex - this.fixedPayloadStartIndex); - - // Copy in the final structures to the front - fixed (byte* bufferBytes = this.buffer) + foreach (var exemplar in exemplars) { - var ptr = (BinaryHeader*)bufferBytes; - ptr->EventId = (ushort)MetricEventType.TLV; - ptr->BodyLength = bodyLength; + if (exemplar.Timestamp != default) + { + SerializeSingleExmeplar(exemplar, buffer, ref bufferIndex); + } } - } - finally - { - } - - return bodyLength; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void SerializeHistogramBucket(in HistogramBucket bucket, ref int bufferIndex, double lastExplicitBound) - { - if (bucket.ExplicitBound != double.PositiveInfinity) - { - MetricSerializer.SerializeUInt64(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt64(bucket.ExplicitBound)); - } - else - { - // The bucket to catch the overflows is one greater than the last bound provided - MetricSerializer.SerializeUInt64(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt64(lastExplicitBound + 1)); - } - - MetricSerializer.SerializeUInt32(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt32(bucket.BucketCount)); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void SerializeHistogramBucketWithTLV(in HistogramBucket bucket, ref int bufferIndex, double lastExplicitBound) - { - if (bucket.ExplicitBound != double.PositiveInfinity) - { - MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(bucket.ExplicitBound)); - } - else - { - // The bucket to catch the overflows is one greater than the last bound provided - MetricSerializer.SerializeUInt64(this.buffer, ref bufferIndex, Convert.ToUInt64(lastExplicitBound + 1)); + var payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(buffer, ref payloadTypeStartIndex, payloadTypeLength); } - - MetricSerializer.SerializeUInt32(this.buffer, ref bufferIndex, Convert.ToUInt32(bucket.BucketCount)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void SerializeExemplar(Exemplar exemplar, ref int bufferIndex) + private static void SerializeSingleExmeplar(Exemplar exemplar, byte[] buffer, ref int bufferIndex) { - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, 0); // version + MetricSerializer.SerializeByte(buffer, ref bufferIndex, 0); // version var bufferIndexForLength = bufferIndex; bufferIndex++; @@ -924,15 +734,15 @@ private void SerializeExemplar(Exemplar exemplar, ref int bufferIndex) if (isWholeNumber) { flags |= ExemplarFlags.IsMetricValueDoubleStoredAsLong; - MetricSerializer.SerializeInt64AsBase128(this.buffer, ref bufferIndex, valueAsLong); // serialize long value + MetricSerializer.SerializeInt64AsBase128(buffer, ref bufferIndex, valueAsLong); // serialize long value } else { - MetricSerializer.SerializeFloat64(this.buffer, ref bufferIndex, value); // serialize double value + MetricSerializer.SerializeFloat64(buffer, ref bufferIndex, value); // serialize double value } var bufferIndexForNumberOfLabels = bufferIndex; - MetricSerializer.SerializeByte(this.buffer, ref bufferIndex, 0); // serialize zero as the count of labels; this would be updated later if the exemplar has labels + MetricSerializer.SerializeByte(buffer, ref bufferIndex, 0); // serialize zero as the count of labels; this would be updated later if the exemplar has labels byte numberOfLabels = 0; // Convert exemplar timestamp to unix nanoseconds @@ -941,13 +751,13 @@ private void SerializeExemplar(Exemplar exemplar, ref int bufferIndex) .Subtract(new DateTime(1970, 1, 1)) .TotalMilliseconds * 1000000; - MetricSerializer.SerializeInt64(this.buffer, ref bufferIndex, (long)unixNanoSeconds); // serialize timestamp + MetricSerializer.SerializeInt64(buffer, ref bufferIndex, (long)unixNanoSeconds); // serialize timestamp if (exemplar.TraceId.HasValue) { Span traceIdBytes = stackalloc byte[16]; exemplar.TraceId.Value.CopyTo(traceIdBytes); - MetricSerializer.SerializeSpanOfBytes(this.buffer, ref bufferIndex, traceIdBytes, traceIdBytes.Length); // serialize traceId + MetricSerializer.SerializeSpanOfBytes(buffer, ref bufferIndex, traceIdBytes, traceIdBytes.Length); // serialize traceId flags |= ExemplarFlags.TraceIdExists; } @@ -956,7 +766,7 @@ private void SerializeExemplar(Exemplar exemplar, ref int bufferIndex) { Span spanIdBytes = stackalloc byte[8]; exemplar.SpanId.Value.CopyTo(spanIdBytes); - MetricSerializer.SerializeSpanOfBytes(this.buffer, ref bufferIndex, spanIdBytes, spanIdBytes.Length); // serialize spanId + MetricSerializer.SerializeSpanOfBytes(buffer, ref bufferIndex, spanIdBytes, spanIdBytes.Length); // serialize spanId flags |= ExemplarFlags.SpanIdExists; } @@ -966,18 +776,129 @@ private void SerializeExemplar(Exemplar exemplar, ref int bufferIndex) { foreach (var tag in exemplar.FilteredTags) { - MetricSerializer.SerializeBase128String(this.buffer, ref bufferIndex, tag.Key); - MetricSerializer.SerializeBase128String(this.buffer, ref bufferIndex, Convert.ToString(tag.Value, CultureInfo.InvariantCulture)); + MetricSerializer.SerializeBase128String(buffer, ref bufferIndex, tag.Key); + MetricSerializer.SerializeBase128String(buffer, ref bufferIndex, Convert.ToString(tag.Value, CultureInfo.InvariantCulture)); numberOfLabels++; } - MetricSerializer.SerializeByte(this.buffer, ref bufferIndexForNumberOfLabels, numberOfLabels); + MetricSerializer.SerializeByte(buffer, ref bufferIndexForNumberOfLabels, numberOfLabels); } - MetricSerializer.SerializeByte(this.buffer, ref bufferIndexForFlags, (byte)flags); + MetricSerializer.SerializeByte(buffer, ref bufferIndexForFlags, (byte)flags); var exemplarLength = bufferIndex - bufferIndexForLength + 1; - MetricSerializer.SerializeByte(this.buffer, ref bufferIndexForLength, (byte)exemplarLength); + MetricSerializer.SerializeByte(buffer, ref bufferIndexForLength, (byte)exemplarLength); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SerializeNonHistogramMetricData(MetricEventType eventType, MetricData value, long timestamp, byte[] buffer, ref int bufferIndex) + { + var payloadType = eventType == MetricEventType.ULongMetric ? PayloadType.ULongMetric : PayloadType.DoubleMetric; + MetricSerializer.SerializeByte(buffer, ref bufferIndex, (byte)payloadType); + + // Get a placeholder to add the payloadType length + int payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + MetricSerializer.SerializeUInt64(buffer, ref bufferIndex, (ulong)timestamp); // timestamp + + if (payloadType == PayloadType.ULongMetric) + { + MetricSerializer.SerializeUInt64(buffer, ref bufferIndex, value.UInt64Value); + } + else + { + MetricSerializer.SerializeFloat64(buffer, ref bufferIndex, value.DoubleValue); + } + + var payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(buffer, ref payloadTypeStartIndex, payloadTypeLength); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SerializeHistogramMetricData(HistogramBuckets buckets, double sum, uint count, double min, double max, long timestamp, byte[] buffer, ref int bufferIndex) + { + MetricSerializer.SerializeByte(buffer, ref bufferIndex, (byte)PayloadType.ExternallyAggregatedULongDistributionMetric); + + // Get a placeholder to add the payloadType length + int payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + // Serialize sum, count, min, and max + MetricSerializer.SerializeUInt32(buffer, ref bufferIndex, count); // histogram count + MetricSerializer.SerializeUInt32(buffer, ref bufferIndex, 0); // padding + MetricSerializer.SerializeUInt64(buffer, ref bufferIndex, (ulong)timestamp); // timestamp + MetricSerializer.SerializeUInt64(buffer, ref bufferIndex, Convert.ToUInt64(sum)); // histogram sum + MetricSerializer.SerializeUInt64(buffer, ref bufferIndex, Convert.ToUInt64(min)); // histogram min + MetricSerializer.SerializeUInt64(buffer, ref bufferIndex, Convert.ToUInt64(max)); // histogram max + + var payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(buffer, ref payloadTypeStartIndex, payloadTypeLength); + + // Serialize histogram buckets as value-count pairs + MetricSerializer.SerializeByte(buffer, ref bufferIndex, (byte)PayloadType.HistogramULongValueCountPairs); + + // Get a placeholder to add the payloadType length + payloadTypeStartIndex = bufferIndex; + bufferIndex += 2; + + // Get a placeholder to add the number of value-count pairs added + // with value being the bucket boundary and count being the respective count + + var itemsWrittenIndex = bufferIndex; + MetricSerializer.SerializeUInt16(buffer, ref bufferIndex, 0); + + // Bucket values + ushort bucketCount = 0; + double lastExplicitBound = default; + foreach (var bucket in buckets) + { + if (bucket.BucketCount > 0) + { + SerializeHistogramBucketWithTLV(bucket, buffer, ref bufferIndex, lastExplicitBound); + bucketCount++; + } + + lastExplicitBound = bucket.ExplicitBound; + } + + // Write the number of items in distribution emitted and reset back to end. + MetricSerializer.SerializeUInt16(buffer, ref itemsWrittenIndex, bucketCount); + + payloadTypeLength = (ushort)(bufferIndex - payloadTypeStartIndex - 2); + MetricSerializer.SerializeUInt16(buffer, ref payloadTypeStartIndex, payloadTypeLength); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void SerializeHistogramBucketWithTLV(in HistogramBucket bucket, byte[] buffer, ref int bufferIndex, double lastExplicitBound) + { + if (bucket.ExplicitBound != double.PositiveInfinity) + { + MetricSerializer.SerializeUInt64(buffer, ref bufferIndex, Convert.ToUInt64(bucket.ExplicitBound)); + } + else + { + // The bucket to catch the overflows is one greater than the last bound provided + MetricSerializer.SerializeUInt64(buffer, ref bufferIndex, Convert.ToUInt64(lastExplicitBound + 1)); + } + + MetricSerializer.SerializeUInt32(buffer, ref bufferIndex, Convert.ToUInt32(bucket.BucketCount)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SerializeHistogramBucket(in HistogramBucket bucket, ref int bufferIndex, double lastExplicitBound) + { + if (bucket.ExplicitBound != double.PositiveInfinity) + { + MetricSerializer.SerializeUInt64(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt64(bucket.ExplicitBound)); + } + else + { + // The bucket to catch the overflows is one greater than the last bound provided + MetricSerializer.SerializeUInt64(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt64(lastExplicitBound + 1)); + } + + MetricSerializer.SerializeUInt32(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt32(bucket.BucketCount)); } private List SerializePrepopulatedDimensionsKeys(IEnumerable keys)