From 9c6f77308367449cb058e18dc826659d315d03f1 Mon Sep 17 00:00:00 2001 From: JoshLove-msft <54595583+JoshLove-msft@users.noreply.github.com> Date: Tue, 1 Mar 2022 17:44:01 -0800 Subject: [PATCH 1/4] Add logging of cache size --- .../src/LruCache.cs | 43 ++++-- ...zure.Data.SchemaRegistry.ApacheAvro.csproj | 1 + .../src/SchemaRegistryAvroEventSource.cs | 64 +++++++++ .../src/SchemaRegistryAvroSerializer.cs | 127 +++++++++--------- .../tests/EventSourceLiveTests.cs | 103 ++++++++++++++ ...egistryAvroObjectSerializerLiveTestBase.cs | 23 ++++ ...maRegistryAvroObjectSerializerLiveTests.cs | 12 +- .../UpdatingCacheLogsEvents.json | 105 +++++++++++++++ .../UpdatingCacheLogsEventsAsync.json | 105 +++++++++++++++ 9 files changed, 499 insertions(+), 84 deletions(-) create mode 100644 sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroEventSource.cs create mode 100644 sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/EventSourceLiveTests.cs create mode 100644 sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SchemaRegistryAvroObjectSerializerLiveTestBase.cs create mode 100644 sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SessionRecords/EventSourceLiveTests/UpdatingCacheLogsEvents.json create mode 100644 sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SessionRecords/EventSourceLiveTests/UpdatingCacheLogsEventsAsync.json diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs index 60699fc8989d8..29c6fdd161821 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System.Collections; using System.Collections.Generic; namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro @@ -8,24 +9,26 @@ namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro /// /// A simple LRU cache implementation using a doubly linked list and dictionary. /// - /// The type of key - /// The type of value - internal class LruCache + /// The type of key + /// The type of value + internal class LruCache : IEnumerable> { private readonly int _capacity; - private readonly LinkedList> _linkedList; - private readonly Dictionary>> _map; + internal readonly LinkedList> _linkedList; + private readonly Dictionary>> _map; private readonly object _syncLock; + internal int Count => _linkedList.Count; + public LruCache(int capacity) { _capacity = capacity; - _linkedList = new LinkedList>(); - _map = new Dictionary>>(); + _linkedList = new LinkedList>(); + _map = new Dictionary>>(); _syncLock = new object(); } - public bool TryGet(K key, out V value) + public bool TryGet(TKey key, out TValue value) { lock (_syncLock) { @@ -37,12 +40,12 @@ public bool TryGet(K key, out V value) return true; } - value = default(V); + value = default(TValue); return false; } } - public void AddOrUpdate(K key, V val) + public void AddOrUpdate(TKey key, TValue val) { lock (_syncLock) { @@ -53,18 +56,34 @@ public void AddOrUpdate(K key, V val) } // add new node - var node = new LinkedListNode>(new KeyValuePair(key, val)); + var node = new LinkedListNode>(new KeyValuePair(key, val)); _linkedList.AddFirst(node); _map[key] = node; if (_map.Count > _capacity) { // remove least recently used node - LinkedListNode> last = _linkedList.Last; + LinkedListNode> last = _linkedList.Last; _linkedList.RemoveLast(); _map.Remove(last.Value.Key); } } } + + internal int ComputeSize() + { + int size = 0; + foreach (KeyValuePair kvp in _linkedList) + { + size += kvp.Key.ToString().Length * sizeof(char); + size += kvp.Value.ToString().Length * sizeof(char); + } + + return size; + } + + public IEnumerator> GetEnumerator() => _linkedList.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } } \ No newline at end of file diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj index 7aa39faee6fdc..6d1b41dae5907 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj @@ -16,6 +16,7 @@ + diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroEventSource.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroEventSource.cs new file mode 100644 index 0000000000000..f265af48315c5 --- /dev/null +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroEventSource.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Linq; +using Avro; +using Azure.Core.Diagnostics; + +namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro +{ + /// + /// Serves as an ETW event source for logging of information about + /// Entity's client. + /// + /// + /// + /// When defining Start/Complete tasks, it is highly recommended that the + /// the CompleteEvent.Id must be exactly StartEvent.Id + 1. + /// + [EventSource(Name = EventSourceName)] + internal class SchemaRegistryAvroEventSource : AzureEventSource + { + /// The name to use for the event source. + private const string EventSourceName = "Microsoft-Azure-Data-SchemaRegistry-ApacheAvro"; + + internal const int CacheUpdatedEvent = 1; + + /// + /// Provides a singleton instance of the event source for callers to + /// use for logging. + /// + public static SchemaRegistryAvroEventSource Log { get; } = new SchemaRegistryAvroEventSource(); + + /// + /// Prevents an instance of the class from being + /// created outside the scope of the instance, as well as setting up the + /// integration with AzureEventSourceListener. + /// + protected SchemaRegistryAvroEventSource() : base(EventSourceName) + { + } + + [NonEvent] + public virtual void CacheUpdated(LruCache idToSchemaCache, LruCache schemaToIdCache) + { + if (IsEnabled()) + { + int totalSchemaLength = idToSchemaCache.Sum(kvp => kvp.Value.SchemaLength); + totalSchemaLength += schemaToIdCache.Sum(kvp => kvp.Value.SchemaLength); + CacheUpdatedCore(idToSchemaCache.Count + schemaToIdCache.Count, totalSchemaLength); + } + } + + [Event(CacheUpdatedEvent, Level = EventLevel.Verbose, Message = "Cache entry added or updated. Total number of entries: {0}; Total schema length: {1}")] + public virtual void CacheUpdatedCore(int entryCount, int totalSchemaLength) + { + if (IsEnabled()) + { + WriteEvent(CacheUpdatedEvent, entryCount, totalSchemaLength); + } + } + } +} \ No newline at end of file diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroSerializer.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroSerializer.cs index 82f6ce631ea55..50dd3692e4f40 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroSerializer.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroSerializer.cs @@ -20,7 +20,7 @@ namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro { /// /// A uses the to - /// encode and decode Avro payloads. + /// serialize and deserialize Avro payloads. /// public class SchemaRegistryAvroSerializer { @@ -45,8 +45,8 @@ public SchemaRegistryAvroSerializer(SchemaRegistryClient client, string groupNam private const int RecordFormatIndicatorLength = 4; private const int SchemaIdLength = 32; private const int PayloadStartPosition = RecordFormatIndicatorLength + SchemaIdLength; - private readonly LruCache _idToSchemaMap = new(CacheCapacity); - private readonly LruCache _schemaToIdMap = new(CacheCapacity); + private readonly LruCache _idToSchemaMap = new(CacheCapacity); + private readonly LruCache _schemaToIdMap = new(CacheCapacity); private enum SupportedType { @@ -57,41 +57,41 @@ private enum SupportedType #region Serialize /// - /// Encodes the message data as Avro and stores it in . The - /// will be set to "avro/binary+schemaId" where schemaId is the ID of the schema used to encode the data. + /// Serializes the message data as Avro and stores it in . The + /// will be set to "avro/binary+schemaId" where schemaId is the ID of the schema used to serialize the data. /// - /// The data to serialize to Avro and encode into the message. + /// The data to serialize to Avro and serialize into the message. /// An optional instance to signal the request to cancel the operation. - /// The type to encode the data into. - /// The type of the data to encode. + /// The type to serialize the data into. + /// The type of the data to serialize. public TEnvelope Serialize( TData data, CancellationToken cancellationToken = default) where TEnvelope : BinaryContent, new() => (TEnvelope) SerializeInternalAsync(data, typeof(TData), typeof(TEnvelope), false, cancellationToken).EnsureCompleted(); /// - /// Encodes the message data as Avro and stores it in . The - /// will be set to "avro/binary+schemaId" where schemaId is the ID of the schema used to encode the data. + /// serializes the message data as Avro and stores it in . The + /// will be set to "avro/binary+schemaId" where schemaId is the ID of the schema used to serialize the data. /// - /// The data to serialize to Avro and encode into the message. + /// The data to serialize to Avro and serialize into the message. /// An optional instance to signal the request to cancel the operation. - /// The type to encode the data into. - /// The type of the data to encode. + /// The type to serialize the data into. + /// The type of the data to serialize. public async ValueTask SerializeAsync( TData data, CancellationToken cancellationToken = default) where TEnvelope : BinaryContent, new() => (TEnvelope) await SerializeInternalAsync(data, typeof(TData), typeof(TEnvelope), true, cancellationToken).ConfigureAwait(false); /// - /// Encodes the message data as Avro and stores it in . The - /// will be set to "avro/binary+schemaId" where schemaId is the ID of the schema used to encode the data. + /// serializes the message data as Avro and stores it in . The + /// will be set to "avro/binary+schemaId" where schemaId is the ID of the schema used to serialize the data. /// - /// The data to serialize to Avro and encode into the message. - /// The type of the data to encode. If left blank, the type will be determined at runtime by + /// The data to serialize to Avro and serialize into the message. + /// The type of the data to serialize. If left blank, the type will be determined at runtime by /// calling . - /// The type of message to encode the data into. Must extend from , and + /// The type of message to serialize the data into. Must extend from , and /// have a parameterless constructor. - /// If left blank, the data will be encoded into a instance. + /// If left blank, the data will be serialized into a instance. /// An optional instance to signal the request to cancel the operation. public BinaryContent Serialize( object data, @@ -101,15 +101,15 @@ public BinaryContent Serialize( => SerializeInternalAsync(data, dataType, messageType, false, cancellationToken).EnsureCompleted(); /// - /// Encodes the message data as Avro and stores it in . The - /// will be set to "avro/binary+schemaId" where schemaId is the ID of the schema used to encode the data. + /// serializes the message data as Avro and stores it in . The + /// will be set to "avro/binary+schemaId" where schemaId is the ID of the schema used to serialize the data. /// - /// The data to serialize to Avro and encode into the message. - /// The type of the data to encode. If left blank, the type will be determined at runtime by + /// The data to serialize to Avro and serialize into the message. + /// The type of the data to serialize. If left blank, the type will be determined at runtime by /// calling . - /// The type of message to encode the data into. Must extend from , and + /// The type of message to serialize the data into. Must extend from , and /// have a parameterless constructor. - /// If left blank, the data will be encoded into a instance. + /// If left blank, the data will be serialized into a instance. /// An optional instance to signal the request to cancel the operation. public async ValueTask SerializeAsync( object data, @@ -168,33 +168,35 @@ internal async ValueTask SerializeInternalAsync( private async Task GetSchemaIdAsync(Schema schema, bool async, CancellationToken cancellationToken) { - if (_schemaToIdMap.TryGet(schema, out string schemaId)) + if (_schemaToIdMap.TryGet(schema, out var value)) { - return schemaId; + return value.SchemaId; } SchemaProperties schemaProperties; + string schemaString = schema.ToString(); if (async) { schemaProperties = _options.AutoRegisterSchemas ? (await _client - .RegisterSchemaAsync(_groupName, schema.Fullname, schema.ToString(), SchemaFormat.Avro, cancellationToken) + .RegisterSchemaAsync(_groupName, schema.Fullname, schemaString, SchemaFormat.Avro, cancellationToken) .ConfigureAwait(false)).Value : await _client - .GetSchemaPropertiesAsync(_groupName, schema.Fullname, schema.ToString(), SchemaFormat.Avro, cancellationToken) + .GetSchemaPropertiesAsync(_groupName, schema.Fullname, schemaString, SchemaFormat.Avro, cancellationToken) .ConfigureAwait(false); } else { schemaProperties = _options.AutoRegisterSchemas - ? _client.RegisterSchema(_groupName, schema.Fullname, schema.ToString(), SchemaFormat.Avro, cancellationToken) - : _client.GetSchemaProperties(_groupName, schema.Fullname, schema.ToString(), SchemaFormat.Avro, cancellationToken); + ? _client.RegisterSchema(_groupName, schema.Fullname, schemaString, SchemaFormat.Avro, cancellationToken) + : _client.GetSchemaProperties(_groupName, schema.Fullname, schemaString, SchemaFormat.Avro, cancellationToken); } string id = schemaProperties.Id; - _schemaToIdMap.AddOrUpdate(schema, id); - _idToSchemaMap.AddOrUpdate(id, schema); + _schemaToIdMap.AddOrUpdate(schema, (id, schemaString.Length)); + _idToSchemaMap.AddOrUpdate(id, (schema, schemaString.Length)); + SchemaRegistryAvroEventSource.Log.CacheUpdated(_idToSchemaMap, _schemaToIdMap); return id; } @@ -229,66 +231,66 @@ private static SupportedType GetSupportedTypeOrThrow(Type type) } #endregion - #region Decode + #region Deserialize /// - /// Decodes the message data into the specified type using the schema information populated in . + /// Deserializes the message data into the specified type using the schema information populated in . /// - /// The message containing the data to decode. + /// The message containing the data to deserialize. /// An optional instance to signal the request to cancel the operation. - /// The type to decode the message data into. + /// The type to deserialize the message data into. /// The deserialized data. /// Thrown if the content type is not in the expected format. - /// Thrown if an attempt is made to decode non-Avro data. + /// Thrown if an attempt is made to deserialize non-Avro data. public TData Deserialize( BinaryContent content, CancellationToken cancellationToken = default) - => (TData) DecodeMessageDataInternalAsync(content.Data, typeof(TData), content.ContentType, false, cancellationToken).EnsureCompleted(); + => (TData) DeserializeMessageDataInternalAsync(content.Data, typeof(TData), content.ContentType, false, cancellationToken).EnsureCompleted(); /// - /// Decodes the message data into the specified type using the schema information populated in . + /// deserializes the message data into the specified type using the schema information populated in . /// /// The content to deserialize. /// An optional instance to signal the request to cancel the operation. - /// The type to decode the message data into. + /// The type to deserialize the message data into. /// The deserialized data. /// Thrown if the content type is not in the expected format. - /// Thrown if an attempt is made to decode non-Avro data. + /// Thrown if an attempt is made to deserialize non-Avro data. public async ValueTask DeserializeAsync( BinaryContent content, CancellationToken cancellationToken = default) - => (TData) await DecodeMessageDataInternalAsync(content.Data, typeof(TData), content.ContentType, true, cancellationToken).ConfigureAwait(false); + => (TData) await DeserializeMessageDataInternalAsync(content.Data, typeof(TData), content.ContentType, true, cancellationToken).ConfigureAwait(false); /// - /// Decodes the message data into the specified type using the schema information populated in . + /// Deserializes the message data into the specified type using the schema information populated in . /// - /// The message containing the data to decode. - /// The type to decode the message data into. + /// The message containing the data to deserialize. + /// The type to deserialize the message data into. /// An optional instance to signal the request to cancel the operation. /// The deserialized data. /// Thrown if the content type is not in the expected format. - /// Thrown if an attempt is made to decode non-Avro data. + /// Thrown if an attempt is made to deserialize non-Avro data. public object Deserialize( BinaryContent content, Type dataType, CancellationToken cancellationToken = default) - => DecodeMessageDataInternalAsync(content.Data, dataType, content.ContentType, false, cancellationToken).EnsureCompleted(); + => DeserializeMessageDataInternalAsync(content.Data, dataType, content.ContentType, false, cancellationToken).EnsureCompleted(); /// - /// Decodes the message data into the specified type using the schema information populated in . + /// Deserializes the message data into the specified type using the schema information populated in . /// - /// The message containing the data to decode. - /// The type to decode the message data into. + /// The message containing the data to deserialize. + /// The type to deserialize the message data into. /// An optional instance to signal the request to cancel the operation. /// The deserialized data. /// Thrown if the content type is not in the expected format. - /// Thrown if an attempt is made to decode non-Avro data. + /// Thrown if an attempt is made to deserialize non-Avro data. public async ValueTask DeserializeAsync( BinaryContent content, Type dataType, CancellationToken cancellationToken = default) - => await DecodeMessageDataInternalAsync(content.Data, dataType, content.ContentType, true, cancellationToken).ConfigureAwait(false); + => await DeserializeMessageDataInternalAsync(content.Data, dataType, content.ContentType, true, cancellationToken).ConfigureAwait(false); - private async ValueTask DecodeMessageDataInternalAsync( + private async ValueTask DeserializeMessageDataInternalAsync( BinaryData data, Type dataType, ContentType? contentType, @@ -322,7 +324,7 @@ private async ValueTask DecodeMessageDataInternalAsync( if (contentTypeArray[0] != AvroMimeType) { - throw new InvalidOperationException("An avro encoder may only be used on content that is of 'avro/binary' type"); + throw new InvalidOperationException("An avro serializer may only be used on content that is of 'avro/binary' type"); } schemaId = contentTypeArray[1]; @@ -330,14 +332,14 @@ private async ValueTask DecodeMessageDataInternalAsync( if (async) { - return await DecodeInternalAsync(data, dataType, schemaId, true, cancellationToken).ConfigureAwait(false); } + return await DeserializeInternalAsync(data, dataType, schemaId, true, cancellationToken).ConfigureAwait(false); } else { - return DecodeInternalAsync(data, dataType, schemaId, false, cancellationToken).EnsureCompleted(); + return DeserializeInternalAsync(data, dataType, schemaId, false, cancellationToken).EnsureCompleted(); } } - private async ValueTask DecodeInternalAsync( + private async ValueTask DeserializeInternalAsync( BinaryData data, Type dataType, string schemaId, @@ -375,9 +377,9 @@ private async ValueTask DecodeInternalAsync( private async Task GetSchemaByIdAsync(string schemaId, bool async, CancellationToken cancellationToken) { - if (_idToSchemaMap.TryGet(schemaId, out Schema cachedSchema)) + if (_idToSchemaMap.TryGet(schemaId, out var cachedSchema)) { - return cachedSchema; + return cachedSchema.Schema; } string schemaDefinition; @@ -390,8 +392,9 @@ private async Task GetSchemaByIdAsync(string schemaId, bool async, Cance schemaDefinition = _client.GetSchema(schemaId, cancellationToken).Value.Definition; } var schema = Schema.Parse(schemaDefinition); - _idToSchemaMap.AddOrUpdate(schemaId, schema); - _schemaToIdMap.AddOrUpdate(schema, schemaId); + _idToSchemaMap.AddOrUpdate(schemaId, (schema, schemaDefinition.Length)); + _schemaToIdMap.AddOrUpdate(schema, (schemaId, schemaDefinition.Length)); + SchemaRegistryAvroEventSource.Log.CacheUpdated(_idToSchemaMap, _schemaToIdMap); return schema; } diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/EventSourceLiveTests.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/EventSourceLiveTests.cs new file mode 100644 index 0000000000000..bc90cbae3395e --- /dev/null +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/EventSourceLiveTests.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Diagnostics.Tracing; +using System.Linq; +using System.Threading.Tasks; +using Azure.Core.Diagnostics; +using Azure.Core.TestFramework; +using Azure.Messaging.EventHubs; +using NUnit.Framework; +using TestSchema; + +namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.Tests +{ + [NonParallelizable] + public class EventSourceLiveTests : SchemaRegistryAvroObjectSerializerLiveTestBase + { + private TestEventListener _listener; + + public EventSourceLiveTests(bool isAsync) : base(isAsync) + { + } + + [SetUp] + public void Setup() + { + _listener = new TestEventListener(); + _listener.EnableEvents(SchemaRegistryAvroEventSource.Log, EventLevel.Verbose); + } + + [TearDown] + public void TearDown() + { + _listener.Dispose(); + } + + [RecordedTest] + public async Task UpdatingCacheLogsEvents() + { + using var logger = AzureEventSourceListener.CreateConsoleLogger(EventLevel.Verbose); + var client = CreateClient(); + var groupName = TestEnvironment.SchemaRegistryGroup; + + var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions { AutoRegisterSchemas = true }); + + var employee = new Employee { Age = 42, Name = "Caketown" }; + EventData eventData = await serializer.SerializeAsync(employee); + + Assert.IsFalse(eventData.IsReadOnly); + string[] contentType = eventData.ContentType.Split('+'); + Assert.AreEqual(2, contentType.Length); + Assert.AreEqual("avro/binary", contentType[0]); + Assert.IsNotEmpty(contentType[1]); + + Employee deserialized = await serializer.DeserializeAsync(eventData); + + // decoding should not alter the message + contentType = eventData.ContentType.Split('+'); + Assert.AreEqual(2, contentType.Length); + Assert.AreEqual("avro/binary", contentType[0]); + Assert.IsNotEmpty(contentType[1]); + + // verify the payload was decoded correctly + Assert.IsNotNull(deserialized); + Assert.AreEqual("Caketown", deserialized.Name); + Assert.AreEqual(42, deserialized.Age); + + // Use different schema so we can see the cache updated + eventData = await serializer.SerializeAsync(new Employee_V2{ Age = 42, Name = "Caketown", City = "Redmond"}); + + Assert.IsFalse(eventData.IsReadOnly); + eventData.ContentType.Split('+'); + Assert.AreEqual(2, contentType.Length); + Assert.AreEqual("avro/binary", contentType[0]); + Assert.IsNotEmpty(contentType[1]); + + await serializer.DeserializeAsync(eventData); + + // decoding should not alter the message + contentType = eventData.ContentType.Split('+'); + Assert.AreEqual(2, contentType.Length); + Assert.AreEqual("avro/binary", contentType[0]); + Assert.IsNotEmpty(contentType[1]); + + // verify the payload was decoded correctly + Assert.IsNotNull(deserialized); + Assert.AreEqual("Caketown", deserialized.Name); + Assert.AreEqual(42, deserialized.Age); + + var events = _listener.EventsById(SchemaRegistryAvroEventSource.CacheUpdatedEvent).ToArray(); + Assert.AreEqual(2, events.Length); + + // first log entry should have 2 as the total number of entries as we maintain two caches for each schema + Assert.AreEqual(2, events[0].Payload[0]); + // the second payload element is the total schema length + Assert.AreEqual(334, events[0].Payload[1]); + + // second entry will include both V1 and V2 schemas - so 4 total entries + Assert.AreEqual(4, events[1].Payload[0]); + Assert.AreEqual(732, events[1].Payload[1]); + } + } +} \ No newline at end of file diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SchemaRegistryAvroObjectSerializerLiveTestBase.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SchemaRegistryAvroObjectSerializerLiveTestBase.cs new file mode 100644 index 0000000000000..41efae67e98e5 --- /dev/null +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SchemaRegistryAvroObjectSerializerLiveTestBase.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Core.TestFramework; +using Azure.Data.SchemaRegistry; + +namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.Tests +{ + public class SchemaRegistryAvroObjectSerializerLiveTestBase : RecordedTestBase + { + public SchemaRegistryAvroObjectSerializerLiveTestBase(bool isAsync) : base(isAsync) + { + TestDiagnostics = false; + } + + protected SchemaRegistryClient CreateClient() => + InstrumentClient(new SchemaRegistryClient( + TestEnvironment.SchemaRegistryEndpoint, + TestEnvironment.Credential, + InstrumentClientOptions(new SchemaRegistryClientOptions()) + )); + } +} \ No newline at end of file diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SchemaRegistryAvroObjectSerializerLiveTests.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SchemaRegistryAvroObjectSerializerLiveTests.cs index 856201943ef8c..0dea367f7037e 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SchemaRegistryAvroObjectSerializerLiveTests.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SchemaRegistryAvroObjectSerializerLiveTests.cs @@ -18,20 +18,12 @@ namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.Tests { - public class SchemaRegistryAvroObjectSerializerLiveTests : RecordedTestBase + public class SchemaRegistryAvroObjectSerializerLiveTests : SchemaRegistryAvroObjectSerializerLiveTestBase { public SchemaRegistryAvroObjectSerializerLiveTests(bool isAsync) : base(isAsync) { - TestDiagnostics = false; } - private SchemaRegistryClient CreateClient() => - InstrumentClient(new SchemaRegistryClient( - TestEnvironment.SchemaRegistryEndpoint, - TestEnvironment.Credential, - InstrumentClientOptions(new SchemaRegistryClientOptions()) - )); - [RecordedTest] public async Task CanSerializeAndDeserialize() { @@ -224,7 +216,7 @@ public async Task CanUseEncoderWithEventDataUsingGenerics() #endif #endregion - Assert.IsFalse(((BinaryContent) eventData).IsReadOnly); + Assert.IsFalse(eventData.IsReadOnly); string[] contentType = eventData.ContentType.Split('+'); Assert.AreEqual(2, contentType.Length); Assert.AreEqual("avro/binary", contentType[0]); diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SessionRecords/EventSourceLiveTests/UpdatingCacheLogsEvents.json b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SessionRecords/EventSourceLiveTests/UpdatingCacheLogsEvents.json new file mode 100644 index 0000000000000..7f44237ffc3aa --- /dev/null +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SessionRecords/EventSourceLiveTests/UpdatingCacheLogsEvents.json @@ -0,0 +1,105 @@ +{ + "Entries": [ + { + "RequestUri": "https://jolovschemaregistry.servicebus.windows.net/$schemaGroups/azsdk_net_test_group/schemas/TestSchema.Employee?api-version=2021-10", + "RequestMethod": "PUT", + "RequestHeaders": { + "Accept": "application/json", + "Authorization": "Sanitized", + "Content-Length": "167", + "Content-Type": "application/json; serialization=Avro", + "User-Agent": "azsdk-net-Data.SchemaRegistry/1.1.0 (.NET 6.0.0-rtm.21522.10; Microsoft Windows 10.0.22000)", + "x-ms-client-request-id": "8b7b6a8ca8f177dc5b99d5b4a234eeb7", + "x-ms-return-client-request-id": "true" + }, + "RequestBody": { + "type": "record", + "name": "Employee", + "namespace": "TestSchema", + "aliases": [ + "TestSchema.EmployeeV2" + ], + "fields": [ + { + "name": "Name", + "type": "string" + }, + { + "name": "Age", + "type": "int" + } + ] + }, + "StatusCode": 204, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Wed, 02 Mar 2022 01:37:00 GMT", + "Location": "https://jolovschemaregistry.servicebus.windows.net/$schemagroups/azsdk_net_test_group/schemas/TestSchema.Employee/versions/1?api-version=2021-10", + "Schema-Group-Name": "azsdk_net_test_group", + "Schema-Id": "a3db73ebc56140e5954ef667d154720f", + "Schema-Id-Location": "https://jolovschemaregistry.servicebus.windows.net:443/$schemagroups/$schemas/a3db73ebc56140e5954ef667d154720f?api-version=2021-10", + "Schema-Name": "TestSchema.Employee", + "Schema-Version": "1", + "Schema-Versions-Location": "https://jolovschemaregistry.servicebus.windows.net:443/$schemagroups/azsdk_net_test_group/schemas/TestSchema.Employee/versions?api-version=2021-10", + "Server": "Microsoft-HTTPAPI/2.0", + "Strict-Transport-Security": "max-age=31536000" + }, + "ResponseBody": null + }, + { + "RequestUri": "https://jolovschemaregistry.servicebus.windows.net/$schemaGroups/azsdk_net_test_group/schemas/TestSchema.EmployeeV2?api-version=2021-10", + "RequestMethod": "PUT", + "RequestHeaders": { + "Accept": "application/json", + "Authorization": "Sanitized", + "Content-Length": "199", + "Content-Type": "application/json; serialization=Avro", + "User-Agent": "azsdk-net-Data.SchemaRegistry/1.1.0 (.NET 6.0.0-rtm.21522.10; Microsoft Windows 10.0.22000)", + "x-ms-client-request-id": "9e2acd7172518dd5a6a55257c3123f08", + "x-ms-return-client-request-id": "true" + }, + "RequestBody": { + "type": "record", + "name": "EmployeeV2", + "namespace": "TestSchema", + "aliases": [ + "TestSchema.Employee" + ], + "fields": [ + { + "name": "Name", + "type": "string" + }, + { + "name": "Age", + "type": "int" + }, + { + "name": "City", + "type": "string" + } + ] + }, + "StatusCode": 204, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Wed, 02 Mar 2022 01:37:01 GMT", + "Location": "https://jolovschemaregistry.servicebus.windows.net/$schemagroups/azsdk_net_test_group/schemas/TestSchema.EmployeeV2/versions/1?api-version=2021-10", + "Schema-Group-Name": "azsdk_net_test_group", + "Schema-Id": "eaf6015ee54f477d9fb00916438c5131", + "Schema-Id-Location": "https://jolovschemaregistry.servicebus.windows.net:443/$schemagroups/$schemas/eaf6015ee54f477d9fb00916438c5131?api-version=2021-10", + "Schema-Name": "TestSchema.EmployeeV2", + "Schema-Version": "1", + "Schema-Versions-Location": "https://jolovschemaregistry.servicebus.windows.net:443/$schemagroups/azsdk_net_test_group/schemas/TestSchema.EmployeeV2/versions?api-version=2021-10", + "Server": "Microsoft-HTTPAPI/2.0", + "Strict-Transport-Security": "max-age=31536000" + }, + "ResponseBody": null + } + ], + "Variables": { + "RandomSeed": "882402998", + "SCHEMAREGISTRY_ENDPOINT": "jolovschemaregistry.servicebus.windows.net", + "SCHEMAREGISTRY_GROUP": "azsdk_net_test_group" + } +} diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SessionRecords/EventSourceLiveTests/UpdatingCacheLogsEventsAsync.json b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SessionRecords/EventSourceLiveTests/UpdatingCacheLogsEventsAsync.json new file mode 100644 index 0000000000000..5b508b2998d79 --- /dev/null +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/SessionRecords/EventSourceLiveTests/UpdatingCacheLogsEventsAsync.json @@ -0,0 +1,105 @@ +{ + "Entries": [ + { + "RequestUri": "https://jolovschemaregistry.servicebus.windows.net/$schemaGroups/azsdk_net_test_group/schemas/TestSchema.Employee?api-version=2021-10", + "RequestMethod": "PUT", + "RequestHeaders": { + "Accept": "application/json", + "Authorization": "Sanitized", + "Content-Length": "167", + "Content-Type": "application/json; serialization=Avro", + "User-Agent": "azsdk-net-Data.SchemaRegistry/1.1.0 (.NET 6.0.0-rtm.21522.10; Microsoft Windows 10.0.22000)", + "x-ms-client-request-id": "348acdae25edc05e660c9bf037b8e09f", + "x-ms-return-client-request-id": "true" + }, + "RequestBody": { + "type": "record", + "name": "Employee", + "namespace": "TestSchema", + "aliases": [ + "TestSchema.EmployeeV2" + ], + "fields": [ + { + "name": "Name", + "type": "string" + }, + { + "name": "Age", + "type": "int" + } + ] + }, + "StatusCode": 204, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Wed, 02 Mar 2022 01:37:01 GMT", + "Location": "https://jolovschemaregistry.servicebus.windows.net/$schemagroups/azsdk_net_test_group/schemas/TestSchema.Employee/versions/1?api-version=2021-10", + "Schema-Group-Name": "azsdk_net_test_group", + "Schema-Id": "a3db73ebc56140e5954ef667d154720f", + "Schema-Id-Location": "https://jolovschemaregistry.servicebus.windows.net:443/$schemagroups/$schemas/a3db73ebc56140e5954ef667d154720f?api-version=2021-10", + "Schema-Name": "TestSchema.Employee", + "Schema-Version": "1", + "Schema-Versions-Location": "https://jolovschemaregistry.servicebus.windows.net:443/$schemagroups/azsdk_net_test_group/schemas/TestSchema.Employee/versions?api-version=2021-10", + "Server": "Microsoft-HTTPAPI/2.0", + "Strict-Transport-Security": "max-age=31536000" + }, + "ResponseBody": null + }, + { + "RequestUri": "https://jolovschemaregistry.servicebus.windows.net/$schemaGroups/azsdk_net_test_group/schemas/TestSchema.EmployeeV2?api-version=2021-10", + "RequestMethod": "PUT", + "RequestHeaders": { + "Accept": "application/json", + "Authorization": "Sanitized", + "Content-Length": "199", + "Content-Type": "application/json; serialization=Avro", + "User-Agent": "azsdk-net-Data.SchemaRegistry/1.1.0 (.NET 6.0.0-rtm.21522.10; Microsoft Windows 10.0.22000)", + "x-ms-client-request-id": "1c30f86c2076d9c19b2a1039d5810ed4", + "x-ms-return-client-request-id": "true" + }, + "RequestBody": { + "type": "record", + "name": "EmployeeV2", + "namespace": "TestSchema", + "aliases": [ + "TestSchema.Employee" + ], + "fields": [ + { + "name": "Name", + "type": "string" + }, + { + "name": "Age", + "type": "int" + }, + { + "name": "City", + "type": "string" + } + ] + }, + "StatusCode": 204, + "ResponseHeaders": { + "Content-Length": "0", + "Date": "Wed, 02 Mar 2022 01:37:03 GMT", + "Location": "https://jolovschemaregistry.servicebus.windows.net/$schemagroups/azsdk_net_test_group/schemas/TestSchema.EmployeeV2/versions/1?api-version=2021-10", + "Schema-Group-Name": "azsdk_net_test_group", + "Schema-Id": "eaf6015ee54f477d9fb00916438c5131", + "Schema-Id-Location": "https://jolovschemaregistry.servicebus.windows.net:443/$schemagroups/$schemas/eaf6015ee54f477d9fb00916438c5131?api-version=2021-10", + "Schema-Name": "TestSchema.EmployeeV2", + "Schema-Version": "1", + "Schema-Versions-Location": "https://jolovschemaregistry.servicebus.windows.net:443/$schemagroups/azsdk_net_test_group/schemas/TestSchema.EmployeeV2/versions?api-version=2021-10", + "Server": "Microsoft-HTTPAPI/2.0", + "Strict-Transport-Security": "max-age=31536000" + }, + "ResponseBody": null + } + ], + "Variables": { + "RandomSeed": "398677715", + "SCHEMAREGISTRY_ENDPOINT": "jolovschemaregistry.servicebus.windows.net", + "SCHEMAREGISTRY_GROUP": "azsdk_net_test_group" + } +} From c73e417e9321477174a206f9236888fd2c44d7a9 Mon Sep 17 00:00:00 2001 From: JoshLove-msft <54595583+JoshLove-msft@users.noreply.github.com> Date: Tue, 1 Mar 2022 17:45:22 -0800 Subject: [PATCH 2/4] Remove unused method --- .../src/LruCache.cs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs index 29c6fdd161821..6e35fc2d49a76 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs @@ -70,18 +70,6 @@ public void AddOrUpdate(TKey key, TValue val) } } - internal int ComputeSize() - { - int size = 0; - foreach (KeyValuePair kvp in _linkedList) - { - size += kvp.Key.ToString().Length * sizeof(char); - size += kvp.Value.ToString().Length * sizeof(char); - } - - return size; - } - public IEnumerator> GetEnumerator() => _linkedList.GetEnumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); From e0c15149a711e020833edd52560b0e159835622d Mon Sep 17 00:00:00 2001 From: JoshLove-msft <54595583+JoshLove-msft@users.noreply.github.com> Date: Tue, 1 Mar 2022 21:50:00 -0800 Subject: [PATCH 3/4] Fix --- .../src/LruCache.cs | 2 +- .../src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs index 6e35fc2d49a76..aa0ab67ea8f91 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs @@ -14,7 +14,7 @@ namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro internal class LruCache : IEnumerable> { private readonly int _capacity; - internal readonly LinkedList> _linkedList; + private readonly LinkedList> _linkedList; private readonly Dictionary>> _map; private readonly object _syncLock; diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj index 6d1b41dae5907..770eead949121 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.csproj @@ -16,7 +16,7 @@ - + From 8dadfe6264e6820a04342abcf6899e5089441d5f Mon Sep 17 00:00:00 2001 From: JoshLove-msft <54595583+JoshLove-msft@users.noreply.github.com> Date: Wed, 2 Mar 2022 12:17:56 -0800 Subject: [PATCH 4/4] Cache total length --- .../src/LruCache.cs | 21 ++++++++++++------- .../src/SchemaRegistryAvroEventSource.cs | 8 ++----- .../src/SchemaRegistryAvroSerializer.cs | 16 +++++++------- .../tests/LruCacheTests.cs | 20 +++++++++++------- 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs index aa0ab67ea8f91..79bf051728093 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/LruCache.cs @@ -15,16 +15,18 @@ internal class LruCache : IEnumerable> { private readonly int _capacity; private readonly LinkedList> _linkedList; - private readonly Dictionary>> _map; + private readonly Dictionary> Node, int Length)> _map; private readonly object _syncLock; internal int Count => _linkedList.Count; + internal int TotalLength { get; private set; } + public LruCache(int capacity) { _capacity = capacity; _linkedList = new LinkedList>(); - _map = new Dictionary>>(); + _map = new Dictionary>, int)>(); _syncLock = new object(); } @@ -32,8 +34,9 @@ public bool TryGet(TKey key, out TValue value) { lock (_syncLock) { - if (_map.TryGetValue(key, out var node)) + if (_map.TryGetValue(key, out var mapValue)) { + var node = mapValue.Node; value = node.Value.Value; _linkedList.Remove(node); _linkedList.AddFirst(node); @@ -45,27 +48,31 @@ public bool TryGet(TKey key, out TValue value) } } - public void AddOrUpdate(TKey key, TValue val) + public void AddOrUpdate(TKey key, TValue val, int length) { lock (_syncLock) { - if (_map.TryGetValue(key, out var existingNode)) + if (_map.TryGetValue(key, out var existingValue)) { // remove node - we will re-add a new node for this key at the head of the list, as the value may be different - _linkedList.Remove(existingNode); + _linkedList.Remove(existingValue.Node); + TotalLength -= _map[key].Length; } // add new node var node = new LinkedListNode>(new KeyValuePair(key, val)); _linkedList.AddFirst(node); - _map[key] = node; + _map[key] = (node, length); + TotalLength += length; if (_map.Count > _capacity) { // remove least recently used node LinkedListNode> last = _linkedList.Last; _linkedList.RemoveLast(); + var toRemove = _map[last.Value.Key]; _map.Remove(last.Value.Key); + TotalLength -= toRemove.Length; } } } diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroEventSource.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroEventSource.cs index f265af48315c5..4138f3254e86b 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroEventSource.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroEventSource.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using System.Collections.Generic; using System.Diagnostics.Tracing; -using System.Linq; using Avro; using Azure.Core.Diagnostics; @@ -42,13 +40,11 @@ protected SchemaRegistryAvroEventSource() : base(EventSourceName) } [NonEvent] - public virtual void CacheUpdated(LruCache idToSchemaCache, LruCache schemaToIdCache) + public virtual void CacheUpdated(LruCache idToSchemaCache, LruCache schemaToIdCache) { if (IsEnabled()) { - int totalSchemaLength = idToSchemaCache.Sum(kvp => kvp.Value.SchemaLength); - totalSchemaLength += schemaToIdCache.Sum(kvp => kvp.Value.SchemaLength); - CacheUpdatedCore(idToSchemaCache.Count + schemaToIdCache.Count, totalSchemaLength); + CacheUpdatedCore(idToSchemaCache.Count + schemaToIdCache.Count, idToSchemaCache.TotalLength + schemaToIdCache.TotalLength); } } diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroSerializer.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroSerializer.cs index 50dd3692e4f40..5970826dd525c 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroSerializer.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/src/SchemaRegistryAvroSerializer.cs @@ -45,8 +45,8 @@ public SchemaRegistryAvroSerializer(SchemaRegistryClient client, string groupNam private const int RecordFormatIndicatorLength = 4; private const int SchemaIdLength = 32; private const int PayloadStartPosition = RecordFormatIndicatorLength + SchemaIdLength; - private readonly LruCache _idToSchemaMap = new(CacheCapacity); - private readonly LruCache _schemaToIdMap = new(CacheCapacity); + private readonly LruCache _idToSchemaMap = new(CacheCapacity); + private readonly LruCache _schemaToIdMap = new(CacheCapacity); private enum SupportedType { @@ -170,7 +170,7 @@ private async Task GetSchemaIdAsync(Schema schema, bool async, Cancellat { if (_schemaToIdMap.TryGet(schema, out var value)) { - return value.SchemaId; + return value; } SchemaProperties schemaProperties; @@ -194,8 +194,8 @@ private async Task GetSchemaIdAsync(Schema schema, bool async, Cancellat string id = schemaProperties.Id; - _schemaToIdMap.AddOrUpdate(schema, (id, schemaString.Length)); - _idToSchemaMap.AddOrUpdate(id, (schema, schemaString.Length)); + _schemaToIdMap.AddOrUpdate(schema, id, schemaString.Length); + _idToSchemaMap.AddOrUpdate(id, schema, schemaString.Length); SchemaRegistryAvroEventSource.Log.CacheUpdated(_idToSchemaMap, _schemaToIdMap); return id; } @@ -379,7 +379,7 @@ private async Task GetSchemaByIdAsync(string schemaId, bool async, Cance { if (_idToSchemaMap.TryGet(schemaId, out var cachedSchema)) { - return cachedSchema.Schema; + return cachedSchema; } string schemaDefinition; @@ -392,8 +392,8 @@ private async Task GetSchemaByIdAsync(string schemaId, bool async, Cance schemaDefinition = _client.GetSchema(schemaId, cancellationToken).Value.Definition; } var schema = Schema.Parse(schemaDefinition); - _idToSchemaMap.AddOrUpdate(schemaId, (schema, schemaDefinition.Length)); - _schemaToIdMap.AddOrUpdate(schema, (schemaId, schemaDefinition.Length)); + _idToSchemaMap.AddOrUpdate(schemaId, schema, schemaDefinition.Length); + _schemaToIdMap.AddOrUpdate(schema, schemaId, schemaDefinition.Length); SchemaRegistryAvroEventSource.Log.CacheUpdated(_idToSchemaMap, _schemaToIdMap); return schema; } diff --git a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/LruCacheTests.cs b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/LruCacheTests.cs index 19fdcf9f6df19..da73dbafd20d5 100644 --- a/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/LruCacheTests.cs +++ b/sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/LruCacheTests.cs @@ -13,8 +13,10 @@ public void CacheCapacityRespected() var cache = new LruCache(10); for (int i = 0; i < 20; i++) { - cache.AddOrUpdate(i.ToString(), i); + cache.AddOrUpdate(i.ToString(), i, i.ToString().Length); } + Assert.AreEqual(10, cache.Count); + Assert.AreEqual(20, cache.TotalLength); for (int i = 0; i < 10; i++) { @@ -33,17 +35,17 @@ public void CacheEvictsLeastRecentlyAccessedItemFirst() { var cache = new LruCache(3); - cache.AddOrUpdate("1", 1); - cache.AddOrUpdate("2", 2); - cache.AddOrUpdate("3", 3); + cache.AddOrUpdate("1", 1, 1); + cache.AddOrUpdate("2", 2, 1); + cache.AddOrUpdate("3", 3, 1); // 1 is moved to head of list Assert.IsTrue(cache.TryGet("1", out _)); // 4 is added to head of list, which evicts 2, the least recently used item - cache.AddOrUpdate("4", 4); + cache.AddOrUpdate("4", 4, 1); // 2 should be evicted Assert.IsFalse(cache.TryGet("2", out _)); // 5 is moved to head of list - cache.AddOrUpdate("5", 4); + cache.AddOrUpdate("5", 4, 1); // 3 should be evicted Assert.IsFalse(cache.TryGet("3", out _)); } @@ -53,10 +55,12 @@ public void CanUpdateExistingValue() { var cache = new LruCache(10); - cache.AddOrUpdate("1", 1); + cache.AddOrUpdate("1", 1, 1); cache.TryGet("1", out int val); Assert.AreEqual(1, val); - cache.AddOrUpdate("1", 10); + Assert.AreEqual(1, cache.TotalLength); + cache.AddOrUpdate("1", 10, 2); + Assert.AreEqual(2, cache.TotalLength); cache.TryGet("1", out val); Assert.AreEqual(10, val); }