From a01d4c47e57a3e078ed23c278e6c01d5826182f9 Mon Sep 17 00:00:00 2001 From: AdrianStrugala Date: Sun, 7 Apr 2024 13:07:18 +0200 Subject: [PATCH 1/2] fix + draft of test engine --- .../Features/AvroToJson/Decoder.cs | 38 ++++++------ tests/AvroConvertTests/TestEngine.cs | 58 +++++++++++++++++++ 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/src/AvroConvert/Features/AvroToJson/Decoder.cs b/src/AvroConvert/Features/AvroToJson/Decoder.cs index fa1c70b8..32d5752e 100644 --- a/src/AvroConvert/Features/AvroToJson/Decoder.cs +++ b/src/AvroConvert/Features/AvroToJson/Decoder.cs @@ -27,6 +27,7 @@ using SolTechnology.Avro.AvroObjectServices.FileHeader; using SolTechnology.Avro.AvroObjectServices.FileHeader.Codec; using SolTechnology.Avro.AvroObjectServices.Read; +using SolTechnology.Avro.AvroObjectServices.Schemas; using SolTechnology.Avro.AvroObjectServices.Schemas.Abstract; using SolTechnology.Avro.Infrastructure.Exceptions; @@ -65,7 +66,7 @@ internal object Decode(Stream stream, TypeSchema schema) { var header = reader.ReadHeader(); - schema = schema ?? Schema.Create(header.GetMetadata(DataFileConstants.SchemaMetadataKey)); + schema ??= Schema.Create(header.GetMetadata(DataFileConstants.SchemaMetadataKey)); var resolver = new Resolver(schema); reader.ReadFixed(header.SyncData); @@ -83,30 +84,33 @@ internal object Read(Reader reader, Header header, AbstractCodec codec, Resolver return string.Empty; } - var result = new List(); do { - long itemsCount = reader.ReadLong(); - var data = reader.ReadDataBlock(header.SyncData, codec); + result.AddRange(ReadSingleBlock(reader, header, codec, resolver)); + } while (!reader.IsReadToEnd()); - reader = new Reader(new MemoryStream(data)); + if (header.Schema is not ArraySchema && result.Count < 2) + { + return result.SingleOrDefault(); + } - if (itemsCount > 1) - { - for (int i = 0; i < itemsCount; i++) - { - result.Add(resolver.Resolve(reader)); - } - } - else - { - return resolver.Resolve(reader); - } + return result; + } - } while (!reader.IsReadToEnd()); + private static List ReadSingleBlock(Reader reader, Header header, AbstractCodec codec, Resolver resolver) + { + var result = new List(); + long itemsCount = reader.ReadLong(); + var data = reader.ReadDataBlock(header.SyncData, codec); + + reader = new Reader(new MemoryStream(data)); + for (int i = 0; i < itemsCount; i++) + { + result.Add(resolver.Resolve(reader)); + } return result; } diff --git a/tests/AvroConvertTests/TestEngine.cs b/tests/AvroConvertTests/TestEngine.cs index 4c7b4aed..2223a575 100644 --- a/tests/AvroConvertTests/TestEngine.cs +++ b/tests/AvroConvertTests/TestEngine.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.IO; +using Newtonsoft.Json; using SolTechnology.Avro; namespace AvroConvertComponentTests; @@ -21,6 +23,8 @@ public static IEnumerable All() yield return Deflate; yield return Gzip; + + yield return Json; } public static IEnumerable Core() @@ -242,4 +246,58 @@ private static object[] BrotliWithSchema return new object[] { @default }; } } + + private static object[] DefaultDeserializeOnly + { + get + { + var func = new Func((input, type) => AvroConvert.Deserialize(input, type)); + + return new object[] { func }; + } + } + + private static object[] GenericJsonDeserializeOnly + { + get + { + var func = new Func((input, type) => + { + var json= AvroConvert.Avro2Json(input); + return JsonConvert.DeserializeObject(json, type); + }); + + return new object[] { func }; + } + } + + private static object[] DeserializeByLine + { + get + { + var func = new Func((input, type) => + { + var result = new List(); + + object openDeserializer = typeof(AvroConvert) + .GetMethod("OpenDeserializer", new[] { typeof(Stream) }) + ?.MakeGenericMethod(type); + // .Invoke(null, new object[] { new MemoryStream(input) }); + + using (var reader = AvroConvert.OpenDeserializer(new MemoryStream(input))) + { + while (reader.HasNext()) + { + var item = reader.ReadNext(); + + result.Add(item); + } + } + + return result; + }); + + return new object[] { func }; + } + } } \ No newline at end of file From 4687fb967194c1ff9a3fd2ea94238313dd450c69 Mon Sep 17 00:00:00 2001 From: AdrianStrugala Date: Mon, 8 Apr 2024 11:59:44 +0200 Subject: [PATCH 2/2] added test for the case --- src/AvroConvert/AvroConvert.Deserialize.cs | 17 ++++-- .../Write/Resolvers/Array.cs | 6 +- .../Default/DeserializeFilesTests.cs | 60 ++++++++++--------- tests/AvroConvertTests/TestEngine.cs | 50 +++++----------- 4 files changed, 64 insertions(+), 69 deletions(-) diff --git a/src/AvroConvert/AvroConvert.Deserialize.cs b/src/AvroConvert/AvroConvert.Deserialize.cs index fa36fc80..fafdf74e 100644 --- a/src/AvroConvert/AvroConvert.Deserialize.cs +++ b/src/AvroConvert/AvroConvert.Deserialize.cs @@ -22,6 +22,7 @@ using System; using System.IO; +using System.Reflection; using SolTechnology.Avro.AvroObjectServices.BuildSchema; using SolTechnology.Avro.Features.Deserialize; @@ -87,10 +88,18 @@ public static T Deserialize(byte[] avroBytes, AvroConvertOptions options) /// public static dynamic Deserialize(byte[] avroBytes, Type targetType) { - object result = typeof(AvroConvert) - .GetMethod(nameof(Deserialize), new[] { typeof(byte[]) }) - ?.MakeGenericMethod(targetType) - .Invoke(null, new object[] { avroBytes }); + object result; + try + { + result = typeof(AvroConvert) + .GetMethod(nameof(Deserialize), new[] { typeof(byte[]) }) + ?.MakeGenericMethod(targetType) + .Invoke(null, new object[] { avroBytes }); + } + catch (TargetInvocationException ex) + { + throw ex.InnerException!; + } return result; } diff --git a/src/AvroConvert/AvroObjectServices/Write/Resolvers/Array.cs b/src/AvroConvert/AvroObjectServices/Write/Resolvers/Array.cs index 71eddbbb..8f3fb5f8 100644 --- a/src/AvroConvert/AvroObjectServices/Write/Resolvers/Array.cs +++ b/src/AvroConvert/AvroObjectServices/Write/Resolvers/Array.cs @@ -36,10 +36,10 @@ private void WriteArray(Encoder.WriteItem itemWriter, object @object, IWriter en { List list = EnsureArrayObject(@object); - long l = list?.Count ?? 0; + long count = list?.Count ?? 0; encoder.WriteArrayStart(); - encoder.WriteItemCount(l); - WriteArrayValues(list, itemWriter, encoder, l); + encoder.WriteItemCount(count); + WriteArrayValues(list, itemWriter, encoder, count); encoder.WriteArrayEnd(); } diff --git a/tests/AvroConvertTests/FilesDeserialization/Default/DeserializeFilesTests.cs b/tests/AvroConvertTests/FilesDeserialization/Default/DeserializeFilesTests.cs index 49c82fb8..0dc1efa1 100644 --- a/tests/AvroConvertTests/FilesDeserialization/Default/DeserializeFilesTests.cs +++ b/tests/AvroConvertTests/FilesDeserialization/Default/DeserializeFilesTests.cs @@ -14,6 +14,39 @@ public class DeserializeFilesTests private readonly byte[] _schemaOnlyAvroBytes = File.ReadAllBytes("header_only.avro"); + [Theory] + [MemberData(nameof(TestEngine.DeserializeOnly), MemberType = typeof(TestEngine))] + public void Deserialize_FileContainsNoAvroData_NoExceptionIsThrown(Func engine) + { + //Arrange + + + //Act + var result = engine.Invoke(_schemaOnlyAvroBytes, typeof(List)); + + + //Assert + Assert.Equal(null, result); + } + + + [Theory] + [MemberData(nameof(TestEngine.DeserializeOnly), MemberType = typeof(TestEngine))] + [Trait("Fix", "https://github.com/AdrianStrugala/AvroConvert/issues/152")] + public void Deserialize_FileWithMultipleBlocks_EveryItemIsRead(Func engine) + { + //Arrange + + + //Act + var result = (List)engine.Invoke(File.ReadAllBytes("userdata1.avro"), typeof(List)); + + + //Assert + result.Should().HaveCount(1000); + } + + [Fact] public void Deserialize_CustomSchema_OnlyValuesFromCustomSchemaAreReturned() { @@ -64,19 +97,6 @@ public void Deserialize_NonGenericMethod_OnlyValuesFromCustomSchemaAreReturned() Assert.Equal(expectedResult, result); } - [Fact] - public void Deserialize_FileContainsNoAvroData_NoExceptionIsThrown() - { - //Arrange - - //Act - var result = AvroConvert.Deserialize(_schemaOnlyAvroBytes, typeof(List)); - - - //Assert - Assert.Equal(null, result); - } - [Fact] public void Deserialize_InvalidFile_InvalidAvroObjectExceptionIsThrown() { @@ -92,20 +112,6 @@ public void Deserialize_InvalidFile_InvalidAvroObjectExceptionIsThrown() Assert.IsType(result); } - [Fact] - public void Deserialize_FileWithMultipleBlocks_EveryItemIsRead() - { - //Arrange - - - //Act - var result = AvroConvert.Deserialize>(File.ReadAllBytes("userdata1.avro")); - - - //Assert - result.Should().HaveCount(1000); - } - [Fact] public void Deserialization_with_ReadOnlySpan_Works() { diff --git a/tests/AvroConvertTests/TestEngine.cs b/tests/AvroConvertTests/TestEngine.cs index 2223a575..ce7c46a0 100644 --- a/tests/AvroConvertTests/TestEngine.cs +++ b/tests/AvroConvertTests/TestEngine.cs @@ -15,13 +15,13 @@ public static IEnumerable All() yield return Headless; yield return GenericJson; - + yield return Brotli; - + yield return Snappy; - + yield return Deflate; - + yield return Gzip; yield return Json; @@ -59,6 +59,16 @@ public static IEnumerable DefaultOnly() yield return Snappy; } + public static IEnumerable DeserializeOnly() + { + yield return DefaultDeserializeOnly; + + // TODO: need to create proper test strategy for this feature + // yield return DeserializeByLine; + + yield return GenericJsonDeserializeOnly; + } + private static object[] Default { get @@ -263,41 +273,11 @@ private static object[] GenericJsonDeserializeOnly { var func = new Func((input, type) => { - var json= AvroConvert.Avro2Json(input); + var json = AvroConvert.Avro2Json(input); return JsonConvert.DeserializeObject(json, type); }); return new object[] { func }; } } - - private static object[] DeserializeByLine - { - get - { - var func = new Func((input, type) => - { - var result = new List(); - - object openDeserializer = typeof(AvroConvert) - .GetMethod("OpenDeserializer", new[] { typeof(Stream) }) - ?.MakeGenericMethod(type); - // .Invoke(null, new object[] { new MemoryStream(input) }); - - using (var reader = AvroConvert.OpenDeserializer(new MemoryStream(input))) - { - while (reader.HasNext()) - { - var item = reader.ReadNext(); - - result.Add(item); - } - } - - return result; - }); - - return new object[] { func }; - } - } } \ No newline at end of file