Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/152 #154

Merged
merged 2 commits into from
Apr 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions src/AvroConvert/AvroConvert.Deserialize.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

using System;
using System.IO;
using System.Reflection;
using SolTechnology.Avro.AvroObjectServices.BuildSchema;
using SolTechnology.Avro.Features.Deserialize;

Expand Down Expand Up @@ -87,10 +88,18 @@ public static T Deserialize<T>(byte[] avroBytes, AvroConvertOptions options)
/// </remarks>
public static dynamic Deserialize(byte[] avroBytes, Type targetType)
{
object result = typeof(AvroConvert)
.GetMethod(nameof(Deserialize), new[] { typeof(byte[]) })
?.MakeGenericMethod(targetType)
.Invoke(null, new object[] { avroBytes });
object result;
try
{
result = typeof(AvroConvert)
.GetMethod(nameof(Deserialize), new[] { typeof(byte[]) })
?.MakeGenericMethod(targetType)
.Invoke(null, new object[] { avroBytes });
}
catch (TargetInvocationException ex)
{
throw ex.InnerException!;
}

return result;
}
Expand Down
6 changes: 3 additions & 3 deletions src/AvroConvert/AvroObjectServices/Write/Resolvers/Array.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ private void WriteArray(Encoder.WriteItem itemWriter, object @object, IWriter en
{
List<object> list = EnsureArrayObject(@object);

long l = list?.Count ?? 0;
long count = list?.Count ?? 0;
encoder.WriteArrayStart();
encoder.WriteItemCount(l);
WriteArrayValues(list, itemWriter, encoder, l);
encoder.WriteItemCount(count);
WriteArrayValues(list, itemWriter, encoder, count);
encoder.WriteArrayEnd();
}

Expand Down
38 changes: 21 additions & 17 deletions src/AvroConvert/Features/AvroToJson/Decoder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
using SolTechnology.Avro.AvroObjectServices.FileHeader;
using SolTechnology.Avro.AvroObjectServices.FileHeader.Codec;
using SolTechnology.Avro.AvroObjectServices.Read;
using SolTechnology.Avro.AvroObjectServices.Schemas;
using SolTechnology.Avro.AvroObjectServices.Schemas.Abstract;
using SolTechnology.Avro.Infrastructure.Exceptions;

Expand Down Expand Up @@ -65,7 +66,7 @@ internal object Decode(Stream stream, TypeSchema schema)
{
var header = reader.ReadHeader();

schema = schema ?? Schema.Create(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
schema ??= Schema.Create(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
var resolver = new Resolver(schema);

reader.ReadFixed(header.SyncData);
Expand All @@ -83,30 +84,33 @@ internal object Read(Reader reader, Header header, AbstractCodec codec, Resolver
return string.Empty;
}


var result = new List<object>();

do
{
long itemsCount = reader.ReadLong();
var data = reader.ReadDataBlock(header.SyncData, codec);
result.AddRange(ReadSingleBlock(reader, header, codec, resolver));
} while (!reader.IsReadToEnd());

reader = new Reader(new MemoryStream(data));
if (header.Schema is not ArraySchema && result.Count < 2)
{
return result.SingleOrDefault();
}

if (itemsCount > 1)
{
for (int i = 0; i < itemsCount; i++)
{
result.Add(resolver.Resolve(reader));
}
}
else
{
return resolver.Resolve(reader);
}
return result;
}

} while (!reader.IsReadToEnd());
private static List<object> ReadSingleBlock(Reader reader, Header header, AbstractCodec codec, Resolver resolver)
{
var result = new List<object>();
long itemsCount = reader.ReadLong();
var data = reader.ReadDataBlock(header.SyncData, codec);

reader = new Reader(new MemoryStream(data));

for (int i = 0; i < itemsCount; i++)
{
result.Add(resolver.Resolve(reader));
}

return result;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,39 @@ public class DeserializeFilesTests
private readonly byte[] _schemaOnlyAvroBytes = File.ReadAllBytes("header_only.avro");


[Theory]
[MemberData(nameof(TestEngine.DeserializeOnly), MemberType = typeof(TestEngine))]
public void Deserialize_FileContainsNoAvroData_NoExceptionIsThrown(Func<byte[], Type, dynamic> engine)
{
//Arrange


//Act
var result = engine.Invoke(_schemaOnlyAvroBytes, typeof(List<UserNameClass>));


//Assert
Assert.Equal(null, result);
}


[Theory]
[MemberData(nameof(TestEngine.DeserializeOnly), MemberType = typeof(TestEngine))]
[Trait("Fix", "https://github.com/AdrianStrugala/AvroConvert/issues/152")]
public void Deserialize_FileWithMultipleBlocks_EveryItemIsRead(Func<byte[], Type, dynamic> engine)
{
//Arrange


//Act
var result = (List<kylosample>)engine.Invoke(File.ReadAllBytes("userdata1.avro"), typeof(List<kylosample>));


//Assert
result.Should().HaveCount(1000);
}


[Fact]
public void Deserialize_CustomSchema_OnlyValuesFromCustomSchemaAreReturned()
{
Expand Down Expand Up @@ -64,19 +97,6 @@ public void Deserialize_NonGenericMethod_OnlyValuesFromCustomSchemaAreReturned()
Assert.Equal(expectedResult, result);
}

[Fact]
public void Deserialize_FileContainsNoAvroData_NoExceptionIsThrown()
{
//Arrange

//Act
var result = AvroConvert.Deserialize(_schemaOnlyAvroBytes, typeof(List<UserNameClass>));


//Assert
Assert.Equal(null, result);
}

[Fact]
public void Deserialize_InvalidFile_InvalidAvroObjectExceptionIsThrown()
{
Expand All @@ -92,20 +112,6 @@ public void Deserialize_InvalidFile_InvalidAvroObjectExceptionIsThrown()
Assert.IsType<InvalidAvroObjectException>(result);
}

[Fact]
public void Deserialize_FileWithMultipleBlocks_EveryItemIsRead()
{
//Arrange


//Act
var result = AvroConvert.Deserialize<List<kylosample>>(File.ReadAllBytes("userdata1.avro"));


//Assert
result.Should().HaveCount(1000);
}

[Fact]
public void Deserialization_with_ReadOnlySpan_Works()
{
Expand Down
46 changes: 42 additions & 4 deletions tests/AvroConvertTests/TestEngine.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using Newtonsoft.Json;
using SolTechnology.Avro;

namespace AvroConvertComponentTests;
Expand All @@ -13,14 +15,16 @@ public static IEnumerable<object[]> All()
yield return Headless;

yield return GenericJson;

yield return Brotli;

yield return Snappy;

yield return Deflate;

yield return Gzip;

yield return Json;
}

public static IEnumerable<object[]> Core()
Expand Down Expand Up @@ -55,6 +59,16 @@ public static IEnumerable<object[]> DefaultOnly()
yield return Snappy;
}

public static IEnumerable<object[]> DeserializeOnly()
{
yield return DefaultDeserializeOnly;

// TODO: need to create proper test strategy for this feature
// yield return DeserializeByLine;

yield return GenericJsonDeserializeOnly;
}

private static object[] Default
{
get
Expand Down Expand Up @@ -242,4 +256,28 @@ private static object[] BrotliWithSchema
return new object[] { @default };
}
}

private static object[] DefaultDeserializeOnly
{
get
{
var func = new Func<byte[], Type, dynamic>((input, type) => AvroConvert.Deserialize(input, type));

return new object[] { func };
}
}

private static object[] GenericJsonDeserializeOnly
{
get
{
var func = new Func<byte[], Type, dynamic>((input, type) =>
{
var json = AvroConvert.Avro2Json(input);
return JsonConvert.DeserializeObject(json, type);
});

return new object[] { func };
}
}
}
Loading