From ed016905cf56f4cd9f939d731b6486ad3cd6a23e Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:36:53 -0400 Subject: [PATCH 1/2] Remove Jackson from Storage Avro --- .../test/shared/StorageCommonTestUtils.java | 32 ++--- .../checkstyle-suppressions.xml | 12 -- .../azure-storage-internal-avro/pom.xml | 21 --- .../avro/implementation/AvroParser.java | 43 +++--- .../avro/implementation/AvroReader.java | 5 + .../schema/AvroCompositeSchema.java | 4 +- .../implementation/schema/AvroSchema.java | 53 ++++--- .../schema/AvroSimpleSchema.java | 8 +- .../avro/implementation/schema/AvroType.java | 130 +++++++++--------- .../schema/complex/AvroArraySchema.java | 37 ++--- .../schema/complex/AvroMapSchema.java | 42 ++---- .../schema/complex/AvroRecordSchema.java | 18 +-- .../schema/file/AvroBlockSchema.java | 38 ++--- .../schema/primitive/AvroNullSchema.java | 7 +- .../src/main/java/module-info.java | 2 - 15 files changed, 174 insertions(+), 278 deletions(-) diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java index 1462a4a5b8afc..ef7b7c16ee1de 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/StorageCommonTestUtils.java @@ -72,7 +72,7 @@ public final class StorageCommonTestUtils { @SuppressWarnings("deprecation") private static HttpClient createJdkHttpClient() throws ReflectiveOperationException { Class clazz = Class.forName("com.azure.core.http.jdk.httpclient.JdkHttpClientProvider"); - return (HttpClient) clazz.getDeclaredMethod("createInstance").invoke(clazz.newInstance()); + return (HttpClient) clazz.getDeclaredMethod("createInstance").invoke(clazz.newInstance()); } /** @@ -198,8 +198,8 @@ public static boolean compareFiles(File file1, File file2, long offset, long cou * @return The instrumented builder. */ @SuppressWarnings("unchecked") - public static , E extends Enum> T instrument(T builder, - HttpLogOptions logOptions, InterceptorManager interceptorManager) { + public static , E extends Enum> T instrument(T builder, HttpLogOptions logOptions, + InterceptorManager interceptorManager) { // Groovy style reflection. All our builders follow this pattern. builder.httpClient(getHttpClient(interceptorManager)); @@ -210,12 +210,11 @@ public static , E extends Enum> T instrument(T builder if (ENVIRONMENT.getServiceVersion() != null) { try { Method serviceVersionMethod = Arrays.stream(builder.getClass().getDeclaredMethods()) - .filter(method -> "serviceVersion".equals(method.getName()) - && method.getParameterCount() == 1 + .filter(method -> "serviceVersion".equals(method.getName()) && method.getParameterCount() == 1 && ServiceVersion.class.isAssignableFrom(method.getParameterTypes()[0])) .findFirst() - .orElseThrow(() -> new RuntimeException("Unable to find serviceVersion method for builder: " - + builder.getClass())); + .orElseThrow(() -> new RuntimeException( + "Unable to find serviceVersion method for builder: " + builder.getClass())); Class serviceVersionClass = (Class) serviceVersionMethod.getParameterTypes()[0]; ServiceVersion serviceVersion = (ServiceVersion) Enum.valueOf(serviceVersionClass, ENVIRONMENT.getServiceVersion()); @@ -310,13 +309,13 @@ public static File getRandomFile(int size, TestResourceNamer testResourceNamer) public static TokenCredential getTokenCredential(InterceptorManager interceptorManager) { if (interceptorManager.isPlaybackMode()) { return new MockTokenCredential(); - } else if (interceptorManager.isRecordMode()){ + } else if (interceptorManager.isRecordMode()) { return new DefaultAzureCredentialBuilder().build(); } else { //live Configuration config = Configuration.getGlobalConfiguration(); - ChainedTokenCredentialBuilder builder = new ChainedTokenCredentialBuilder() - .addLast(new EnvironmentCredentialBuilder().build()) + ChainedTokenCredentialBuilder builder = new ChainedTokenCredentialBuilder().addLast( + new EnvironmentCredentialBuilder().build()) .addLast(new AzureCliCredentialBuilder().build()) .addLast(new AzureDeveloperCliCredentialBuilder().build()); @@ -325,19 +324,18 @@ public static TokenCredential getTokenCredential(InterceptorManager interceptorM String tenantId = config.get("AZURESUBSCRIPTION_TENANT_ID"); String systemAccessToken = config.get("SYSTEM_ACCESSTOKEN"); - if (!CoreUtils.isNullOrEmpty(serviceConnectionId) - && !CoreUtils.isNullOrEmpty(clientId) - && !CoreUtils.isNullOrEmpty(tenantId) - && !CoreUtils.isNullOrEmpty(systemAccessToken)) { + if (!CoreUtils.isNullOrEmpty(serviceConnectionId) && !CoreUtils.isNullOrEmpty(clientId) + && !CoreUtils.isNullOrEmpty(tenantId) && !CoreUtils.isNullOrEmpty(systemAccessToken)) { - AzurePipelinesCredential pipelinesCredential = new AzurePipelinesCredentialBuilder() - .systemAccessToken(systemAccessToken) + AzurePipelinesCredential pipelinesCredential = new AzurePipelinesCredentialBuilder().systemAccessToken( + systemAccessToken) .clientId(clientId) .tenantId(tenantId) .serviceConnectionId(serviceConnectionId) .build(); - builder.addLast(request -> pipelinesCredential.getToken(request).subscribeOn(Schedulers.boundedElastic())); + builder.addLast( + request -> pipelinesCredential.getToken(request).subscribeOn(Schedulers.boundedElastic())); } builder.addLast(new AzurePowerShellCredentialBuilder().build()); diff --git a/sdk/storage/azure-storage-internal-avro/checkstyle-suppressions.xml b/sdk/storage/azure-storage-internal-avro/checkstyle-suppressions.xml index 4c376ddf2ce15..322f1bdcfae2f 100644 --- a/sdk/storage/azure-storage-internal-avro/checkstyle-suppressions.xml +++ b/sdk/storage/azure-storage-internal-avro/checkstyle-suppressions.xml @@ -3,19 +3,7 @@ - - - - - - - - - - - - diff --git a/sdk/storage/azure-storage-internal-avro/pom.xml b/sdk/storage/azure-storage-internal-avro/pom.xml index 25f929e211375..a41a75c5ccc94 100644 --- a/sdk/storage/azure-storage-internal-avro/pom.xml +++ b/sdk/storage/azure-storage-internal-avro/pom.xml @@ -61,12 +61,6 @@ 12.26.0 - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - 2.13.5 - - org.junit.jupiter junit-jupiter-api @@ -179,21 +173,6 @@ - - - org.apache.maven.plugins - maven-enforcer-plugin - 3.4.1 - - - - - com.fasterxml.jackson.dataformat:jackson-dataformat-xml:[2.13.5] - - - - - diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroParser.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroParser.java index 79d9e0530df09..0ddb4555ac62e 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroParser.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroParser.java @@ -20,12 +20,12 @@ /** * A class that represents a push based AvroParser that can parse avro data from a reactive stream. - * + *

* The parser stores the {@link AvroParserState current state}, the sync marker (parsed from the header), * the file type (parsed from the header metadata), and the list of records collected so far. - * + *

* The {@link AvroParser#parse(ByteBuffer)} method accepts ByteBuffers as they are emitted from the stream. - * + *

* Header Block Block Block .... */ public class AvroParser { @@ -47,7 +47,7 @@ public class AvroParser { /* Holds objects collected so far. */ private List objects; - private boolean partialRead; /* Whether the Avro Parser will read the Header and Block off different + private final boolean partialRead; /* Whether the Avro Parser will read the Header and Block off different streams. This is custom functionality for Changefeed. */ /** @@ -59,21 +59,18 @@ public class AvroParser { this.partialRead = partialRead; /* Start off by adding the header schema to the stack so we can parse it. */ - AvroHeaderSchema headerSchema = new AvroHeaderSchema( - this.state, - this::onFilteredHeader - ); + AvroHeaderSchema headerSchema = new AvroHeaderSchema(this.state, this::onFilteredHeader); headerSchema.pushToStack(); } Mono prepareParserToReadBody(long sourceOffset, long thresholdIndex) { if (!this.partialRead) { - return Mono.error(new IllegalStateException("This method should only be called when parsing header " - + "and body separately.")); + return Mono.error(new IllegalStateException( + "This method should only be called when parsing header " + "and body separately.")); } if (this.objectType == null || this.syncMarker == null) { - return Mono.error(new IllegalStateException("Expected to read entire header before preparing " - + "parser to read body.")); + return Mono.error( + new IllegalStateException("Expected to read entire header before preparing " + "parser to read body.")); } this.state = new AvroParserState(sourceOffset); this.objects = new ArrayList<>(); @@ -111,23 +108,17 @@ private void onFilteredHeader(Object header) { * Block handler. * * @param beginObjectIndex The object index after which to start aggregating events in the block. - * By default, this is 0 to collect all objects in the block. + * By default, this is 0 to collect all objects in the block. */ private void onBlock(Object beginObjectIndex) { /* On reading the block, read another block. */ AvroSchema.checkType("beginObjectIndex", beginObjectIndex, Long.class); - final AvroBlockSchema blockSchema = new AvroBlockSchema( - this.objectType, - (Long) beginObjectIndex, - o -> { - AvroSchema.checkType("object", o, AvroObject.class); - this.objects.add((AvroObject) o); - }, /* Object result handler. */ - this.syncMarker, - this.state, - this::onBlock - ); + final AvroBlockSchema blockSchema = new AvroBlockSchema(this.objectType, (Long) beginObjectIndex, o -> { + AvroSchema.checkType("object", o, AvroObject.class); + this.objects.add((AvroObject) o); + }, /* Object result handler. */ + this.syncMarker, this.state, this::onBlock); blockSchema.pushToStack(); } @@ -158,8 +149,8 @@ public Flux parse(ByteBuffer buffer) { return Flux.empty(); } AvroSchema schema = this.state.peekFromStack(); - while ((schema instanceof AvroCompositeSchema) - || ((schema instanceof AvroSimpleSchema) && ((AvroSimpleSchema) schema).canProgress())) { + while ((schema instanceof AvroCompositeSchema) || ((schema instanceof AvroSimpleSchema) + && ((AvroSimpleSchema) schema).canProgress())) { if (schema instanceof AvroSimpleSchema) { ((AvroSimpleSchema) schema).progress(); } diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroReader.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroReader.java index adf93ab3bc7c4..8b1122e36671f 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroReader.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroReader.java @@ -9,5 +9,10 @@ * An interface that represents an AvroReader. */ public interface AvroReader { + /** + * Read a stream of {@link AvroObject}. + * + * @return A stream of {@link AvroObject}. + */ Flux read(); } diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroCompositeSchema.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroCompositeSchema.java index 40790a5351b9d..948f2da9ec2c7 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroCompositeSchema.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroCompositeSchema.java @@ -9,7 +9,7 @@ /** * An abstract class that represents a composite Avro schema that can return an Object result. - * + *

* Composite avro schemas depend on other avro schemas to populate the result. * * @see AvroSchema @@ -18,7 +18,7 @@ public abstract class AvroCompositeSchema extends AvroSchema { /** * Constructs a new Schema. * - * @param state The state of the parser. + * @param state The state of the parser. * @param onResult The result handler. */ public AvroCompositeSchema(AvroParserState state, Consumer onResult) { diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroSchema.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroSchema.java index 9a2f53be08c4b..85ab8cc8694ab 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroSchema.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroSchema.java @@ -57,7 +57,7 @@ public abstract class AvroSchema { /** * Constructs a new Schema. * - * @param state The state of the parser. + * @param state The state of the parser. * @param onResult The result handler. */ public AvroSchema(AvroParserState state, Consumer onResult) { @@ -72,7 +72,9 @@ public AvroSchema(AvroParserState state, Consumer onResult) { public abstract void pushToStack(); /** - * @return Whether or not the schema is done. Also indicates that the result is ready. + * Whether the schema is done. Also indicates that the result is ready. + * + * @return Whether the schema is done. Also indicates that the result is ready. */ public boolean isDone() { return this.done; @@ -88,8 +90,8 @@ public void publishResult() { /** * Gets the schema associated with the type. * - * @param type The {@link AvroType type} that defines the schema. - * @param state {@link AvroParserState} + * @param type The {@link AvroType type} that defines the schema. + * @param state {@link AvroParserState} * @param onResult {@link Consumer} * @return {@link AvroSchema} * @see AvroType @@ -112,36 +114,36 @@ public static AvroSchema getSchema(AvroType type, AvroParserState state, Consume return new AvroBytesSchema(state, onResult); case STRING: return new AvroStringSchema(state, onResult); - case RECORD: { + case RECORD: checkType("type", type, AvroType.AvroRecordType.class); AvroType.AvroRecordType recordType = (AvroType.AvroRecordType) type; return new AvroRecordSchema(recordType.getName(), recordType.getFields(), state, onResult); - } - case ENUM: { + + case ENUM: checkType("type", type, AvroType.AvroEnumType.class); AvroType.AvroEnumType enumType = (AvroType.AvroEnumType) type; return new AvroEnumSchema(enumType.getSymbols(), state, onResult); - } - case ARRAY: { + + case ARRAY: checkType("type", type, AvroType.AvroArrayType.class); AvroType.AvroArrayType arrayType = (AvroType.AvroArrayType) type; return new AvroArraySchema(arrayType.getItemType(), state, onResult); - } - case MAP: { + + case MAP: checkType("type", type, AvroType.AvroMapType.class); AvroType.AvroMapType mapType = (AvroType.AvroMapType) type; return new AvroMapSchema(mapType.getValueType(), state, onResult); - } - case UNION: { + + case UNION: checkType("type", type, AvroType.AvroUnionType.class); AvroType.AvroUnionType unionType = (AvroType.AvroUnionType) type; return new AvroUnionSchema(unionType.getTypes(), state, onResult); - } - case FIXED: { + + case FIXED: checkType("type", type, AvroType.AvroFixedType.class); AvroType.AvroFixedType fixedType = (AvroType.AvroFixedType) type; return new AvroFixedSchema(fixedType.getSize(), state, onResult); - } + default: throw new RuntimeException("Unsupported type " + type.getType()); } @@ -150,14 +152,14 @@ public static AvroSchema getSchema(AvroType type, AvroParserState state, Consume /** * Checks if the object matches the expected type. * - * @param name The name of the variable. - * @param obj The object. + * @param name The name of the variable. + * @param obj The object. * @param expectedType The expected type. */ public static void checkType(String name, Object obj, Class expectedType) { if (!expectedType.isAssignableFrom(obj.getClass())) { - throw new IllegalStateException(String.format( - "Expected '%s' to be of type %s", name, expectedType.getSimpleName())); + throw new IllegalStateException( + String.format("Expected '%s' to be of type %s", name, expectedType.getSimpleName())); } } @@ -168,13 +170,10 @@ public static void checkType(String name, Object obj, Class expectedType) { * @return The byte array. */ public static byte[] getBytes(List bytes) { - long longTotalBytes = bytes - .stream() - .mapToLong(buffer -> { - checkType("buffer", buffer, ByteBuffer.class); - return ((ByteBuffer) buffer).remaining(); - }) - .sum(); + long longTotalBytes = bytes.stream().mapToLong(buffer -> { + checkType("buffer", buffer, ByteBuffer.class); + return ((ByteBuffer) buffer).remaining(); + }).sum(); if (longTotalBytes > Integer.MAX_VALUE) { throw new IllegalArgumentException("Bytes can not fit into a single array."); diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroSimpleSchema.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroSimpleSchema.java index 90c1dc7af4b9e..12e85e135c67b 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroSimpleSchema.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroSimpleSchema.java @@ -9,7 +9,7 @@ /** * An abstract class that represents a simple Avro schema that can return an Object result. - * + *

* Simple avro schemas directly consume bytes from the state to populate the result. * * @see AvroSchema @@ -18,7 +18,7 @@ public abstract class AvroSimpleSchema extends AvroSchema { /** * Constructs a new Schema. * - * @param state The state of the parser. + * @param state The state of the parser. * @param onResult The result handler. */ public AvroSimpleSchema(AvroParserState state, Consumer onResult) { @@ -26,7 +26,9 @@ public AvroSimpleSchema(AvroParserState state, Consumer onResult) { } /** - * @return Whether or not progress can be made for this schema. + * Whether progress can be made for this schema. + * + * @return Whether progress can be made for this schema. */ public abstract boolean canProgress(); diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroType.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroType.java index bca913115badb..0d6af268ce26e 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroType.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/AvroType.java @@ -3,14 +3,14 @@ package com.azure.storage.internal.avro.implementation.schema; +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; import com.azure.storage.internal.avro.implementation.AvroConstants; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.JsonNodeType; -import java.util.LinkedList; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import static com.azure.storage.internal.avro.implementation.AvroConstants.Types.ARRAY; import static com.azure.storage.internal.avro.implementation.AvroConstants.Types.BOOLEAN; @@ -31,15 +31,15 @@ /** * A class that represents an Avro type. * AvroTypes function as a type that stores all the data a schema may need. - * @see AvroType#getType(JsonNode) + * + * @see AvroType#getType(Object) */ public class AvroType { - private static final ObjectMapper MAPPER = new ObjectMapper(); - private final String type; /** * Creates a new instance of an AvroType. + * * @param type The type associated with the AvroType. */ AvroType(String type) { @@ -61,6 +61,7 @@ static class AvroPrimitiveType extends AvroType { /** * Creates a new instance of an AvroPrimitiveType. + * * @param type The type associated with the AvroType. * @see AvroConstants.Types#PRIMITIVE_TYPES */ @@ -72,6 +73,7 @@ static class AvroPrimitiveType extends AvroType { /** * An avro record type. * A record is defined by an array of fields. + * * @see AvroRecordField */ static class AvroRecordType extends AvroType { @@ -81,6 +83,7 @@ static class AvroRecordType extends AvroType { /** * Creates a new instance of an AvroRecordType. + * * @param name The name of the record. * @param fields The fields in the record. */ @@ -116,6 +119,7 @@ static class AvroEnumType extends AvroType { /** * Creates a new instance of an AvroEnumType. + * * @param name The name of the enum. * @param symbols The symbols associated with the enum. */ @@ -150,6 +154,7 @@ static class AvroArrayType extends AvroType { /** * Creates a new instance of an AvroArrayType. + * * @param itemType The type of the items in the array. */ AvroArrayType(AvroType itemType) { @@ -175,6 +180,7 @@ static class AvroMapType extends AvroType { /** * Creates a new instance of an AvroMapType. + * * @param valueType The type of the values in the map. */ AvroMapType(AvroType valueType) { @@ -200,6 +206,7 @@ static class AvroUnionType extends AvroType { /** * Creates a new instance of an AvroUnionType. + * * @param types The types that define a union. */ AvroUnionType(List types) { @@ -225,6 +232,7 @@ static class AvroFixedType extends AvroType { /** * Creates a new instance of an AvroFixedType. + * * @param size The number of bytes to read. */ AvroFixedType(Long size) { @@ -247,13 +255,11 @@ Long getSize() { * @return {@link AvroType} */ public static AvroType getType(String jsonString) { - JsonNode schemaJson; - try { - schemaJson = MAPPER.readTree(jsonString); - } catch (JsonProcessingException e) { - throw new IllegalStateException(e.getMessage()); + try (JsonReader jsonReader = JsonProviders.createReader(jsonString)) { + return AvroType.getType(jsonReader.readUntyped()); + } catch (IOException e) { + throw new IllegalStateException(e); } - return AvroType.getType(schemaJson); } /** @@ -262,20 +268,19 @@ public static AvroType getType(String jsonString) { * @param jsonSchema the json node that specifies the schema. * @return {@link AvroType} */ - private static AvroType getType(JsonNode jsonSchema) { - JsonNodeType nodeType = jsonSchema.getNodeType(); - switch (nodeType) { + @SuppressWarnings("unchecked") + private static AvroType getType(Object jsonSchema) { + if (jsonSchema instanceof String) { /* Primitive Avro Types. */ - case STRING: - return getJsonStringType(jsonSchema); + return getJsonStringType((String) jsonSchema); + } else if (jsonSchema instanceof List) { /* Union Avro Types. */ - case ARRAY: - return getJsonArrayType(jsonSchema); + return getJsonArrayType((List) jsonSchema); + } else if (jsonSchema instanceof Map) { /* Complex Avro Types. */ - case OBJECT: - return getJsonObjectType(jsonSchema); - default: - throw new RuntimeException("Unsupported type"); + return getJsonObjectType((Map) jsonSchema); + } else { + throw new RuntimeException("Unsupported type"); } } @@ -285,12 +290,11 @@ private static AvroType getType(JsonNode jsonSchema) { * @param jsonSchema the json node that specifies the schema. * @return {@link AvroType} */ - private static AvroType getJsonStringType(JsonNode jsonSchema) { + private static AvroType getJsonStringType(String jsonSchema) { /* TODO (gapra): This could also be another named type. Not required for QQ/CF. */ /* Example: "long" */ - String type = jsonSchema.asText(); - if (PRIMITIVE_TYPES.contains(type)) { - return new AvroPrimitiveType(type); + if (PRIMITIVE_TYPES.contains(jsonSchema)) { + return new AvroPrimitiveType(jsonSchema); } else { throw new RuntimeException("Unsupported type"); } @@ -302,7 +306,7 @@ private static AvroType getJsonStringType(JsonNode jsonSchema) { * @param jsonSchema the json node that specifies the schema. * @return {@link AvroType} */ - private static AvroType getJsonArrayType(JsonNode jsonSchema) { + private static AvroType getJsonArrayType(List jsonSchema) { /* Example: ["null","string"] */ List types = getUnionTypes(jsonSchema); return new AvroUnionType(types); @@ -314,8 +318,9 @@ private static AvroType getJsonArrayType(JsonNode jsonSchema) { * @param jsonSchema the json node that specifies the schema. * @return {@link AvroType} */ - private static AvroType getJsonObjectType(JsonNode jsonSchema) { - String type = jsonSchema.get("type").asText(); + @SuppressWarnings("unchecked") + private static AvroType getJsonObjectType(Map jsonSchema) { + String type = String.valueOf(jsonSchema.get("type")); switch (type) { /* Primitive Types. */ case NULL: @@ -328,7 +333,8 @@ private static AvroType getJsonObjectType(JsonNode jsonSchema) { case STRING: /* Example: {"type": "string"} */ return new AvroPrimitiveType(type); - case RECORD: { + + case RECORD: /* Example: { "type": "record", "name": "test", "fields" : [ @@ -339,12 +345,12 @@ private static AvroType getJsonObjectType(JsonNode jsonSchema) { if (jsonSchema.get("aliases") != null) { throw new IllegalArgumentException("Unexpected aliases in schema."); } - String fullName = jsonSchema.get("name").asText(); + String fullName = String.valueOf(jsonSchema.get("name")); String name = fullName.substring(fullName.lastIndexOf('.') + 1); - List fields = getRecordFields(jsonSchema.withArray("fields")); + List fields = getRecordFields((List>) jsonSchema.get("fields")); return new AvroRecordType(name, fields); - } - case ENUM: { + + case ENUM: /* Example: { "type": "enum", "name": "Suit", "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] @@ -352,25 +358,23 @@ private static AvroType getJsonObjectType(JsonNode jsonSchema) { if (jsonSchema.get("aliases") != null) { throw new IllegalArgumentException("Unexpected aliases in schema."); } - String name = jsonSchema.get("name").asText(); - List symbols = getEnumSymbols(jsonSchema.withArray("symbols")); - return new AvroEnumType(name, symbols); - } - case ARRAY: { + List symbols = getEnumSymbols((List) jsonSchema.get("symbols")); + return new AvroEnumType(String.valueOf(jsonSchema.get("name")), symbols); + + case ARRAY: /* Example: {"type": "array", "items": "string"} */ AvroType items = getType(jsonSchema.get("items")); return new AvroArrayType(items); - } - case MAP: { + + case MAP: /* Example: {"type": "map", "values": "long"} */ AvroType values = getType(jsonSchema.get("values")); return new AvroMapType(values); - } - case FIXED: { + + case FIXED: /* Example: {"type": "fixed", "size": 16, "name": "md5"} */ - Long size = jsonSchema.get("size").asLong(); - return new AvroFixedType(size); - } + return new AvroFixedType(Long.parseLong(String.valueOf(jsonSchema.get("size")))); + default: throw new RuntimeException("Unsupported type"); } @@ -382,13 +386,12 @@ private static AvroType getJsonObjectType(JsonNode jsonSchema) { * @param parent the JsonNode array * @return The types of the union. */ - private static List getUnionTypes(JsonNode parent) { + private static List getUnionTypes(List parent) { /* Example: ["null","string"] */ - List types = new LinkedList<>(); + List types = new ArrayList<>(parent.size()); /* Get the type of each JsonNode in parent. */ - for (JsonNode child : parent) { - AvroType type = getType(child); - types.add(type); + for (Object child : parent) { + types.add(getType(child)); } return types; } @@ -399,11 +402,11 @@ private static List getUnionTypes(JsonNode parent) { * @param parent The JsonNode array * @return The symbols of the enum. */ - private static List getEnumSymbols(JsonNode parent) { + private static List getEnumSymbols(List parent) { /* Example: ["A", "B", "C", "D"] */ - List symbols = new LinkedList<>(); - for (JsonNode child : parent) { - symbols.add(child.asText()); + List symbols = new ArrayList<>(parent.size()); + for (Object child : parent) { + symbols.add(String.valueOf(child)); } return symbols; } @@ -414,15 +417,12 @@ private static List getEnumSymbols(JsonNode parent) { * @param parent The JsonNode array * @return The fields of the record. */ - private static List getRecordFields(JsonNode parent) { + private static List getRecordFields(List> parent) { /* Example: [ {"name": "a", "type": "long"}, {"name": "b", "type": "string"} ] */ - List fields = new LinkedList<>(); + List fields = new ArrayList<>(parent.size()); /* Get the name and type of each JsonNode in parent. */ - for (JsonNode child : parent) { - String name = child.get("name").asText(); - AvroType type = getType(child.get("type")); - - fields.add(new AvroRecordField(name, type)); + for (Map child : parent) { + fields.add(new AvroRecordField(String.valueOf(child.get("name")), getType(child.get("type")))); } return fields; } diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroArraySchema.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroArraySchema.java index 7eafcba174cfa..600aa2ad69726 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroArraySchema.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroArraySchema.java @@ -29,7 +29,7 @@ public class AvroArraySchema extends AvroCompositeSchema { private final AvroType itemType; private Long blockCount; - private List ret; + private final List ret; /** * Constructs a new AvroArraySchema. @@ -48,10 +48,7 @@ public AvroArraySchema(AvroType itemType, AvroParserState state, Consumer 0, read the item, call onItem. */ } else if (bc > 0) { this.blockCount = bc; - AvroSchema itemSchema = getSchema( - this.itemType, - this.state, - this::onItem - ); + AvroSchema itemSchema = getSchema(this.itemType, this.state, this::onItem); itemSchema.pushToStack(); /* If blockCount < 0, use absolute value, read the byteCount, call onByteCount. */ } else { this.blockCount = -bc; - AvroLongSchema byteCountSchema = new AvroLongSchema( - this.state, - this::onByteCount - ); + AvroLongSchema byteCountSchema = new AvroLongSchema(this.state, this::onByteCount); byteCountSchema.pushToStack(); } } @@ -94,11 +84,7 @@ private void onBlockCount(Object blockCount) { */ private void onByteCount(Object byteCount) { /* Read the item, call onItem. */ - AvroSchema itemSchema = getSchema( - this.itemType, - this.state, - this::onItem - ); + AvroSchema itemSchema = getSchema(this.itemType, this.state, this::onItem); itemSchema.pushToStack(); } @@ -116,18 +102,11 @@ private void onItem(Object item) { /* If blockCount = 0, there are no more items in the block, read another blockCount and call onBlockCount. */ if (this.blockCount == 0) { - AvroLongSchema blockCountSchema = new AvroLongSchema( - this.state, - this::onBlockCount - ); + AvroLongSchema blockCountSchema = new AvroLongSchema(this.state, this::onBlockCount); blockCountSchema.pushToStack(); /* If blockCount != 0, there are more items in the block, read another item and call onItem. */ } else { - AvroSchema itemSchema = getSchema( - this.itemType, - this.state, - this::onItem - ); + AvroSchema itemSchema = getSchema(this.itemType, this.state, this::onItem); itemSchema.pushToStack(); } } diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroMapSchema.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroMapSchema.java index 2935c6a297d45..fa170229f072d 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroMapSchema.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroMapSchema.java @@ -20,7 +20,7 @@ * schema. Map keys are assumed to be strings. * If a block's count is negative, its absolute value is used, and the count is followed immediately by a long block * size indicating the number of bytes in the block. - * + *

* Long Key Value Key Value Key Value .... Long Key Value Key Value Key Value .... Long(0) * If initial Long parsed is negative, it can look like * Long(negative) Long Key Value Key Value Key Value .... @@ -32,7 +32,7 @@ public class AvroMapSchema extends AvroCompositeSchema { private final AvroType valueType; private Long blockCount; private String key; - private Map ret; + private final Map ret; /** * Constructs a new AvroMapSchema. @@ -51,10 +51,7 @@ public AvroMapSchema(AvroType valueType, AvroParserState state, Consumer public void pushToStack() { this.state.pushToStack(this); /* Read the block size, call onBlockCount. */ - AvroLongSchema blockSchema = new AvroLongSchema( - this.state, - this::onBlockCount - ); + AvroLongSchema blockSchema = new AvroLongSchema(this.state, this::onBlockCount); blockSchema.pushToStack(); } @@ -65,7 +62,7 @@ public void pushToStack() { */ private void onBlockCount(Object blockCount) { checkType("blockCount", blockCount, Long.class); - Long bc = (Long) blockCount; + long bc = (long) blockCount; /* If blockCount = 0 then we're done.*/ if (bc == 0) { this.result = this.ret; @@ -73,18 +70,12 @@ private void onBlockCount(Object blockCount) { /* If blockCount > 0, read the key, call onKey. */ } else if (bc > 0) { this.blockCount = bc; - AvroStringSchema keySchema = new AvroStringSchema( - this.state, - this::onKey - ); + AvroStringSchema keySchema = new AvroStringSchema(this.state, this::onKey); keySchema.pushToStack(); /* If blockCount < 0, use absolute value, read the byteCount, call onByteCount. */ } else { this.blockCount = -bc; - AvroLongSchema byteCountSchema = new AvroLongSchema( - this.state, - this::onByteCount - ); + AvroLongSchema byteCountSchema = new AvroLongSchema(this.state, this::onByteCount); byteCountSchema.pushToStack(); } } @@ -96,10 +87,7 @@ private void onBlockCount(Object blockCount) { */ private void onByteCount(Object byteCount) { /* Read the key, call onKey. */ - AvroStringSchema keySchema = new AvroStringSchema( - this.state, - this::onKey - ); + AvroStringSchema keySchema = new AvroStringSchema(this.state, this::onKey); keySchema.pushToStack(); } @@ -112,11 +100,7 @@ private void onKey(Object key) { checkType("key", key, String.class); /* Store the key, read the value, call onValue. */ this.key = (String) key; - AvroSchema valueSchema = getSchema( - this.valueType, - this.state, - this::onValue - ); + AvroSchema valueSchema = getSchema(this.valueType, this.state, this::onValue); valueSchema.pushToStack(); } @@ -134,17 +118,11 @@ private void onValue(Object value) { /* If blockCount = 0, there are no more items in the block, read another blockCount and call onBlockCount. */ if (this.blockCount == 0) { - AvroLongSchema blockCountSchema = new AvroLongSchema( - this.state, - this::onBlockCount - ); + AvroLongSchema blockCountSchema = new AvroLongSchema(this.state, this::onBlockCount); blockCountSchema.pushToStack(); /* If blockCount != 0, there are more key/value pairs in the block, read another key and call onKey. */ } else { - AvroStringSchema keySchema = new AvroStringSchema( - this.state, - this::onKey - ); + AvroStringSchema keySchema = new AvroStringSchema(this.state, this::onKey); keySchema.pushToStack(); } } diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroRecordSchema.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroRecordSchema.java index 811930e552575..1a2b6fe031939 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroRecordSchema.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/complex/AvroRecordSchema.java @@ -19,10 +19,10 @@ * A record is encoded by encoding the values of its fields in the order that they are declared. * In other words, a record is encoded as just the concatenation of the encodings of its fields. * Field values are encoded per their schema. - * + *

* The field value schemas will do most of the work, so we need to just keep track of which field we are * working on and add them to the map as they come in. - * + *

* Field1 Field2 Field3 .... */ public class AvroRecordSchema extends AvroCompositeSchema { @@ -30,7 +30,7 @@ public class AvroRecordSchema extends AvroCompositeSchema { private final List fields; private Iterator fieldIterator; private AvroRecordField currentField; - private Map ret; + private final Map ret; /** * Constructs a new AvroRecordSchema. @@ -60,11 +60,7 @@ public void pushToStack() { this.fieldIterator = this.fields.iterator(); this.currentField = this.fieldIterator.next(); - AvroSchema fieldSchema = getSchema( - this.currentField.getType(), - this.state, - this::onField - ); + AvroSchema fieldSchema = getSchema(this.currentField.getType(), this.state, this::onField); fieldSchema.pushToStack(); } @@ -83,11 +79,7 @@ private void onField(Object result) { /* If there are more fields to be read, read the next field and call onField. */ if (this.fieldIterator.hasNext()) { this.currentField = this.fieldIterator.next(); - AvroSchema fieldSchema = getSchema( - this.currentField.getType(), - this.state, - this::onField - ); + AvroSchema fieldSchema = getSchema(this.currentField.getType(), this.state, this::onField); fieldSchema.pushToStack(); /* If there are no more fields, then we're done. */ } else { diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/file/AvroBlockSchema.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/file/AvroBlockSchema.java index 85939f9978287..0d2f3da6e3fe5 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/file/AvroBlockSchema.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/file/AvroBlockSchema.java @@ -23,7 +23,7 @@ * A long indicating the size in bytes of the serialized objects in the current block. * The serialized objects. If a codec is specified, this is compressed by that codec. * The file's 16-byte sync marker. - * + *

* Long Long Object Object Object .... Object SyncMarker */ public class AvroBlockSchema extends AvroCompositeSchema { @@ -35,7 +35,7 @@ public class AvroBlockSchema extends AvroCompositeSchema { private Long blockCount; private final Long beginObjectIndex; private long objectIndex; - private long blockOffset; + private final long blockOffset; private final byte[] syncMarker; /** @@ -64,10 +64,7 @@ public void pushToStack() { this.state.pushToStack(this); /* Read the block count, call onBlockCount. */ - AvroLongSchema blockCountSchema = new AvroLongSchema( - this.state, - this::onBlockCount - ); + AvroLongSchema blockCountSchema = new AvroLongSchema(this.state, this::onBlockCount); blockCountSchema.pushToStack(); } @@ -80,25 +77,19 @@ private void onBlockCount(Object blockCount) { checkType("blockCount", blockCount, Long.class); this.blockCount = (Long) blockCount; /* Read the block size, call onBlockSize. */ - AvroLongSchema blockSizeSchema = new AvroLongSchema( - this.state, - this::onBlockSize - ); + AvroLongSchema blockSizeSchema = new AvroLongSchema(this.state, this::onBlockSize); blockSizeSchema.pushToStack(); } /** * Block size handler. * On reading the block size, ignore it and read an object. + * * @param blockSize The block size. */ private void onBlockSize(Object blockSize) { /* Read the object, call onObject. */ - AvroSchema objectSchema = AvroSchema.getSchema( - this.objectType, - this.state, - this::onObject - ); + AvroSchema objectSchema = AvroSchema.getSchema(this.objectType, this.state, this::onObject); objectSchema.pushToStack(); } @@ -129,25 +120,18 @@ private void onObject(Object schema) { nextObjectIndex = 0; } /* Call the object handler to store this object in the AvroParser. */ - this.onAvroObject.accept(new AvroObject(this.blockOffset, this.objectIndex++, nextBlockOffset, - nextObjectIndex, schema)); + this.onAvroObject.accept( + new AvroObject(this.blockOffset, this.objectIndex++, nextBlockOffset, nextObjectIndex, schema)); } if (this.hasNext()) { /* If the block has another object, read another object and call onObject. */ - AvroSchema objectSchema = AvroSchema.getSchema( - this.objectType, - this.state, - this::onObject - ); + AvroSchema objectSchema = AvroSchema.getSchema(this.objectType, this.state, this::onObject); objectSchema.pushToStack(); } else { /* Otherwise, read the sync marker, call validateSync. */ - AvroFixedSchema syncSchema = new AvroFixedSchema( - AvroConstants.SYNC_MARKER_SIZE, - this.state, - this::validateSync - ); + AvroFixedSchema syncSchema = new AvroFixedSchema(AvroConstants.SYNC_MARKER_SIZE, this.state, + this::validateSync); syncSchema.pushToStack(); } } diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/primitive/AvroNullSchema.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/primitive/AvroNullSchema.java index 1edc39ec8f5ad..e2191fd9a5e71 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/primitive/AvroNullSchema.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/schema/primitive/AvroNullSchema.java @@ -43,8 +43,11 @@ public boolean canProgress() { return true; } - /* We use a custom type to return null since null cannot be emitted in a Flux. - Users of the AvroParser must transform all NullSchema.Null objects to null if necessary. */ + /** + * A custom type to represent null. + * This is necessary since null cannot be emitted in a Flux. + * Users of the AvroParser must transform all NullSchema.Null objects to null if necessary. + */ public static class Null { } diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/module-info.java b/sdk/storage/azure-storage-internal-avro/src/main/java/module-info.java index be88c2cda487b..f6229494b919e 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/module-info.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/module-info.java @@ -5,8 +5,6 @@ requires transitive com.azure.storage.common; requires com.azure.core; - requires com.fasterxml.jackson.dataformat.xml; - exports com.azure.storage.internal.avro.implementation to com.azure.storage.blob, com.azure.storage.blob.changefeed; From cf2b139033af88ac0c0859f2a5f36c9751fa3f93 Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:25:06 -0400 Subject: [PATCH 2/2] Update versions --- sdk/storage/azure-storage-blob-changefeed/pom.xml | 2 +- sdk/storage/azure-storage-blob-cryptography/pom.xml | 2 +- .../storage/internal/avro/implementation/AvroParser.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob-changefeed/pom.xml b/sdk/storage/azure-storage-blob-changefeed/pom.xml index 416f6eb58f272..294dd75438c89 100644 --- a/sdk/storage/azure-storage-blob-changefeed/pom.xml +++ b/sdk/storage/azure-storage-blob-changefeed/pom.xml @@ -62,7 +62,7 @@ com.azure azure-json - 1.1.0 + 1.2.0 com.azure diff --git a/sdk/storage/azure-storage-blob-cryptography/pom.xml b/sdk/storage/azure-storage-blob-cryptography/pom.xml index 0380bcb1b12bb..0ba5b2899468c 100644 --- a/sdk/storage/azure-storage-blob-cryptography/pom.xml +++ b/sdk/storage/azure-storage-blob-cryptography/pom.xml @@ -56,7 +56,7 @@ com.azure azure-json - 1.1.0 + 1.2.0 com.azure diff --git a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroParser.java b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroParser.java index 0ddb4555ac62e..0783a5c05500f 100644 --- a/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroParser.java +++ b/sdk/storage/azure-storage-internal-avro/src/main/java/com/azure/storage/internal/avro/implementation/AvroParser.java @@ -66,11 +66,11 @@ public class AvroParser { Mono prepareParserToReadBody(long sourceOffset, long thresholdIndex) { if (!this.partialRead) { return Mono.error(new IllegalStateException( - "This method should only be called when parsing header " + "and body separately.")); + "This method should only be called when parsing header and body separately.")); } if (this.objectType == null || this.syncMarker == null) { return Mono.error( - new IllegalStateException("Expected to read entire header before preparing " + "parser to read body.")); + new IllegalStateException("Expected to read entire header before preparing parser to read body.")); } this.state = new AvroParserState(sourceOffset); this.objects = new ArrayList<>();