From 4a792b950ee1cc5c0c9eeb42b7871a584c0b44ab Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Mon, 23 Dec 2024 10:04:50 +0100 Subject: [PATCH 1/4] Replacing empty doc string with null. This way doc field does not get created in Avro schema. Doc field is not mandatory and having it empty doe snot help either. --- .../jackson/dataformat/avro/schema/AvroSchemaHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java index 00f0592f6..99aa79937 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java @@ -286,7 +286,7 @@ public static Schema createEnumSchema(BeanDescription bean, List values) * @since 2.11 */ public static Schema createUUIDSchema() { - return Schema.createFixed("UUID", "", "java.util", 16); + return Schema.createFixed("UUID", null, "java.util", 16); } /** From 0384404de44af3a6fd6c4c684dd6bd584a8487cf Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Mon, 23 Dec 2024 10:05:16 +0100 Subject: [PATCH 2/4] Visitor for {@link java.util.UUID} type. When it is created with logicalTypesEnabled enabled, * Avro schema is created with logical type uuid. --- .../dataformat/avro/schema/StringVisitor.java | 6 +-- .../dataformat/avro/schema/UUIDVisitor.java | 42 +++++++++++++++++ .../avro/schema/VisitorFormatWrapperImpl.java | 6 +++ .../UUIDVisitor_builtAvroSchemaTest.java | 45 +++++++++++++++++++ 4 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor.java create mode 100644 avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor_builtAvroSchemaTest.java diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java index b8f3f5147..4aabb9dca 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java @@ -39,11 +39,7 @@ public Schema builtAvroSchema() { // should we construct JavaType for `Character.class` in case of primitive or... ? return AvroSchemaHelper.numericAvroSchema(NumberType.INT, _type); } - // [dataformats-binary#179]: need special help with UUIDs, to coerce into Binary - // (could actually be - if (_type.hasRawClass(java.util.UUID.class)) { - return AvroSchemaHelper.createUUIDSchema(); - } + BeanDescription bean = _provider.getConfig().introspectClassAnnotations(_type); Schema schema = Schema.create(Schema.Type.STRING); // Stringable classes need to include the type diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor.java new file mode 100644 index 000000000..069db06e9 --- /dev/null +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor.java @@ -0,0 +1,42 @@ +package com.fasterxml.jackson.dataformat.avro.schema; + +import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonStringFormatVisitor; +import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonValueFormat; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; + +import java.util.Set; + +/** + * Visitor for {@link java.util.UUID} type. When it is created with logicalTypesEnabled enabled, + * Avro schema is created with logical type uuid. + * + * @since 2.19 + */ +public class UUIDVisitor extends JsonStringFormatVisitor.Base + implements SchemaBuilder { + protected boolean _logicalTypesEnabled = false; + + + public UUIDVisitor(boolean logicalTypesEnabled) { + _logicalTypesEnabled = logicalTypesEnabled; + } + + @Override + public void format(JsonValueFormat format) { + // Ideally, we'd recognize UUIDs, Dates etc if need be, here... + } + + @Override + public void enumTypes(Set enums) { + // Do nothing + } + + @Override + public Schema builtAvroSchema() { + // [dataformats-binary#179]: need special help with UUIDs, to coerce into Binary + // (could actually be + Schema schema = AvroSchemaHelper.createUUIDSchema(); + return this._logicalTypesEnabled ? LogicalTypes.uuid().addToSchema(schema) : schema; + } +} diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java index a2b9d6edc..247d6b816 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java @@ -214,6 +214,12 @@ public JsonStringFormatVisitor expectStringFormat(JavaType type) return v; } + if (type.hasRawClass(java.util.UUID.class)) { + UUIDVisitor v = new UUIDVisitor(this._logicalTypesEnabled); + _builder = v; + return v; + } + StringVisitor v = new StringVisitor(_provider, type); _builder = v; return v; diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor_builtAvroSchemaTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor_builtAvroSchemaTest.java new file mode 100644 index 000000000..30b9cb37c --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor_builtAvroSchemaTest.java @@ -0,0 +1,45 @@ +package com.fasterxml.jackson.dataformat.avro.schema; + +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class UUIDVisitor_builtAvroSchemaTest { + + @Test + public void testLogicalTypesDisabled() { + // GIVEN + boolean logicalTypesEnabled = false; + UUIDVisitor uuidVisitor = new UUIDVisitor(logicalTypesEnabled); + + // WHEN + Schema actualSchema = uuidVisitor.builtAvroSchema(); + + // THEN + assertThat(actualSchema.getType()).isEqualTo(Schema.Type.FIXED); + assertThat(actualSchema.getFixedSize()).isEqualTo(16); + assertThat(actualSchema.getName()).isEqualTo("UUID"); + assertThat(actualSchema.getNamespace()).isEqualTo("java.util"); + assertThat(actualSchema.getProp(LogicalType.LOGICAL_TYPE_PROP)).isNull(); + } + + @Test + public void testLogicalTypesEnabled() { + // GIVEN + boolean logicalTypesEnabled = true; + UUIDVisitor uuidVisitor = new UUIDVisitor(logicalTypesEnabled); + + // WHEN + Schema actualSchema = uuidVisitor.builtAvroSchema(); + + // THEN + assertThat(actualSchema.getType()).isEqualTo(Schema.Type.FIXED); + assertThat(actualSchema.getFixedSize()).isEqualTo(16); + assertThat(actualSchema.getName()).isEqualTo("UUID"); + assertThat(actualSchema.getNamespace()).isEqualTo("java.util"); + assertThat(actualSchema.getProp(LogicalType.LOGICAL_TYPE_PROP)).isEqualTo("uuid"); + } + +} From dfee9f66c8a9c883f39a625ebef13d2aff3d78ec Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Mon, 23 Dec 2024 10:32:52 +0100 Subject: [PATCH 3/4] "Mapping to Logical Types" section extended with UUID --- avro/README.md | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/avro/README.md b/avro/README.md index d01db46a6..dfa91aeaf 100644 --- a/avro/README.md +++ b/avro/README.md @@ -114,32 +114,34 @@ and that's about it, for now. ## Avro Logical Types -Following is an extract from [Logical Types](http://avro.apache.org/docs/current/specification/_print/#logical-types) paragraph in -Avro schema specification: +The following is an excerpt from the [Logical Types](https://avro.apache.org/docs/1.11.1/specification/#logical-types) section of +the Avro schema specification: + > A logical type is an Avro primitive or complex type with extra attributes to represent a derived type. The attribute -> `logicalType` is always be present for a logical type, and is a string with the name of one of the logical types -> defined by Avro specification. +> `logicalType` must always be present for a logical type, and is a string with the name of one of the logical types +> listed later in this section. Other attributes may be defined for particular logical types. + +Logical types are supported for a limited set of `java.time` classes and for 'java.util.UUID'. See the table below for more details. -Generation of logical types for limited set of `java.time` classes is supported at the moment. See a table bellow. +### Mapping to Logical Types -### Mapping to Logical Type +Mapping to Avro type and logical type involves these steps: -Mapping to Avro type and logical type works in few steps: -1. Serializer for particular Java type (or class) determines a Jackson type where the Java type will be serialized into. -2. `AvroSchemaGenerator` determines corresponding Avro type for that Jackson type. -2. If logical type generation is enabled, then `logicalType` is determined for the above combination of Java type and - Avro type. +1. The serializer for a Java type identifies the Jackson type it will serialize into. +2. The `AvroSchemaGenerator` maps that Jackson type to the corresponding Avro type. +3. `logicalType` value is combination of Java type and Jackson type. #### Java type to Avro Logical Type mapping -| Java type | Serialization type | Generated Avro schema with Avro type and logical type -| ----------------------------- | ------------------ | ----------------------------------------------------- -| `java.time.OffsetDateTime` | NumberType.LONG | `{"type": "long", "logicalType": "timestamp-millis"}` -| `java.time.ZonedDateTime` | NumberType.LONG | `{"type": "long", "logicalType": "timestamp-millis"}` -| `java.time.Instant` | NumberType.LONG | `{"type": "long", "logicalType": "timestamp-millis"}` -| `java.time.LocalDate` | NumberType.INT | `{"type": "int", "logicalType": "date"}` -| `java.time.LocalTime` | NumberType.INT | `{"type": "int", "logicalType": "time-millis"}` -| `java.time.LocalDateTime` | NumberType.LONG | `{"type": "long", "logicalType": "local-timestamp-millis"}` +| Java type | Jackson type | Generated Avro schema with logical type | +|----------------------------|-----------------|---------------------------------------------------------------------------------------------------| +| `java.time.OffsetDateTime` | NumberType.LONG | `{"type": "long", "logicalType": "timestamp-millis"}` | +| `java.time.ZonedDateTime` | NumberType.LONG | `{"type": "long", "logicalType": "timestamp-millis"}` | +| `java.time.Instant` | NumberType.LONG | `{"type": "long", "logicalType": "timestamp-millis"}` | +| `java.time.LocalDate` | NumberType.INT | `{"type": "int", "logicalType": "date"}` | +| `java.time.LocalTime` | NumberType.INT | `{"type": "int", "logicalType": "time-millis"}` | +| `java.time.LocalDateTime` | NumberType.LONG | `{"type": "long", "logicalType": "local-timestamp-millis"}` | +| `java.util.UUID` | | `{"type": "fixed", "name": "UUID", "namespace": "java.util", "size": 16, "logicalType" : "uuid"}` | _Provided Avro logical type generation is enabled._ From 42c88c9d31130d3f7e2c01ed93e9956495c7a02e Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Mon, 23 Dec 2024 17:27:28 -0800 Subject: [PATCH 4/4] Add release notes, re-order imports slightly --- avro/README.md | 2 +- .../jackson/dataformat/avro/schema/UUIDVisitor.java | 5 +++-- .../avro/schema/UUIDVisitor_builtAvroSchemaTest.java | 3 ++- release-notes/CREDITS-2.x | 2 ++ release-notes/VERSION-2.x | 5 +++++ 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/avro/README.md b/avro/README.md index dfa91aeaf..6edcab9ec 100644 --- a/avro/README.md +++ b/avro/README.md @@ -141,7 +141,7 @@ Mapping to Avro type and logical type involves these steps: | `java.time.LocalDate` | NumberType.INT | `{"type": "int", "logicalType": "date"}` | | `java.time.LocalTime` | NumberType.INT | `{"type": "int", "logicalType": "time-millis"}` | | `java.time.LocalDateTime` | NumberType.LONG | `{"type": "long", "logicalType": "local-timestamp-millis"}` | -| `java.util.UUID` | | `{"type": "fixed", "name": "UUID", "namespace": "java.util", "size": 16, "logicalType" : "uuid"}` | +| `java.util.UUID` (2.19+) | | `{"type": "fixed", "name": "UUID", "namespace": "java.util", "size": 16, "logicalType" : "uuid"}` | _Provided Avro logical type generation is enabled._ diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor.java index 069db06e9..914fb954e 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor.java @@ -1,12 +1,13 @@ package com.fasterxml.jackson.dataformat.avro.schema; +import java.util.Set; + import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonStringFormatVisitor; import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonValueFormat; + import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; -import java.util.Set; - /** * Visitor for {@link java.util.UUID} type. When it is created with logicalTypesEnabled enabled, * Avro schema is created with logical type uuid. diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor_builtAvroSchemaTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor_builtAvroSchemaTest.java index 30b9cb37c..847fb399a 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor_builtAvroSchemaTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/schema/UUIDVisitor_builtAvroSchemaTest.java @@ -1,8 +1,9 @@ package com.fasterxml.jackson.dataformat.avro.schema; +import org.junit.Test; + import org.apache.avro.LogicalType; import org.apache.avro.Schema; -import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 5c7ddb2ea..d6f98acae 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -225,6 +225,8 @@ Michal Foksa (MichalFoksa@github) * Contributed #494: Avro Schema generation: allow mapping Java Enum properties to Avro String values (2.18.0) +* Contributed #536: (avro) Add Logical Type support for `java.util.UUID` + (2.19.0) Hunter Herman (hherman1@github) diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index 28e71468e..3b808b834 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -14,6 +14,11 @@ Active maintainers: === Releases === ------------------------------------------------------------------------ +2.19.0 (not yet released) + +#536: (avro) Add Logical Type support for `java.util.UUID` + (contributed by Michal F) + 2.18.2 (27-Nov-2024) No changes since 2.18.1