Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Avro] Add support for @Stringable annotation #53

Closed
wants to merge 1 commit into from
Closed

[Avro] Add support for @Stringable annotation #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 31 additions & 4 deletions avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroAnnotationIntrospector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
package com.fasterxml.jackson.dataformat.avro;

import java.io.File;

import org.apache.avro.reflect.AvroDefault;
import org.apache.avro.reflect.AvroIgnore;
import org.apache.avro.reflect.AvroName;
import org.apache.avro.reflect.Stringable;

import com.fasterxml.jackson.core.Version;
import com.fasterxml.jackson.databind.AnnotationIntrospector;
import com.fasterxml.jackson.databind.PropertyName;
import com.fasterxml.jackson.databind.introspect.Annotated;
import com.fasterxml.jackson.databind.introspect.AnnotatedClass;
import com.fasterxml.jackson.databind.introspect.AnnotatedConstructor;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;

import org.apache.avro.reflect.AvroDefault;
import org.apache.avro.reflect.AvroIgnore;
import org.apache.avro.reflect.AvroName;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;

/**
* Adds support for the following annotations from the Apache Avro implementation:
Expand All @@ -18,6 +24,8 @@
* <li>{@link AvroDefault @AvroDefault("default value")} - Alias for <code>JsonProperty.defaultValue</code>, to
* define default value for generated Schemas
* </li>
* <li>{@link Stringable @Stringable} - Alias for <code>JsonCreator</code> on the constructor and <code>JsonValue</code> on
* the {@link #toString()} method. </li>
* </ul>
*
* @since 2.9
Expand Down Expand Up @@ -57,4 +65,23 @@ protected PropertyName _findName(Annotated a)
AvroName ann = _findAnnotation(a, AvroName.class);
return (ann == null) ? null : PropertyName.construct(ann.value());
}

@Override
public boolean hasCreatorAnnotation(Annotated a) {
AnnotatedConstructor constructor = a instanceof AnnotatedConstructor ? (AnnotatedConstructor) a : null;
AnnotatedClass parentClass =
a instanceof AnnotatedConstructor && ((AnnotatedConstructor) a).getTypeContext() instanceof AnnotatedClass
? (AnnotatedClass) ((AnnotatedConstructor) a).getTypeContext()
: null;
return constructor != null && parentClass != null && parentClass.hasAnnotation(Stringable.class)
&& constructor.getParameterCount() == 1 && String.class.equals(constructor.getRawParameterType(0));
}

@Override
public Object findSerializer(Annotated a) {
if (a instanceof AnnotatedClass && a.hasAnnotation(Stringable.class) || a.getRawType() == File.class) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

First part makes sense I guess, but do we really need File? It's serialized using value.getAbsolutePath() already.
Alternatively if this really needs to be changed, I think we should just register serializer via AvroModule, and not through annotation introspector.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Avro uses toString() / getPath() instead of getAbsolutePath(), so this was a quick fix to make the behavior match up. I'll replace it with a proper serializer in the module.

return ToStringSerializer.class;
}
return null;
}
}
16 changes: 15 additions & 1 deletion avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package com.fasterxml.jackson.dataformat.avro;

import java.io.*;
import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.math.BigDecimal;

import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.core.base.ParserBase;
Expand Down Expand Up @@ -281,6 +284,17 @@ public JsonLocation getCurrentLocation()
@Override
public abstract JsonToken nextToken() throws IOException;

@Override
protected void convertNumberToBigDecimal() throws IOException {
// ParserBase uses _textValue instead of _numberDouble for some reason when NR_DOUBLE is set, but _textValue is not set by setNumber()
// Catch and use _numberDouble instead
if ((_numTypesValid & NR_DOUBLE) != 0 && _textValue == null) {
_numberBigDecimal = BigDecimal.valueOf(_numberDouble);
return;
}
super.convertNumberToBigDecimal();
}

/*
/**********************************************************
/* String value handling
Expand Down
16 changes: 14 additions & 2 deletions avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,14 +145,15 @@ public static Schema numericAvroSchema(JsonParser.NumberType type) {
switch (type) {
case INT:
return Schema.create(Schema.Type.INT);
case BIG_INTEGER:
case LONG:
return Schema.create(Schema.Type.LONG);
case FLOAT:
return Schema.create(Schema.Type.FLOAT);
case BIG_DECIMAL:
case DOUBLE:
return Schema.create(Schema.Type.DOUBLE);
case BIG_INTEGER:
case BIG_DECIMAL:
return Schema.create(Schema.Type.STRING);
default:
}
throw new IllegalStateException("Unrecognized number type: "+type);
Expand Down Expand Up @@ -211,6 +212,17 @@ public static Schema parseJsonSchema(String json) {
return parser.parse(json);
}

/**
* Constructs a new enum schema
*
* @param bean Enum type to use for name / description / namespace
* @param values List of enum names
* @return An {@link org.apache.avro.Schema.Type#ENUM ENUM} schema.
*/
public static Schema createEnumSchema(BeanDescription bean, List<String> values) {
return Schema.createEnum(getName(bean.getType()), bean.findClassDescription(), getNamespace(bean.getType()), values);
}

/**
* Returns the Avro type ID for a given type
*/
Expand Down
8 changes: 6 additions & 2 deletions avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/DoubleVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@
import org.apache.avro.Schema;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonNumberFormatVisitor;

public class DoubleVisitor
extends JsonNumberFormatVisitor.Base
implements SchemaBuilder
{
protected final JavaType _hint;
protected JsonParser.NumberType _type;

public DoubleVisitor() { }
public DoubleVisitor(JavaType typeHint) {
_hint = typeHint;
}

@Override
public void numberType(JsonParser.NumberType type) {
Expand All @@ -25,6 +29,6 @@ public Schema builtAvroSchema() {
// would require union most likely
return AvroSchemaHelper.anyNumberSchema();
}
return AvroSchemaHelper.numericAvroSchema(_type);
return AvroSchemaHelper.numericAvroSchema(_type, _hint);
}
}
31 changes: 22 additions & 9 deletions avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/MapVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ public class MapVisitor extends JsonMapFormatVisitor.Base
implements SchemaBuilder
{
protected final JavaType _type;

protected final DefinedSchemas _schemas;

protected Schema _valueSchema;


protected JavaType _keyType;

public MapVisitor(SerializerProvider p, JavaType type, DefinedSchemas schemas)
{
super(p);
Expand All @@ -30,7 +32,23 @@ public Schema builtAvroSchema() {
if (_valueSchema == null) {
throw new IllegalStateException("Missing value type for "+_type);
}
return Schema.createMap(_valueSchema);

Schema schema = Schema.createMap(_valueSchema);

// add the key type if there is one
if (_keyType != null && AvroSchemaHelper.isStringable(getProvider()
.getConfig()
.introspectClassAnnotations(_keyType)
.getClassInfo())) {
schema.addProp(AvroSchemaHelper.AVRO_SCHEMA_PROP_KEY_CLASS, AvroSchemaHelper.getTypeId(_keyType));
} else if (_keyType != null && !_keyType.isEnumType()) {
// Avro handles non-stringable keys by converting the map to an array of key/value records
// TODO add support for these in the schema, and custom serializers / deserializers to handle map restructuring
throw new UnsupportedOperationException(
"Key " + _keyType + " is not stringable and non-stringable map keys are not supported yet.");
}

return schema;
}

/*
Expand All @@ -43,12 +61,7 @@ public Schema builtAvroSchema() {
public void keyFormat(JsonFormatVisitable handler, JavaType keyType)
throws JsonMappingException
{
/* We actually don't care here, since Avro only has String-keyed
* Maps like JSON: meaning that anything Jackson can regularly
* serialize must convert to Strings anyway.
* If we do find problem cases, we can start verifying them here,
* but for now assume it all "just works".
*/
_keyType = keyType;
}

@Override
Expand Down
24 changes: 16 additions & 8 deletions avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,26 @@
import org.apache.avro.Schema;

import com.fasterxml.jackson.core.JsonParser.NumberType;
import com.fasterxml.jackson.databind.BeanDescription;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonStringFormatVisitor;
import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonValueFormat;
import com.fasterxml.jackson.databind.type.TypeFactory;

public class StringVisitor extends JsonStringFormatVisitor.Base
implements SchemaBuilder
{
protected final SerializerProvider _provider;
protected final JavaType _type;
protected final DefinedSchemas _schemas;

protected Set<String> _enums;

public StringVisitor(DefinedSchemas schemas, JavaType t) {
public StringVisitor(SerializerProvider provider, DefinedSchemas schemas, JavaType t) {
_schemas = schemas;
_type = t;
_provider = provider;
}

@Override
Expand All @@ -40,13 +44,17 @@ public Schema builtAvroSchema() {
if (_type.hasRawClass(char.class) || _type.hasRawClass(Character.class)) {
return AvroSchemaHelper.numericAvroSchema(NumberType.INT, TypeFactory.defaultInstance().constructType(Character.class));
}
if (_enums == null) {
return Schema.create(Schema.Type.STRING);
BeanDescription bean = _provider.getConfig().introspectClassAnnotations(_type);
if (_enums != null) {
Schema s = AvroSchemaHelper.createEnumSchema(bean, new ArrayList<>(_enums));
_schemas.addSchema(_type, s);
return s;
}
Schema s = Schema.createEnum(AvroSchemaHelper.getName(_type), "",
AvroSchemaHelper.getNamespace(_type),
new ArrayList<String>(_enums));
_schemas.addSchema(_type, s);
return s;
Schema schema = Schema.create(Schema.Type.STRING);
// Stringable classes need to include the type
if (AvroSchemaHelper.isStringable(bean.getClassInfo())) {
schema.addProp(AvroSchemaHelper.AVRO_SCHEMA_PROP_CLASS, AvroSchemaHelper.getTypeId(_type));
}
return schema;
}
}
4 changes: 2 additions & 2 deletions .../src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,14 @@ public JsonStringFormatVisitor expectStringFormat(JavaType type)
_valueSchema = s;
return null;
}
StringVisitor v = new StringVisitor(_schemas, type);
StringVisitor v = new StringVisitor(_provider, _schemas, type);
_builder = v;
return v;
}

@Override
public JsonNumberFormatVisitor expectNumberFormat(JavaType convertedType) {
DoubleVisitor v = new DoubleVisitor();
DoubleVisitor v = new DoubleVisitor(convertedType);
_builder = v;
return v;
}
Expand Down
34 changes: 31 additions & 3 deletions avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.apache.avro.Schema.Type;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.reflect.Stringable;

import com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaHelper;

Expand Down Expand Up @@ -58,11 +59,17 @@ public int resolveUnion(Schema union, Object datum) {
}
}
} else if (datum instanceof BigDecimal) {
int subOptimal = -1;
for (int i = 0, len = schemas.size(); i < len; i++) {
if (schemas.get(i).getType() == Type.DOUBLE) {
if (schemas.get(i).getType() == Type.STRING) {
return i;
} else if (schemas.get(i).getType() == Type.DOUBLE) {
subOptimal = i;
}
}
if (subOptimal > -1) {
return subOptimal;
}
}

// otherwise just default to base impl, stupid as it is...
Expand All @@ -71,8 +78,29 @@ public int resolveUnion(Schema union, Object datum) {

@Override
protected void write(Schema schema, Object datum, Encoder out) throws IOException {
if ((schema.getType() == Type.DOUBLE) && datum instanceof BigDecimal) {
out.writeDouble(((BigDecimal)datum).doubleValue());
// Cocerce numerical types, like BigDecimal -> double and BigInteger -> long
if (datum instanceof Number) {
switch (schema.getType()) {
case LONG:
super.write(schema, (((Number) datum).longValue()), out);
return;
case INT:
super.write(schema, (((Number) datum).intValue()), out);
return;
case FLOAT:
super.write(schema, (((Number) datum).floatValue()), out);
return;
case DOUBLE:
super.write(schema, (((Number) datum).doubleValue()), out);
return;
case STRING:
super.write(schema, datum.toString(), out);
return;
}
}
// Handle stringable classes
if (schema.getType() == Type.STRING && datum != null && datum.getClass().getAnnotation(Stringable.class) != null) {
super.write(schema, datum.toString(), out);
return;
}
if (datum instanceof String) {
Expand Down
Loading