Skip to content

Commit

Permalink
[Avro] Add support for @Stringable annotation
Browse files Browse the repository at this point in the history
  • Loading branch information
baharclerode committed Mar 2, 2017
1 parent 8321c68 commit f3cb333
Show file tree
Hide file tree
Showing 9 changed files with 417 additions and 31 deletions.
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
package com.fasterxml.jackson.dataformat.avro;

import java.io.File;

import org.apache.avro.reflect.AvroDefault;
import org.apache.avro.reflect.AvroIgnore;
import org.apache.avro.reflect.AvroName;
import org.apache.avro.reflect.Stringable;

import com.fasterxml.jackson.core.Version;
import com.fasterxml.jackson.databind.AnnotationIntrospector;
import com.fasterxml.jackson.databind.PropertyName;
import com.fasterxml.jackson.databind.introspect.Annotated;
import com.fasterxml.jackson.databind.introspect.AnnotatedClass;
import com.fasterxml.jackson.databind.introspect.AnnotatedConstructor;
import com.fasterxml.jackson.databind.introspect.AnnotatedMember;

import org.apache.avro.reflect.AvroDefault;
import org.apache.avro.reflect.AvroIgnore;
import org.apache.avro.reflect.AvroName;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;

/**
* Adds support for the following annotations from the Apache Avro implementation:
Expand All @@ -18,6 +24,8 @@
* <li>{@link AvroDefault @AvroDefault("default value")} - Alias for <code>JsonProperty.defaultValue</code>, to
* define default value for generated Schemas
* </li>
* <li>{@link Stringable @Stringable} - Alias for <code>JsonCreator</code> on the constructor and <code>JsonValue</code> on
* the {@link #toString()} method. </li>
* </ul>
*
* @since 2.9
Expand Down Expand Up @@ -57,4 +65,23 @@ protected PropertyName _findName(Annotated a)
AvroName ann = _findAnnotation(a, AvroName.class);
return (ann == null) ? null : PropertyName.construct(ann.value());
}

@Override
public boolean hasCreatorAnnotation(Annotated a) {
AnnotatedConstructor constructor = a instanceof AnnotatedConstructor ? (AnnotatedConstructor) a : null;
AnnotatedClass parentClass =
a instanceof AnnotatedConstructor && ((AnnotatedConstructor) a).getTypeContext() instanceof AnnotatedClass
? (AnnotatedClass) ((AnnotatedConstructor) a).getTypeContext()
: null;
return constructor != null && parentClass != null && parentClass.hasAnnotation(Stringable.class)
&& constructor.getParameterCount() == 1 && String.class.equals(constructor.getRawParameterType(0));
}

@Override
public Object findSerializer(Annotated a) {
if (a instanceof AnnotatedClass && a.hasAnnotation(Stringable.class) || a.getRawType() == File.class) {
return ToStringSerializer.class;
}
return null;
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package com.fasterxml.jackson.dataformat.avro;

import java.io.*;
import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.math.BigDecimal;

import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.core.base.ParserBase;
Expand Down Expand Up @@ -281,6 +284,17 @@ public JsonLocation getCurrentLocation()
@Override
public abstract JsonToken nextToken() throws IOException;

@Override
protected void convertNumberToBigDecimal() throws IOException {
// ParserBase uses _textValue instead of _numberDouble for some reason when NR_DOUBLE is set, but _textValue is not set by setNumber()
// Catch and use _numberDouble instead
if ((_numTypesValid & NR_DOUBLE) != 0 && _textValue == null) {
_numberBigDecimal = BigDecimal.valueOf(_numberDouble);
return;
}
super.convertNumberToBigDecimal();
}

/*
/**********************************************************
/* String value handling
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,14 +145,15 @@ public static Schema numericAvroSchema(JsonParser.NumberType type) {
switch (type) {
case INT:
return Schema.create(Schema.Type.INT);
case BIG_INTEGER:
case LONG:
return Schema.create(Schema.Type.LONG);
case FLOAT:
return Schema.create(Schema.Type.FLOAT);
case BIG_DECIMAL:
case DOUBLE:
return Schema.create(Schema.Type.DOUBLE);
case BIG_INTEGER:
case BIG_DECIMAL:
return Schema.create(Schema.Type.STRING);
default:
}
throw new IllegalStateException("Unrecognized number type: "+type);
Expand Down Expand Up @@ -211,6 +212,17 @@ public static Schema parseJsonSchema(String json) {
return parser.parse(json);
}

/**
* Constructs a new enum schema
*
* @param bean Enum type to use for name / description / namespace
* @param values List of enum names
* @return An {@link org.apache.avro.Schema.Type#ENUM ENUM} schema.
*/
public static Schema createEnumSchema(BeanDescription bean, List<String> values) {
return Schema.createEnum(getName(bean.getType()), bean.findClassDescription(), getNamespace(bean.getType()), values);
}

/**
* Returns the Avro type ID for a given type
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@
import org.apache.avro.Schema;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonNumberFormatVisitor;

public class DoubleVisitor
extends JsonNumberFormatVisitor.Base
implements SchemaBuilder
{
protected final JavaType _hint;
protected JsonParser.NumberType _type;

public DoubleVisitor() { }
public DoubleVisitor(JavaType typeHint) {
_hint = typeHint;
}

@Override
public void numberType(JsonParser.NumberType type) {
Expand All @@ -25,6 +29,6 @@ public Schema builtAvroSchema() {
// would require union most likely
return AvroSchemaHelper.anyNumberSchema();
}
return AvroSchemaHelper.numericAvroSchema(_type);
return AvroSchemaHelper.numericAvroSchema(_type, _hint);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ public class MapVisitor extends JsonMapFormatVisitor.Base
implements SchemaBuilder
{
protected final JavaType _type;

protected final DefinedSchemas _schemas;

protected Schema _valueSchema;


protected JavaType _keyType;

public MapVisitor(SerializerProvider p, JavaType type, DefinedSchemas schemas)
{
super(p);
Expand All @@ -30,7 +32,23 @@ public Schema builtAvroSchema() {
if (_valueSchema == null) {
throw new IllegalStateException("Missing value type for "+_type);
}
return Schema.createMap(_valueSchema);

Schema schema = Schema.createMap(_valueSchema);

// add the key type if there is one
if (_keyType != null && AvroSchemaHelper.isStringable(getProvider()
.getConfig()
.introspectClassAnnotations(_keyType)
.getClassInfo())) {
schema.addProp(AvroSchemaHelper.AVRO_SCHEMA_PROP_KEY_CLASS, AvroSchemaHelper.getTypeId(_keyType));
} else if (_keyType != null && !_keyType.isEnumType()) {
// Avro handles non-stringable keys by converting the map to an array of key/value records
// TODO add support for these in the schema, and custom serializers / deserializers to handle map restructuring
throw new UnsupportedOperationException(
"Key " + _keyType + " is not stringable and non-stringable map keys are not supported yet.");
}

return schema;
}

/*
Expand All @@ -43,12 +61,7 @@ public Schema builtAvroSchema() {
public void keyFormat(JsonFormatVisitable handler, JavaType keyType)
throws JsonMappingException
{
/* We actually don't care here, since Avro only has String-keyed
* Maps like JSON: meaning that anything Jackson can regularly
* serialize must convert to Strings anyway.
* If we do find problem cases, we can start verifying them here,
* but for now assume it all "just works".
*/
_keyType = keyType;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,26 @@
import org.apache.avro.Schema;

import com.fasterxml.jackson.core.JsonParser.NumberType;
import com.fasterxml.jackson.databind.BeanDescription;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonStringFormatVisitor;
import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonValueFormat;
import com.fasterxml.jackson.databind.type.TypeFactory;

public class StringVisitor extends JsonStringFormatVisitor.Base
implements SchemaBuilder
{
protected final SerializerProvider _provider;
protected final JavaType _type;
protected final DefinedSchemas _schemas;

protected Set<String> _enums;

public StringVisitor(DefinedSchemas schemas, JavaType t) {
public StringVisitor(SerializerProvider provider, DefinedSchemas schemas, JavaType t) {
_schemas = schemas;
_type = t;
_provider = provider;
}

@Override
Expand All @@ -40,13 +44,17 @@ public Schema builtAvroSchema() {
if (_type.hasRawClass(char.class) || _type.hasRawClass(Character.class)) {
return AvroSchemaHelper.numericAvroSchema(NumberType.INT, TypeFactory.defaultInstance().constructType(Character.class));
}
if (_enums == null) {
return Schema.create(Schema.Type.STRING);
BeanDescription bean = _provider.getConfig().introspectClassAnnotations(_type);
if (_enums != null) {
Schema s = AvroSchemaHelper.createEnumSchema(bean, new ArrayList<>(_enums));
_schemas.addSchema(_type, s);
return s;
}
Schema s = Schema.createEnum(AvroSchemaHelper.getName(_type), "",
AvroSchemaHelper.getNamespace(_type),
new ArrayList<String>(_enums));
_schemas.addSchema(_type, s);
return s;
Schema schema = Schema.create(Schema.Type.STRING);
// Stringable classes need to include the type
if (AvroSchemaHelper.isStringable(bean.getClassInfo())) {
schema.addProp(AvroSchemaHelper.AVRO_SCHEMA_PROP_CLASS, AvroSchemaHelper.getTypeId(_type));
}
return schema;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,14 @@ public JsonStringFormatVisitor expectStringFormat(JavaType type)
_valueSchema = s;
return null;
}
StringVisitor v = new StringVisitor(_schemas, type);
StringVisitor v = new StringVisitor(_provider, _schemas, type);
_builder = v;
return v;
}

@Override
public JsonNumberFormatVisitor expectNumberFormat(JavaType convertedType) {
DoubleVisitor v = new DoubleVisitor();
DoubleVisitor v = new DoubleVisitor(convertedType);
_builder = v;
return v;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.apache.avro.Schema.Type;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.reflect.Stringable;

import com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaHelper;

Expand Down Expand Up @@ -58,11 +59,17 @@ public int resolveUnion(Schema union, Object datum) {
}
}
} else if (datum instanceof BigDecimal) {
int subOptimal = -1;
for (int i = 0, len = schemas.size(); i < len; i++) {
if (schemas.get(i).getType() == Type.DOUBLE) {
if (schemas.get(i).getType() == Type.STRING) {
return i;
} else if (schemas.get(i).getType() == Type.DOUBLE) {
subOptimal = i;
}
}
if (subOptimal > -1) {
return subOptimal;
}
}

// otherwise just default to base impl, stupid as it is...
Expand All @@ -71,8 +78,29 @@ public int resolveUnion(Schema union, Object datum) {

@Override
protected void write(Schema schema, Object datum, Encoder out) throws IOException {
if ((schema.getType() == Type.DOUBLE) && datum instanceof BigDecimal) {
out.writeDouble(((BigDecimal)datum).doubleValue());
// Cocerce numerical types, like BigDecimal -> double and BigInteger -> long
if (datum instanceof Number) {
switch (schema.getType()) {
case LONG:
super.write(schema, (((Number) datum).longValue()), out);
return;
case INT:
super.write(schema, (((Number) datum).intValue()), out);
return;
case FLOAT:
super.write(schema, (((Number) datum).floatValue()), out);
return;
case DOUBLE:
super.write(schema, (((Number) datum).doubleValue()), out);
return;
case STRING:
super.write(schema, datum.toString(), out);
return;
}
}
// Handle stringable classes
if (schema.getType() == Type.STRING && datum != null && datum.getClass().getAnnotation(Stringable.class) != null) {
super.write(schema, datum.toString(), out);
return;
}
if (datum instanceof String) {
Expand Down
Loading

0 comments on commit f3cb333

Please sign in to comment.