Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Avro file output and test #46

Merged
merged 1 commit into from
Feb 22, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,13 @@ public enum Feature
*
* @since 2.7
*/
AVRO_BUFFERING(true)
AVRO_BUFFERING(true),

/**
* Feature that tells Avro to write data in file format (i.e. including the schema with the data)
* rather than the RPC format
*/
AVRO_FILE_OUTPUT(false)
;

protected final boolean _defaultState;
Expand Down Expand Up @@ -600,8 +606,12 @@ protected void _complete() throws IOException
// do not want to hide the original problem...
// First one sanity check, for a (relatively?) common case
if (_rootContext != null) {
_rootContext.complete();
_encoder.flush();
if (isEnabled(Feature.AVRO_FILE_OUTPUT)) {
_rootContext.complete(_output);
} else {
_rootContext.complete();
_encoder.flush();
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.fasterxml.jackson.dataformat.avro.ser;

import java.io.IOException;
import java.io.OutputStream;

import org.apache.avro.Schema;
import org.apache.avro.Schema.Type;
Expand Down Expand Up @@ -97,6 +98,10 @@ public void complete() throws IOException {
throw new IllegalStateException("Can not be called on "+getClass().getName());
}

public void complete(OutputStream outputStream) throws IOException {
throw new IllegalStateException("Can not be called on "+getClass().getName());
}

@Deprecated // remove from 2.9
public void complete(BinaryEncoder encoder) throws IOException { complete(); }

Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
package com.fasterxml.jackson.dataformat.avro.ser;

import java.io.IOException;
import java.io.OutputStream;

import org.apache.avro.Schema;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.*;
import org.apache.avro.io.BinaryEncoder;

import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.dataformat.avro.AvroGenerator;
import org.apache.avro.io.DatumWriter;

class RootContext
extends AvroWriteContext
Expand Down Expand Up @@ -105,6 +108,16 @@ public void complete() throws IOException {
_rootValue = null;
}

@Override
public void complete(OutputStream outputStream) throws IOException {
DatumWriter<Object> datumWriter = new NonBSGenericDatumWriter<>(_schema);
DataFileWriter<Object> dataFileWriter = new DataFileWriter<>(datumWriter);

dataFileWriter.create(_schema, outputStream);
dataFileWriter.append(_rootValue);
dataFileWriter.close();
}

@Override
public void appendDesc(StringBuilder sb) {
sb.append("/");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;

import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.SeekableByteArrayInput;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.DatumReader;
import org.junit.Assert;

import com.fasterxml.jackson.annotation.JsonPropertyOrder;
Expand Down Expand Up @@ -160,4 +165,31 @@ public void testIgnoringOfUnknownObject() throws Exception
BinaryAndNumber output = mapper.reader(SCHEMA_WITH_BINARY_JSON).forType(BinaryAndNumber.class).readValue(bytes);
assertEquals("Bob", output.name);
}

public void testFileOutput() throws Exception
{
Employee empl = new Employee();
empl.name = "Bobbee";
empl.age = 39;
empl.emails = new String[] { "[email protected]", "[email protected]" };
empl.boss = null;

AvroFactory af = new AvroFactory();
ObjectMapper mapper = new ObjectMapper(af);

af.enable(AvroGenerator.Feature.AVRO_FILE_OUTPUT);

AvroSchema schema = getEmployeeSchema();
byte[] bytes = mapper.writer(schema).writeValueAsBytes(empl);

assertNotNull(bytes);
assertEquals(301, bytes.length);

DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(schema.getAvroSchema());
DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new SeekableByteArrayInput(bytes), datumReader);
GenericRecord output = dataFileReader.next();

assertNotNull(output);
assertEquals(output.get("name").toString(), empl.name);
}
}