Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Fixed error in writing json (#720)
Browse files Browse the repository at this point in the history
  • Loading branch information
jorgecarleitao authored Dec 29, 2021
1 parent a3d469d commit 25a35f8
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 12 deletions.
2 changes: 1 addition & 1 deletion src/io/json/write/serialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ where
let item = iter.next().unwrap();
record.push((name.as_ref(), item));
});
serialize_item(buffer, &record, format, false);
serialize_item(buffer, &record, format, is_first_row);
is_first_row = false;
})
}
6 changes: 2 additions & 4 deletions tests/it/io/json/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@ fn read_batch(data: String, fields: Vec<Field>) -> Result<RecordBatch> {
json_read::deserialize(rows, fields)
}

fn write_batch(batch: RecordBatch) -> Result<Vec<u8>> {
let format = json_write::LineDelimited::default();

fn write_batch<F: json_write::JsonFormat>(batch: RecordBatch, format: F) -> Result<Vec<u8>> {
let batches = vec![Ok(batch)].into_iter();

let blocks = json_write::Serializer::new(batches, vec![], format);
Expand All @@ -41,7 +39,7 @@ fn round_trip(data: String) -> Result<()> {

let batch = read_batch(data.clone(), fields)?;

let buf = write_batch(batch)?;
let buf = write_batch(batch, json_write::LineDelimited::default())?;

let result = String::from_utf8(buf).unwrap();
println!("{}", result);
Expand Down
35 changes: 28 additions & 7 deletions tests/it/io/json/write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ fn write_simple_rows() -> Result<()> {

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();

let buf = write_batch(batch)?;
let buf = write_batch(batch, json_write::LineDelimited::default())?;

assert_eq!(
String::from_utf8(buf).unwrap(),
Expand All @@ -37,6 +37,27 @@ fn write_simple_rows() -> Result<()> {
Ok(())
}

#[test]
fn write_simple_rows_array() -> Result<()> {
let schema = Schema::new(vec![
Field::new("c1", DataType::Int32, false),
Field::new("c2", DataType::Utf8, false),
]);

let a = Int32Array::from([Some(1), Some(2), Some(3), None, Some(5)]);
let b = Utf8Array::<i32>::from(&vec![Some("a"), Some("b"), Some("c"), Some("d"), None]);

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();

let buf = write_batch(batch, json_write::JsonArray::default())?;

assert_eq!(
String::from_utf8(buf).unwrap(),
r#"[{"c1":1,"c2":"a"},{"c1":2,"c2":"b"},{"c1":3,"c2":"c"},{"c1":null,"c2":"d"},{"c1":5,"c2":null}]"#
);
Ok(())
}

#[test]
fn write_nested_struct_with_validity() -> Result<()> {
let inner = vec![
Expand Down Expand Up @@ -71,7 +92,7 @@ fn write_nested_struct_with_validity() -> Result<()> {

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();

let buf = write_batch(batch)?;
let buf = write_batch(batch, json_write::LineDelimited::default())?;

assert_eq!(
String::from_utf8(buf).unwrap(),
Expand Down Expand Up @@ -116,7 +137,7 @@ fn write_nested_structs() -> Result<()> {

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();

let buf = write_batch(batch)?;
let buf = write_batch(batch, json_write::LineDelimited::default())?;

assert_eq!(
String::from_utf8(buf).unwrap(),
Expand Down Expand Up @@ -153,7 +174,7 @@ fn write_struct_with_list_field() -> Result<()> {

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();

let buf = write_batch(batch)?;
let buf = write_batch(batch, json_write::LineDelimited::default())?;

assert_eq!(
String::from_utf8(buf).unwrap(),
Expand Down Expand Up @@ -199,7 +220,7 @@ fn write_nested_list() -> Result<()> {

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();

let buf = write_batch(batch)?;
let buf = write_batch(batch, json_write::LineDelimited::default())?;

assert_eq!(
String::from_utf8(buf).unwrap(),
Expand Down Expand Up @@ -259,7 +280,7 @@ fn write_list_of_struct() -> Result<()> {

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();

let buf = write_batch(batch)?;
let buf = write_batch(batch, json_write::LineDelimited::default())?;

assert_eq!(
String::from_utf8(buf).unwrap(),
Expand All @@ -278,7 +299,7 @@ fn write_escaped_utf8() -> Result<()> {

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap();

let buf = write_batch(batch)?;
let buf = write_batch(batch, json_write::LineDelimited::default())?;

assert_eq!(
String::from_utf8(buf).unwrap().as_bytes(),
Expand Down

0 comments on commit 25a35f8

Please sign in to comment.