Re-Encode Offsets When Writing ListArray to IPC

[JasonLi-cn]

Describe the bug
When serializing a String ListArray slice(0, 1), the offsets of the inner StringArray is all serialized into the buffer. Imagine that if the inner StringArray is 10k in length, then the offset is (10k + 1) * 4 bytes, incurring significant overhead.

use arrow_array::{Array, ListArray, RecordBatch, StringArray};
use arrow_buffer::{Buffer, ToByteSlice};
use arrow_data::ArrayDataBuilder;
use arrow_ipc::writer::{DictionaryTracker, IpcDataGenerator, IpcWriteOptions};
use arrow_schema::{DataType, Field, Schema};
use std::sync::Arc;

fn main() {
    test_string_list();
}

fn test_string_list() {
    const INNER_STRING_ARRAY_LEN: usize = 8192 * 10;
    let rows: Vec<String> = std::iter::repeat("hello world".to_string())
        .take(INNER_STRING_ARRAY_LEN)
        .collect();
    let string_array = StringArray::from(rows);

    let list_array: ListArray = ArrayDataBuilder::new(DataType::List(Arc::new(Field::new(
        "item",
        DataType::Utf8,
        true,
    ))))
    .len(6)
    .add_buffer(Buffer::from(
        [0_i32, 1, 2, 3, 4, 5, INNER_STRING_ARRAY_LEN as _].to_byte_slice(),
    ))
    .add_child_data(string_array.into_data())
    .null_bit_buffer(None)
    .build()
    .unwrap()
    .into();

    let columns = vec![Arc::new(list_array) as _];

    let schema = Arc::new(Schema::new(vec![Field::new_list(
        "values",
        Field::new("item", DataType::Utf8, true),
        true,
    )]));
    let batch = RecordBatch::try_new(schema, columns).unwrap();
    encode(&batch);

    for i in 0..=6 {
        let slice = batch.slice(0, i);
        encode(&slice);
    }
}

fn encode(batch: &RecordBatch) {
    let option = IpcWriteOptions::default();
    let generator = IpcDataGenerator::default();
    let mut dict_tracker = DictionaryTracker::new(false);

    let (_, encode_data) = generator
        .encoded_batch(&batch, &mut dict_tracker, &option)
        .unwrap();

    println!(
        "batch slice len: {}, ipc_message len: {}, arrow data len: {}",
        batch.num_rows(),
        encode_data.ipc_message.len(),
        encode_data.arrow_data.len()
    );
}

batch slice len: 6, ipc_message len: 192, arrow data len: 1239232
batch slice len: 0, ipc_message len: 172, arrow data len: 0
batch slice len: 1, ipc_message len: 192, arrow data len: 328000
batch slice len: 2, ipc_message len: 192, arrow data len: 328000
batch slice len: 3, ipc_message len: 192, arrow data len: 328000
batch slice len: 4, ipc_message len: 192, arrow data len: 328000
batch slice len: 5, ipc_message len: 192, arrow data len: 328000
batch slice len: 6, ipc_message len: 192, arrow data len: 1239232

To Reproduce

Expected behavior

Additional context