Skip to content

Commit

Permalink
Relax check on patch_bits overflows in delta decoding (#118)
Browse files Browse the repository at this point in the history
For some reason, some files written with pyorc have a `patch_bit_width`
larger than needing, causing the previous check to fail, even when
decoding to `u64`.

This changes the check to only fail when the patch bits actually
overflow, instead of checking whether they may overflow.

Closes datafusion-contrib/datafusion-orc#97
  • Loading branch information
progval authored Aug 6, 2024
1 parent 85344c1 commit 56c9497
Showing 1 changed file with 12 additions and 7 deletions.
19 changes: 12 additions & 7 deletions src/reader/decode/rle_v2/patched_base.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ impl<N: NInt, R: Read> RleReaderV2<N, R> {
pub fn read_patched_base(&mut self, header: u8) -> Result<()> {
let encoded_bit_width = (header >> 1) & 0x1F;
let value_bit_width = rle_v2_decode_bit_width(encoded_bit_width);
let value_bit_width_u32 = u32::try_from(value_bit_width).or_else(|_| {
OutOfSpecSnafu {
msg: "value_bit_width overflows u32",
}
.fail()
})?;

let second_byte = read_u8(&mut self.reader)?;
let length = extract_run_length_from_header(header, second_byte);
Expand All @@ -52,12 +58,6 @@ impl<N: NInt, R: Read> RleReaderV2<N, R> {
}
.fail();
}
if (patch_bit_width + value_bit_width) > (N::BYTE_SIZE * 8) {
return OutOfSpecSnafu {
msg: "combined patch width and value width cannot exceed the size of the integer type being decoded",
}
.fail();
}

let patch_list_length = (fourth_byte & 0x1f) as usize;

Expand Down Expand Up @@ -105,7 +105,12 @@ impl<N: NInt, R: Read> RleReaderV2<N, R> {

for (idx, value) in self.decoded_ints.iter_mut().enumerate() {
if idx == actual_gap as usize {
let patch_bits = current_patch << value_bit_width;
let patch_bits =
current_patch
.checked_shl(value_bit_width_u32)
.context(OutOfSpecSnafu {
msg: "Overflow while shifting patch bits by value_bit_width",
})?;
// Safe conversion without loss as we check the bit width prior
let patch_bits = N::from_u64(patch_bits);
let patched_value = *value | patch_bits;
Expand Down

0 comments on commit 56c9497

Please sign in to comment.