From b28d202ef535b4312feb098e6e4786553fa46330 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Tue, 10 Sep 2024 15:51:53 -0400 Subject: [PATCH] GH-43956: [Format] Allow Decimal32/Decimal64 in format (#43976) ### Rationale for this change Widening the Decimal128/256 type to allow for bitwidths of 32 and 64 allows for more interoperability with other libraries and utilities which already support these types. This provides even more opportunities for zero-copy interactions between things such as libcudf and various databases. ### What changes are included in this PR? Updating the documentation in Schema.fbs to explicitly state that 32-bit and 64-bit is now allowed for bitwidths of Decimal types. This is the only area in the the spec that mentions the allowed decimal bitwidths. * GitHub Issue: #43956 --------- Co-authored-by: Antoine Pitrou --- docs/source/format/Columnar.rst | 2 +- docs/source/format/Versioning.rst | 5 +++++ format/Schema.fbs | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/source/format/Columnar.rst b/docs/source/format/Columnar.rst index 4c758c5294325..697c39b0cb1d9 100644 --- a/docs/source/format/Columnar.rst +++ b/docs/source/format/Columnar.rst @@ -21,7 +21,7 @@ Arrow Columnar Format ********************* -*Version: 1.4* +*Version: 1.5* .. seealso:: :ref:`Additions to the Arrow columnar format since version 1.0.0 ` diff --git a/docs/source/format/Versioning.rst b/docs/source/format/Versioning.rst index 8fcf11b21f0cc..d46d07a90906c 100644 --- a/docs/source/format/Versioning.rst +++ b/docs/source/format/Versioning.rst @@ -105,3 +105,8 @@ Version 1.4 * Added :ref:`listview-layout` and the associated ListView and LargeListView types. * Added :ref:`variadic-buffers`. + +Version 1.5 +----------- + +* Expanded Decimal type bit widths to allow 32-bit and 64-bit types. diff --git a/format/Schema.fbs b/format/Schema.fbs index a03ca31ae97c4..e8e14b112a771 100644 --- a/format/Schema.fbs +++ b/format/Schema.fbs @@ -24,6 +24,7 @@ /// Version 1.3 - Add Run-End Encoded. /// Version 1.4 - Add BinaryView, Utf8View, variadicBufferCounts, ListView, and /// LargeListView. +/// Version 1.5 - Add 32-bit and 64-bit as allowed bit widths for Decimal namespace org.apache.arrow.flatbuf; @@ -222,9 +223,9 @@ table RunEndEncoded { } /// Exact decimal value represented as an integer value in two's -/// complement. Currently only 128-bit (16-byte) and 256-bit (32-byte) integers -/// are used. The representation uses the endianness indicated -/// in the Schema. +/// complement. Currently 32-bit (4-byte), 64-bit (8-byte), +/// 128-bit (16-byte) and 256-bit (32-byte) integers are used. +/// The representation uses the endianness indicated in the Schema. table Decimal { /// Total number of decimal digits precision: int; @@ -232,7 +233,7 @@ table Decimal { /// Number of digits after the decimal point "." scale: int; - /// Number of bits per value. The only accepted widths are 128 and 256. + /// Number of bits per value. The accepted widths are 32, 64, 128 and 256. /// We use bitWidth for consistency with Int::bitWidth. bitWidth: int = 128; }