From 2132cb361c386ab25d72b4990772ca6e5c31312c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 11 Jan 2024 22:45:22 +0900 Subject: [PATCH 01/92] GH-39270: [C++] Avoid creating memory manager instance for every buffer view/copy (#39271) ### Rationale for this change We can use `arrow::default_cpu_memory_manager()` for `default_memory_pool()`. ### What changes are included in this PR? Check the given `pool` and use `arrow::default_cpu_memory_manager()` if it's `arrow::default_memory_pool()`. This also caches `arrow::CPUDevice::memory_manager()` result to avoid calling it multiple times. Note that we can avoid creating needless memory manager instance without this. This just avoid calling it multiple times. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #39270 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- cpp/src/arrow/device.cc | 6 +++++- cpp/src/arrow/ipc/message.cc | 20 +++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cpp/src/arrow/device.cc b/cpp/src/arrow/device.cc index 14d3bac0af1b7..de709923dc44e 100644 --- a/cpp/src/arrow/device.cc +++ b/cpp/src/arrow/device.cc @@ -241,7 +241,11 @@ bool CPUDevice::Equals(const Device& other) const { } std::shared_ptr CPUDevice::memory_manager(MemoryPool* pool) { - return CPUMemoryManager::Make(Instance(), pool); + if (pool == default_memory_pool()) { + return default_cpu_memory_manager(); + } else { + return CPUMemoryManager::Make(Instance(), pool); + } } std::shared_ptr CPUDevice::default_memory_manager() { diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index fbcd6f139b6d2..e196dd7bf5389 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -607,6 +607,7 @@ class MessageDecoder::MessageDecoderImpl { MemoryPool* pool, bool skip_body) : listener_(std::move(listener)), pool_(pool), + memory_manager_(CPUDevice::memory_manager(pool_)), state_(initial_state), next_required_size_(initial_next_required_size), chunks_(), @@ -822,8 +823,7 @@ class MessageDecoder::MessageDecoderImpl { if (buffer->is_cpu()) { metadata_ = buffer; } else { - ARROW_ASSIGN_OR_RAISE(metadata_, - Buffer::ViewOrCopy(buffer, CPUDevice::memory_manager(pool_))); + ARROW_ASSIGN_OR_RAISE(metadata_, Buffer::ViewOrCopy(buffer, memory_manager_)); } return ConsumeMetadata(); } @@ -834,16 +834,15 @@ class MessageDecoder::MessageDecoderImpl { if (chunks_[0]->is_cpu()) { metadata_ = std::move(chunks_[0]); } else { - ARROW_ASSIGN_OR_RAISE( - metadata_, - Buffer::ViewOrCopy(chunks_[0], CPUDevice::memory_manager(pool_))); + ARROW_ASSIGN_OR_RAISE(metadata_, + Buffer::ViewOrCopy(chunks_[0], memory_manager_)); } chunks_.erase(chunks_.begin()); } else { metadata_ = SliceBuffer(chunks_[0], 0, next_required_size_); if (!chunks_[0]->is_cpu()) { - ARROW_ASSIGN_OR_RAISE( - metadata_, Buffer::ViewOrCopy(metadata_, CPUDevice::memory_manager(pool_))); + ARROW_ASSIGN_OR_RAISE(metadata_, + Buffer::ViewOrCopy(metadata_, memory_manager_)); } chunks_[0] = SliceBuffer(chunks_[0], next_required_size_); } @@ -911,8 +910,7 @@ class MessageDecoder::MessageDecoderImpl { if (buffer->is_cpu()) { return util::SafeLoadAs(buffer->data()); } else { - ARROW_ASSIGN_OR_RAISE(auto cpu_buffer, - Buffer::ViewOrCopy(buffer, CPUDevice::memory_manager(pool_))); + ARROW_ASSIGN_OR_RAISE(auto cpu_buffer, Buffer::ViewOrCopy(buffer, memory_manager_)); return util::SafeLoadAs(cpu_buffer->data()); } } @@ -924,8 +922,7 @@ class MessageDecoder::MessageDecoderImpl { std::shared_ptr last_chunk; for (auto& chunk : chunks_) { if (!chunk->is_cpu()) { - ARROW_ASSIGN_OR_RAISE( - chunk, Buffer::ViewOrCopy(chunk, CPUDevice::memory_manager(pool_))); + ARROW_ASSIGN_OR_RAISE(chunk, Buffer::ViewOrCopy(chunk, memory_manager_)); } auto data = chunk->data(); auto data_size = chunk->size(); @@ -951,6 +948,7 @@ class MessageDecoder::MessageDecoderImpl { std::shared_ptr listener_; MemoryPool* pool_; + std::shared_ptr memory_manager_; State state_; int64_t next_required_size_; std::vector> chunks_; From 04afe3f017668d7fbd74301717ce467394e216d7 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Thu, 11 Jan 2024 22:50:56 +0900 Subject: [PATCH 02/92] GH-39427: [GLib] Update script and documentation (#39428) ### Rationale for this change Modify scripts and update documentation for building GLib and running test code in a Mac environment. ### What changes are included in this PR? * Update Documentation (`c_glib/README.md`) * Update Script (`c_glib/test/run-test.sh`) ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. * Closes: #39427 Lead-authored-by: Hyunseok Seo Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- c_glib/README.md | 22 +++++++++++++++++++++- c_glib/test/run-test.sh | 23 +++++++++++++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/c_glib/README.md b/c_glib/README.md index d571053c3dce8..2a4d6b8a6628c 100644 --- a/c_glib/README.md +++ b/c_glib/README.md @@ -142,6 +142,17 @@ $ meson compile -C c_glib.build $ sudo meson install -C c_glib.build ``` +> [!WARNING] +> +> When building Arrow GLib, it typically uses the Arrow C++ installed via Homebrew. However, this can lead to build failures +> if there are mismatches between the changes in Arrow's GLib and C++ libraries. To resolve this, you may need to +> reference the Arrow C++ library built locally. In such cases, use the `--cmake-prefix-path` option with the `meson setup` +> command to explicitly specify the library path. +> +> ```console +> $ meson setup c_glib.build c_glib --cmake-prefix-path=${arrow_cpp_install_prefix} -Dgtk_doc=true +> ``` + Others: ```console @@ -231,9 +242,18 @@ Now, you can run unit tests by the followings: ```console $ cd c_glib.build -$ bundle exec ../c_glib/test/run-test.sh +$ BUNDLE_GEMFILE=../c_glib/Gemfile bundle exec ../c_glib/test/run-test.sh ``` + +> [!NOTE] +> +> If debugging is necessary, you can proceed using the `DEBUGGER` option as follows: +> +> ```console +> $ DEBUGGER=lldb BUNDLE_GEMFILE=../c_glib/Gemfile bundle exec ../c_glib/test/run-test.sh +> ``` + ## Common build problems ### build failed - /usr/bin/ld: cannot find -larrow diff --git a/c_glib/test/run-test.sh b/c_glib/test/run-test.sh index 33e9fbf85d026..c7bc6edca5f0d 100755 --- a/c_glib/test/run-test.sh +++ b/c_glib/test/run-test.sh @@ -34,14 +34,14 @@ for module in "${modules[@]}"; do module_build_dir="${build_dir}/${module}" if [ -d "${module_build_dir}" ]; then LD_LIBRARY_PATH="${module_build_dir}:${LD_LIBRARY_PATH}" + DYLD_LIBRARY_PATH="${module_build_dir}:${DYLD_LIBRARY_PATH}" fi done export LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH if [ "${BUILD}" != "no" ]; then - if [ -f "Makefile" ]; then - make -j8 > /dev/null || exit $? - elif [ -f "build.ninja" ]; then + if [ -f "build.ninja" ]; then ninja || exit $? fi fi @@ -59,4 +59,19 @@ for module in "${modules[@]}"; do done export GI_TYPELIB_PATH -${GDB} ruby ${test_dir}/run-test.rb "$@" +if type rbenv > /dev/null 2>&1; then + RUBY="$(rbenv which ruby)" +else + RUBY=ruby +fi +DEBUGGER_ARGS=() +case "${DEBUGGER}" in + "gdb") + DEBUGGER_ARGS+=(--args) + ;; + "lldb") + DEBUGGER_ARGS+=(--one-line "env DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}") + DEBUGGER_ARGS+=(--) + ;; +esac +${DEBUGGER} "${DEBUGGER_ARGS[@]}" "${RUBY}" ${test_dir}/run-test.rb "$@" From 5709c0dbb52e2075bd89a4fde69030c3eac385dd Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 11 Jan 2024 23:38:41 +0800 Subject: [PATCH 03/92] GH-39233: [Compute] Add some duration kernels (#39358) ### Rationale for this change Add kernels for durations. ### What changes are included in this PR? In this PR I added the ones that require only registration and unit tests. More complicated ones will be in another PR for readability. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #39233 Authored-by: Jin Shang Signed-off-by: Antoine Pitrou --- .../compute/kernels/scalar_arithmetic.cc | 35 +++++++++ .../arrow/compute/kernels/scalar_compare.cc | 9 +++ .../compute/kernels/scalar_compare_test.cc | 7 +- .../compute/kernels/scalar_temporal_test.cc | 12 +++ .../arrow/compute/kernels/scalar_validity.cc | 6 +- .../compute/kernels/scalar_validity_test.cc | 7 ++ docs/source/cpp/compute.rst | 78 +++++++++---------- 7 files changed, 113 insertions(+), 41 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc b/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc index ad33d7f8951f4..44f5fea79078a 100644 --- a/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc +++ b/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc @@ -1286,12 +1286,27 @@ void RegisterScalarArithmetic(FunctionRegistry* registry) { auto absolute_value = MakeUnaryArithmeticFunction("abs", absolute_value_doc); AddDecimalUnaryKernels(absolute_value.get()); + + // abs(duration) + for (auto unit : TimeUnit::values()) { + auto exec = ArithmeticExecFromOp(duration(unit)); + DCHECK_OK( + absolute_value->AddKernel({duration(unit)}, OutputType(duration(unit)), exec)); + } + DCHECK_OK(registry->AddFunction(std::move(absolute_value))); // ---------------------------------------------------------------------- auto absolute_value_checked = MakeUnaryArithmeticFunctionNotNull( "abs_checked", absolute_value_checked_doc); AddDecimalUnaryKernels(absolute_value_checked.get()); + // abs_checked(duraton) + for (auto unit : TimeUnit::values()) { + auto exec = + ArithmeticExecFromOp(duration(unit)); + DCHECK_OK(absolute_value_checked->AddKernel({duration(unit)}, + OutputType(duration(unit)), exec)); + } DCHECK_OK(registry->AddFunction(std::move(absolute_value_checked))); // ---------------------------------------------------------------------- @@ -1545,12 +1560,27 @@ void RegisterScalarArithmetic(FunctionRegistry* registry) { // ---------------------------------------------------------------------- auto negate = MakeUnaryArithmeticFunction("negate", negate_doc); AddDecimalUnaryKernels(negate.get()); + + // Add neg(duration) -> duration + for (auto unit : TimeUnit::values()) { + auto exec = ArithmeticExecFromOp(duration(unit)); + DCHECK_OK(negate->AddKernel({duration(unit)}, OutputType(duration(unit)), exec)); + } + DCHECK_OK(registry->AddFunction(std::move(negate))); // ---------------------------------------------------------------------- auto negate_checked = MakeUnarySignedArithmeticFunctionNotNull( "negate_checked", negate_checked_doc); AddDecimalUnaryKernels(negate_checked.get()); + + // Add neg_checked(duration) -> duration + for (auto unit : TimeUnit::values()) { + auto exec = ArithmeticExecFromOp(duration(unit)); + DCHECK_OK( + negate_checked->AddKernel({duration(unit)}, OutputType(duration(unit)), exec)); + } + DCHECK_OK(registry->AddFunction(std::move(negate_checked))); // ---------------------------------------------------------------------- @@ -1581,6 +1611,11 @@ void RegisterScalarArithmetic(FunctionRegistry* registry) { // ---------------------------------------------------------------------- auto sign = MakeUnaryArithmeticFunctionWithFixedIntOutType("sign", sign_doc); + // sign(duration) + for (auto unit : TimeUnit::values()) { + auto exec = ScalarUnary::Exec; + DCHECK_OK(sign->AddKernel({duration(unit)}, int8(), std::move(exec))); + } DCHECK_OK(registry->AddFunction(std::move(sign))); // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/compute/kernels/scalar_compare.cc b/cpp/src/arrow/compute/kernels/scalar_compare.cc index aad648ca275c3..daf8ed76d628d 100644 --- a/cpp/src/arrow/compute/kernels/scalar_compare.cc +++ b/cpp/src/arrow/compute/kernels/scalar_compare.cc @@ -22,6 +22,7 @@ #include "arrow/compute/api_scalar.h" #include "arrow/compute/kernels/common_internal.h" +#include "arrow/type.h" #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_ops.h" @@ -806,6 +807,14 @@ std::shared_ptr MakeScalarMinMax(std::string name, FunctionDoc d kernel.mem_allocation = MemAllocation::type::PREALLOCATE; DCHECK_OK(func->AddKernel(std::move(kernel))); } + for (const auto& ty : DurationTypes()) { + auto exec = GeneratePhysicalNumeric(ty); + ScalarKernel kernel{KernelSignature::Make({ty}, ty, /*is_varargs=*/true), exec, + MinMaxState::Init}; + kernel.null_handling = NullHandling::type::COMPUTED_NO_PREALLOCATE; + kernel.mem_allocation = MemAllocation::type::PREALLOCATE; + DCHECK_OK(func->AddKernel(std::move(kernel))); + } for (const auto& ty : BaseBinaryTypes()) { auto exec = GenerateTypeAgnosticVarBinaryBase(ty); diff --git a/cpp/src/arrow/compute/kernels/scalar_compare_test.cc b/cpp/src/arrow/compute/kernels/scalar_compare_test.cc index 48fa780b03104..8f5952b40500a 100644 --- a/cpp/src/arrow/compute/kernels/scalar_compare_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_compare_test.cc @@ -1281,7 +1281,7 @@ using CompareNumericBasedTypes = ::testing::Types; using CompareParametricTemporalTypes = - ::testing::Types; + ::testing::Types; using CompareFixedSizeBinaryTypes = ::testing::Types; TYPED_TEST_SUITE(TestVarArgsCompareNumeric, CompareNumericBasedTypes); @@ -2121,6 +2121,11 @@ TEST(TestMaxElementWiseMinElementWise, CommonTemporal) { ScalarFromJSON(date64(), "172800000"), }), ResultWith(ScalarFromJSON(date64(), "86400000"))); + EXPECT_THAT(MinElementWise({ + ScalarFromJSON(duration(TimeUnit::SECOND), "1"), + ScalarFromJSON(duration(TimeUnit::MILLI), "12000"), + }), + ResultWith(ScalarFromJSON(duration(TimeUnit::MILLI), "1000"))); } } // namespace compute diff --git a/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc b/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc index d4482334285bc..8dac6525fe2e6 100644 --- a/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_temporal_test.cc @@ -3665,5 +3665,17 @@ TEST_F(ScalarTemporalTest, TestCeilFloorRoundTemporalDate) { CheckScalarUnary("ceil_temporal", arr_ns, arr_ns, &round_to_2_hours); } +TEST_F(ScalarTemporalTest, DurationUnaryArithmetics) { + auto arr = ArrayFromJSON(duration(TimeUnit::SECOND), "[2, -1, null, 3, 0]"); + CheckScalarUnary("negate", arr, + ArrayFromJSON(duration(TimeUnit::SECOND), "[-2, 1, null, -3, 0]")); + CheckScalarUnary("negate_checked", arr, + ArrayFromJSON(duration(TimeUnit::SECOND), "[-2, 1, null, -3, 0]")); + CheckScalarUnary("abs", arr, + ArrayFromJSON(duration(TimeUnit::SECOND), "[2, 1, null, 3, 0]")); + CheckScalarUnary("abs_checked", arr, + ArrayFromJSON(duration(TimeUnit::SECOND), "[2, 1, null, 3, 0]")); + CheckScalarUnary("sign", arr, ArrayFromJSON(int8(), "[1, -1, null, 1, 0]")); +} } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/scalar_validity.cc b/cpp/src/arrow/compute/kernels/scalar_validity.cc index 6b1cec0f5ccc6..8505fc4c6e0af 100644 --- a/cpp/src/arrow/compute/kernels/scalar_validity.cc +++ b/cpp/src/arrow/compute/kernels/scalar_validity.cc @@ -169,6 +169,7 @@ std::shared_ptr MakeIsFiniteFunction(std::string name, FunctionD func->AddKernel({InputType(Type::DECIMAL128)}, boolean(), ConstBoolExec)); DCHECK_OK( func->AddKernel({InputType(Type::DECIMAL256)}, boolean(), ConstBoolExec)); + DCHECK_OK(func->AddKernel({InputType(Type::DURATION)}, boolean(), ConstBoolExec)); return func; } @@ -187,7 +188,8 @@ std::shared_ptr MakeIsInfFunction(std::string name, FunctionDoc func->AddKernel({InputType(Type::DECIMAL128)}, boolean(), ConstBoolExec)); DCHECK_OK( func->AddKernel({InputType(Type::DECIMAL256)}, boolean(), ConstBoolExec)); - + DCHECK_OK( + func->AddKernel({InputType(Type::DURATION)}, boolean(), ConstBoolExec)); return func; } @@ -205,6 +207,8 @@ std::shared_ptr MakeIsNanFunction(std::string name, FunctionDoc func->AddKernel({InputType(Type::DECIMAL128)}, boolean(), ConstBoolExec)); DCHECK_OK( func->AddKernel({InputType(Type::DECIMAL256)}, boolean(), ConstBoolExec)); + DCHECK_OK( + func->AddKernel({InputType(Type::DURATION)}, boolean(), ConstBoolExec)); return func; } diff --git a/cpp/src/arrow/compute/kernels/scalar_validity_test.cc b/cpp/src/arrow/compute/kernels/scalar_validity_test.cc index 94d951c838209..d1462838f3be6 100644 --- a/cpp/src/arrow/compute/kernels/scalar_validity_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_validity_test.cc @@ -103,6 +103,9 @@ TEST(TestValidityKernels, IsFinite) { } CheckScalar("is_finite", {std::make_shared(4)}, ArrayFromJSON(boolean(), "[null, null, null, null]")); + CheckScalar("is_finite", + {ArrayFromJSON(duration(TimeUnit::SECOND), "[0, 1, 42, null]")}, + ArrayFromJSON(boolean(), "[true, true, true, null]")); } TEST(TestValidityKernels, IsInf) { @@ -116,6 +119,8 @@ TEST(TestValidityKernels, IsInf) { } CheckScalar("is_inf", {std::make_shared(4)}, ArrayFromJSON(boolean(), "[null, null, null, null]")); + CheckScalar("is_inf", {ArrayFromJSON(duration(TimeUnit::SECOND), "[0, 1, 42, null]")}, + ArrayFromJSON(boolean(), "[false, false, false, null]")); } TEST(TestValidityKernels, IsNan) { @@ -129,6 +134,8 @@ TEST(TestValidityKernels, IsNan) { } CheckScalar("is_nan", {std::make_shared(4)}, ArrayFromJSON(boolean(), "[null, null, null, null]")); + CheckScalar("is_nan", {ArrayFromJSON(duration(TimeUnit::SECOND), "[0, 1, 42, null]")}, + ArrayFromJSON(boolean(), "[false, false, false, null]")); } TEST(TestValidityKernels, IsValidIsNullNullType) { diff --git a/docs/source/cpp/compute.rst b/docs/source/cpp/compute.rst index 17d003b261dca..e7310d2c0c711 100644 --- a/docs/source/cpp/compute.rst +++ b/docs/source/cpp/compute.rst @@ -458,45 +458,45 @@ floating-point arguments will cast all arguments to floating-point, while mixed decimal and integer arguments will cast all arguments to decimals. Mixed time resolution temporal inputs will be cast to finest input resolution. -+------------------+--------+-------------------------+----------------------+-------+ -| Function name | Arity | Input types | Output type | Notes | -+==================+========+=========================+======================+=======+ -| abs | Unary | Numeric | Numeric | | -+------------------+--------+-------------------------+----------------------+-------+ -| abs_checked | Unary | Numeric | Numeric | | -+------------------+--------+-------------------------+----------------------+-------+ -| add | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | -+------------------+--------+-------------------------+----------------------+-------+ -| add_checked | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | -+------------------+--------+-------------------------+----------------------+-------+ -| divide | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | -+------------------+--------+-------------------------+----------------------+-------+ -| divide_checked | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | -+------------------+--------+-------------------------+----------------------+-------+ -| exp | Unary | Numeric | Float32/Float64 | | -+------------------+--------+-------------------------+----------------------+-------+ -| multiply | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | -+------------------+--------+-------------------------+----------------------+-------+ -| multiply_checked | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | -+------------------+--------+-------------------------+----------------------+-------+ -| negate | Unary | Numeric | Numeric | | -+------------------+--------+-------------------------+----------------------+-------+ -| negate_checked | Unary | Signed Numeric | Signed Numeric | | -+------------------+--------+-------------------------+----------------------+-------+ -| power | Binary | Numeric | Numeric | | -+------------------+--------+-------------------------+----------------------+-------+ -| power_checked | Binary | Numeric | Numeric | | -+------------------+--------+-------------------------+----------------------+-------+ -| sign | Unary | Numeric | Int8/Float32/Float64 | \(2) | -+------------------+--------+-------------------------+----------------------+-------+ -| sqrt | Unary | Numeric | Numeric | | -+------------------+--------+-------------------------+----------------------+-------+ -| sqrt_checked | Unary | Numeric | Numeric | | -+------------------+--------+-------------------------+----------------------+-------+ -| subtract | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | -+------------------+--------+-------------------------+----------------------+-------+ -| subtract_checked | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | -+------------------+--------+-------------------------+----------------------+-------+ ++------------------+--------+-------------------------+---------------------------+-------+ +| Function name | Arity | Input types | Output type | Notes | ++==================+========+=========================+===========================+=======+ +| abs | Unary | Numeric/Duration | Numeric/Duration | | ++------------------+--------+-------------------------+---------------------------+-------+ +| abs_checked | Unary | Numeric/Duration | Numeric/Duration | | ++------------------+--------+-------------------------+---------------------------+-------+ +| add | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | ++------------------+--------+-------------------------+---------------------------+-------+ +| add_checked | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | ++------------------+--------+-------------------------+---------------------------+-------+ +| divide | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | ++------------------+--------+-------------------------+---------------------------+-------+ +| divide_checked | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | ++------------------+--------+-------------------------+---------------------------+-------+ +| exp | Unary | Numeric | Float32/Float64 | | ++------------------+--------+-------------------------+---------------------------+-------+ +| multiply | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | ++------------------+--------+-------------------------+---------------------------+-------+ +| multiply_checked | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | ++------------------+--------+-------------------------+---------------------------+-------+ +| negate | Unary | Numeric/Duration | Numeric/Duration | | ++------------------+--------+-------------------------+---------------------------+-------+ +| negate_checked | Unary | Signed Numeric/Duration | Signed Numeric/Duration | | ++------------------+--------+-------------------------+---------------------------+-------+ +| power | Binary | Numeric | Numeric | | ++------------------+--------+-------------------------+---------------------------+-------+ +| power_checked | Binary | Numeric | Numeric | | ++------------------+--------+-------------------------+---------------------------+-------+ +| sign | Unary | Numeric/Duration | Int8/Float32/Float64 | \(2) | ++------------------+--------+-------------------------+---------------------------+-------+ +| sqrt | Unary | Numeric | Numeric | | ++------------------+--------+-------------------------+---------------------------+-------+ +| sqrt_checked | Unary | Numeric | Numeric | | ++------------------+--------+-------------------------+---------------------------+-------+ +| subtract | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | ++------------------+--------+-------------------------+---------------------------+-------+ +| subtract_checked | Binary | Numeric/Temporal | Numeric/Temporal | \(1) | ++------------------+--------+-------------------------+---------------------------+-------+ * \(1) Precision and scale of computed DECIMAL results From d18107245e8e82a8d7ec40e0ae27f083ffbb7cc4 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Thu, 11 Jan 2024 23:40:49 +0800 Subject: [PATCH 04/92] GH-38833: [C++] Avoid hash_mean overflow (#39349) ### Rationale for this change hash_mean overflows if the sum of a group is larger than uint64 max. ### What changes are included in this PR? Save the intermediate sum as double to avoid overflow ### Are these changes tested? yes ### Are there any user-facing changes? no * Closes: #38833 Authored-by: Jin Shang Signed-off-by: Antoine Pitrou --- cpp/src/arrow/acero/hash_aggregate_test.cc | 36 +++++++++++++++++++ .../arrow/compute/kernels/hash_aggregate.cc | 24 +++++++++---- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/acero/hash_aggregate_test.cc b/cpp/src/arrow/acero/hash_aggregate_test.cc index a4874f3581040..2626fd50379dd 100644 --- a/cpp/src/arrow/acero/hash_aggregate_test.cc +++ b/cpp/src/arrow/acero/hash_aggregate_test.cc @@ -1694,6 +1694,42 @@ TEST_P(GroupBy, SumMeanProductScalar) { } } +TEST_P(GroupBy, MeanOverflow) { + BatchesWithSchema input; + // would overflow if intermediate sum is integer + input.batches = { + ExecBatchFromJSON({int64(), int64()}, {ArgShape::SCALAR, ArgShape::ARRAY}, + + "[[9223372036854775805, 1], [9223372036854775805, 1], " + "[9223372036854775805, 2], [9223372036854775805, 3]]"), + ExecBatchFromJSON({int64(), int64()}, {ArgShape::SCALAR, ArgShape::ARRAY}, + "[[null, 1], [null, 1], [null, 2], [null, 3]]"), + ExecBatchFromJSON({int64(), int64()}, + "[[9223372036854775805, 1], [9223372036854775805, 2], " + "[9223372036854775805, 3]]"), + }; + input.schema = schema({field("argument", int64()), field("key", int64())}); + for (bool use_threads : {true, false}) { + SCOPED_TRACE(use_threads ? "parallel/merged" : "serial"); + ASSERT_OK_AND_ASSIGN(Datum actual, + RunGroupBy(input, {"key"}, + { + {"hash_mean", nullptr, "argument", "hash_mean"}, + }, + use_threads)); + Datum expected = ArrayFromJSON(struct_({ + field("key", int64()), + field("hash_mean", float64()), + }), + R"([ + [1, 9223372036854775805], + [2, 9223372036854775805], + [3, 9223372036854775805] + ])"); + AssertDatumsApproxEqual(expected, actual, /*verbose=*/true); + } +} + TEST_P(GroupBy, VarianceAndStddev) { auto batch = RecordBatchFromJSON( schema({field("argument", int32()), field("key", int64())}), R"([ diff --git a/cpp/src/arrow/compute/kernels/hash_aggregate.cc b/cpp/src/arrow/compute/kernels/hash_aggregate.cc index c37e45513d040..5052d8dd66694 100644 --- a/cpp/src/arrow/compute/kernels/hash_aggregate.cc +++ b/cpp/src/arrow/compute/kernels/hash_aggregate.cc @@ -38,6 +38,7 @@ #include "arrow/compute/row/grouper.h" #include "arrow/record_batch.h" #include "arrow/stl_allocator.h" +#include "arrow/type_traits.h" #include "arrow/util/bit_run_reader.h" #include "arrow/util/bitmap_ops.h" #include "arrow/util/bitmap_writer.h" @@ -441,9 +442,10 @@ struct GroupedCountImpl : public GroupedAggregator { // ---------------------------------------------------------------------- // Sum/Mean/Product implementation -template +template ::Type> struct GroupedReducingAggregator : public GroupedAggregator { - using AccType = typename FindAccumulatorType::Type; + using AccType = AccumulateType; using CType = typename TypeTraits::CType; using InputCType = typename TypeTraits::CType; @@ -483,7 +485,8 @@ struct GroupedReducingAggregator : public GroupedAggregator { Status Merge(GroupedAggregator&& raw_other, const ArrayData& group_id_mapping) override { - auto other = checked_cast*>(&raw_other); + auto other = + checked_cast*>(&raw_other); CType* reduced = reduced_.mutable_data(); int64_t* counts = counts_.mutable_data(); @@ -733,9 +736,18 @@ using GroupedProductFactory = // ---------------------------------------------------------------------- // Mean implementation +template +struct GroupedMeanAccType { + using Type = typename std::conditional::value, DoubleType, + typename FindAccumulatorType::Type>::type; +}; + template -struct GroupedMeanImpl : public GroupedReducingAggregator> { - using Base = GroupedReducingAggregator>; +struct GroupedMeanImpl + : public GroupedReducingAggregator, + typename GroupedMeanAccType::Type> { + using Base = GroupedReducingAggregator, + typename GroupedMeanAccType::Type>; using CType = typename Base::CType; using InputCType = typename Base::InputCType; using MeanType = @@ -746,7 +758,7 @@ struct GroupedMeanImpl : public GroupedReducingAggregator static enable_if_number Reduce(const DataType&, const CType u, const InputCType v) { - return static_cast(to_unsigned(u) + to_unsigned(static_cast(v))); + return static_cast(u) + static_cast(v); } static CType Reduce(const DataType&, const CType u, const CType v) { From adef53715b5f0c908dc30acfb9ed99722481cd56 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 11 Jan 2024 18:12:59 +0100 Subject: [PATCH 05/92] GH-39549: [C++] Pass -jN to make in external projects (#39550) ### Rationale for this change Previous issues with sub-make fragility are no longer an issue with our new minimum CMake version 3.16. ### What changes are included in this PR? Remove special casing from jemalloc, pass -jN to all make based eps. ### Are these changes tested? CI * Closes: #39549 Authored-by: Jacob Wujciak-Jens Signed-off-by: Jacob Wujciak-Jens --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a2627c190f738..6bb9c0f6af2ca 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1005,14 +1005,8 @@ if("${MAKE}" STREQUAL "") endif() endif() -# Using make -j in sub-make is fragile -# see discussion https://github.com/apache/arrow/pull/2779 -if(${CMAKE_GENERATOR} MATCHES "Makefiles") - set(MAKE_BUILD_ARGS "") -else() - # limit the maximum number of jobs for ninja - set(MAKE_BUILD_ARGS "-j${NPROC}") -endif() +# Args for external projects using make. +set(MAKE_BUILD_ARGS "-j${NPROC}") include(FetchContent) set(FC_DECLARE_COMMON_OPTIONS) @@ -2042,10 +2036,6 @@ macro(build_jemalloc) endif() set(JEMALLOC_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}) - # Paralleism for Make fails with CMake > 3.28 see #39517 - if(${CMAKE_GENERATOR} MATCHES "Makefiles") - list(APPEND JEMALLOC_BUILD_COMMAND "-j1") - endif() if(CMAKE_OSX_SYSROOT) list(APPEND JEMALLOC_BUILD_COMMAND "SDKROOT=${CMAKE_OSX_SYSROOT}") From c009024ab6d29ff834d5ac2e7b27675ddda0a47a Mon Sep 17 00:00:00 2001 From: emkornfield Date: Thu, 11 Jan 2024 09:40:51 -0800 Subject: [PATCH 06/92] GH-39525: [C++][Parquet] Pass memory pool to decoders (#39526) ### Rationale for this change Memory pools should be plumbed through where ever possible. ### What changes are included in this PR? Pass through memory pool to decoders ### Are these changes tested? Not directly; this was caught via some internal fuzz targets. ### Are there any user-facing changes? No. * Closes: #39525 Authored-by: Micah Kornfield Signed-off-by: mwish --- cpp/src/parquet/column_reader.cc | 44 +++++++------------------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/cpp/src/parquet/column_reader.cc b/cpp/src/parquet/column_reader.cc index f5d9734aa1e01..ac4627d69c0f6 100644 --- a/cpp/src/parquet/column_reader.cc +++ b/cpp/src/parquet/column_reader.cc @@ -760,7 +760,7 @@ class ColumnReaderImplBase { if (page->encoding() == Encoding::PLAIN_DICTIONARY || page->encoding() == Encoding::PLAIN) { - auto dictionary = MakeTypedDecoder(Encoding::PLAIN, descr_); + auto dictionary = MakeTypedDecoder(Encoding::PLAIN, descr_, pool_); dictionary->SetData(page->num_values(), page->data(), page->size()); // The dictionary is fully decoded during DictionaryDecoder::Init, so the @@ -883,47 +883,21 @@ class ColumnReaderImplBase { current_decoder_ = it->second.get(); } else { switch (encoding) { - case Encoding::PLAIN: { - auto decoder = MakeTypedDecoder(Encoding::PLAIN, descr_); - current_decoder_ = decoder.get(); - decoders_[static_cast(encoding)] = std::move(decoder); - break; - } - case Encoding::BYTE_STREAM_SPLIT: { - auto decoder = MakeTypedDecoder(Encoding::BYTE_STREAM_SPLIT, descr_); - current_decoder_ = decoder.get(); - decoders_[static_cast(encoding)] = std::move(decoder); - break; - } - case Encoding::RLE: { - auto decoder = MakeTypedDecoder(Encoding::RLE, descr_); + case Encoding::PLAIN: + case Encoding::BYTE_STREAM_SPLIT: + case Encoding::RLE: + case Encoding::DELTA_BINARY_PACKED: + case Encoding::DELTA_BYTE_ARRAY: + case Encoding::DELTA_LENGTH_BYTE_ARRAY: { + auto decoder = MakeTypedDecoder(encoding, descr_, pool_); current_decoder_ = decoder.get(); decoders_[static_cast(encoding)] = std::move(decoder); break; } + case Encoding::RLE_DICTIONARY: throw ParquetException("Dictionary page must be before data page."); - case Encoding::DELTA_BINARY_PACKED: { - auto decoder = MakeTypedDecoder(Encoding::DELTA_BINARY_PACKED, descr_); - current_decoder_ = decoder.get(); - decoders_[static_cast(encoding)] = std::move(decoder); - break; - } - case Encoding::DELTA_BYTE_ARRAY: { - auto decoder = MakeTypedDecoder(Encoding::DELTA_BYTE_ARRAY, descr_); - current_decoder_ = decoder.get(); - decoders_[static_cast(encoding)] = std::move(decoder); - break; - } - case Encoding::DELTA_LENGTH_BYTE_ARRAY: { - auto decoder = - MakeTypedDecoder(Encoding::DELTA_LENGTH_BYTE_ARRAY, descr_); - current_decoder_ = decoder.get(); - decoders_[static_cast(encoding)] = std::move(decoder); - break; - } - default: throw ParquetException("Unknown encoding type."); } From 2b4a70320232647f730b19d2fea5746c3baec752 Mon Sep 17 00:00:00 2001 From: Jin Shang Date: Fri, 12 Jan 2024 01:56:46 +0800 Subject: [PATCH 07/92] GH-39231: [C++][Compute] Add binary_slice kernel for fixed size binary (#39245) ### Rationale for this change Add binary_slice kernel for fixed size binary ### What changes are included in this PR? Add binary_slice kernel for fixed size binary ### Are these changes tested? Yes ### Are there any user-facing changes? No * Closes: #39231 Lead-authored-by: Jin Shang Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- .../compute/kernels/scalar_string_ascii.cc | 117 ++++++++++---- .../compute/kernels/scalar_string_internal.h | 2 + .../compute/kernels/scalar_string_test.cc | 146 ++++++++++++++++-- python/pyarrow/tests/test_compute.py | 10 +- 4 files changed, 233 insertions(+), 42 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_string_ascii.cc b/cpp/src/arrow/compute/kernels/scalar_string_ascii.cc index 6764845dfca81..8fdc6172aa6d3 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_ascii.cc +++ b/cpp/src/arrow/compute/kernels/scalar_string_ascii.cc @@ -95,7 +95,7 @@ struct FixedSizeBinaryTransformExecBase { ctx->Allocate(output_width * input_nstrings)); uint8_t* output_str = values_buffer->mutable_data(); - const uint8_t* input_data = input.GetValues(1); + const uint8_t* input_data = input.GetValues(1, input.offset * input_width); for (int64_t i = 0; i < input_nstrings; i++) { if (!input.IsNull(i)) { const uint8_t* input_string = input_data + i * input_width; @@ -132,7 +132,8 @@ struct FixedSizeBinaryTransformExecWithState DCHECK_EQ(1, types.size()); const auto& options = State::Get(ctx); const int32_t input_width = types[0].type->byte_width(); - const int32_t output_width = StringTransform::FixedOutputSize(options, input_width); + ARROW_ASSIGN_OR_RAISE(const int32_t output_width, + StringTransform::FixedOutputSize(options, input_width)); return fixed_size_binary(output_width); } }; @@ -2377,7 +2378,8 @@ struct BinaryReplaceSliceTransform : ReplaceStringSliceTransformBase { return output - output_start; } - static int32_t FixedOutputSize(const ReplaceSliceOptions& opts, int32_t input_width) { + static Result FixedOutputSize(const ReplaceSliceOptions& opts, + int32_t input_width) { int32_t before_slice = 0; int32_t after_slice = 0; const int32_t start = static_cast(opts.start); @@ -2436,6 +2438,7 @@ void AddAsciiStringReplaceSlice(FunctionRegistry* registry) { namespace { struct SliceBytesTransform : StringSliceTransformBase { + using StringSliceTransformBase::StringSliceTransformBase; int64_t MaxCodeunits(int64_t ninputs, int64_t input_bytes) override { const SliceOptions& opt = *this->options; if ((opt.start >= 0) != (opt.stop >= 0)) { @@ -2454,22 +2457,15 @@ struct SliceBytesTransform : StringSliceTransformBase { return SliceBackward(input, input_string_bytes, output); } - int64_t SliceForward(const uint8_t* input, int64_t input_string_bytes, - uint8_t* output) { - // Slice in forward order (step > 0) - const SliceOptions& opt = *this->options; - const uint8_t* begin = input; - const uint8_t* end = input + input_string_bytes; - const uint8_t* begin_sliced; - const uint8_t* end_sliced; - - if (!input_string_bytes) { - return 0; - } - // First, compute begin_sliced and end_sliced + static std::pair SliceForwardRange(const SliceOptions& opt, + int64_t input_string_bytes) { + int64_t begin = 0; + int64_t end = input_string_bytes; + int64_t begin_sliced = 0; + int64_t end_sliced = 0; if (opt.start >= 0) { // start counting from the left - begin_sliced = std::min(begin + opt.start, end); + begin_sliced = std::min(opt.start, end); if (opt.stop > opt.start) { // continue counting from begin_sliced const int64_t length = opt.stop - opt.start; @@ -2479,7 +2475,7 @@ struct SliceBytesTransform : StringSliceTransformBase { end_sliced = std::max(end + opt.stop, begin_sliced); } else { // zero length slice - return 0; + return {0, 0}; } } else { // start counting from the right @@ -2491,7 +2487,7 @@ struct SliceBytesTransform : StringSliceTransformBase { // and therefore we also need this if (end_sliced <= begin_sliced) { // zero length slice - return 0; + return {0, 0}; } } else if ((opt.stop < 0) && (opt.stop > opt.start)) { // stop is negative, but larger than start, so we count again from the right @@ -2501,12 +2497,30 @@ struct SliceBytesTransform : StringSliceTransformBase { end_sliced = std::max(end + opt.stop, begin_sliced); } else { // zero length slice - return 0; + return {0, 0}; } } + return {begin_sliced, end_sliced}; + } + + int64_t SliceForward(const uint8_t* input, int64_t input_string_bytes, + uint8_t* output) { + // Slice in forward order (step > 0) + if (!input_string_bytes) { + return 0; + } + + const SliceOptions& opt = *this->options; + auto [begin_index, end_index] = SliceForwardRange(opt, input_string_bytes); + const uint8_t* begin_sliced = input + begin_index; + const uint8_t* end_sliced = input + end_index; + + if (begin_sliced == end_sliced) { + return 0; + } // Second, copy computed slice to output - DCHECK(begin_sliced <= end_sliced); + DCHECK(begin_sliced < end_sliced); if (opt.step == 1) { // fast case, where we simply can finish with a memcpy std::copy(begin_sliced, end_sliced, output); @@ -2525,18 +2539,13 @@ struct SliceBytesTransform : StringSliceTransformBase { return dest - output; } - int64_t SliceBackward(const uint8_t* input, int64_t input_string_bytes, - uint8_t* output) { + static std::pair SliceBackwardRange(const SliceOptions& opt, + int64_t input_string_bytes) { // Slice in reverse order (step < 0) - const SliceOptions& opt = *this->options; - const uint8_t* begin = input; - const uint8_t* end = input + input_string_bytes; - const uint8_t* begin_sliced = begin; - const uint8_t* end_sliced = end; - - if (!input_string_bytes) { - return 0; - } + int64_t begin = 0; + int64_t end = input_string_bytes; + int64_t begin_sliced = begin; + int64_t end_sliced = end; if (opt.start >= 0) { // +1 because begin_sliced acts as as the end of a reverse iterator @@ -2555,6 +2564,28 @@ struct SliceBytesTransform : StringSliceTransformBase { } end_sliced--; + if (begin_sliced <= end_sliced) { + // zero length slice + return {0, 0}; + } + + return {begin_sliced, end_sliced}; + } + + int64_t SliceBackward(const uint8_t* input, int64_t input_string_bytes, + uint8_t* output) { + if (!input_string_bytes) { + return 0; + } + + const SliceOptions& opt = *this->options; + auto [begin_index, end_index] = SliceBackwardRange(opt, input_string_bytes); + const uint8_t* begin_sliced = input + begin_index; + const uint8_t* end_sliced = input + end_index; + + if (begin_sliced == end_sliced) { + return 0; + } // Copy computed slice to output uint8_t* dest = output; const uint8_t* i = begin_sliced; @@ -2568,6 +2599,22 @@ struct SliceBytesTransform : StringSliceTransformBase { return dest - output; } + + static Result FixedOutputSize(SliceOptions options, int32_t input_width_32) { + auto step = options.step; + if (step == 0) { + return Status::Invalid("Slice step cannot be zero"); + } + if (step > 0) { + // forward slice + auto [begin_index, end_index] = SliceForwardRange(options, input_width_32); + return static_cast((end_index - begin_index + step - 1) / step); + } else { + // backward slice + auto [begin_index, end_index] = SliceBackwardRange(options, input_width_32); + return static_cast((end_index - begin_index + step + 1) / step); + } + } }; template @@ -2594,6 +2641,12 @@ void AddAsciiStringSlice(FunctionRegistry* registry) { DCHECK_OK( func->AddKernel({ty}, ty, std::move(exec), SliceBytesTransform::State::Init)); } + using TransformExec = FixedSizeBinaryTransformExecWithState; + ScalarKernel fsb_kernel({InputType(Type::FIXED_SIZE_BINARY)}, + OutputType(TransformExec::OutputType), TransformExec::Exec, + StringSliceTransformBase::State::Init); + fsb_kernel.mem_allocation = MemAllocation::NO_PREALLOCATE; + DCHECK_OK(func->AddKernel(std::move(fsb_kernel))); DCHECK_OK(registry->AddFunction(std::move(func))); } diff --git a/cpp/src/arrow/compute/kernels/scalar_string_internal.h b/cpp/src/arrow/compute/kernels/scalar_string_internal.h index 7a5d5a7c86e85..6723d11c8deb8 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_internal.h +++ b/cpp/src/arrow/compute/kernels/scalar_string_internal.h @@ -250,6 +250,8 @@ struct StringSliceTransformBase : public StringTransformBase { using State = OptionsWrapper; const SliceOptions* options; + StringSliceTransformBase() = default; + explicit StringSliceTransformBase(const SliceOptions& options) : options{&options} {} Status PreExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) override { options = &State::Get(ctx); diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc b/cpp/src/arrow/compute/kernels/scalar_string_test.cc index 5dec16d89e29c..d7e35d07334ea 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc @@ -33,10 +33,10 @@ #include "arrow/compute/kernels/test_util.h" #include "arrow/testing/gtest_util.h" #include "arrow/type.h" +#include "arrow/type_fwd.h" #include "arrow/util/value_parsing.h" -namespace arrow { -namespace compute { +namespace arrow::compute { // interesting utf8 characters for testing (lower case / upper case): // * ῦ / Υ͂ (3 to 4 code units) (Note, we don't support this yet, utf8proc does not use @@ -712,11 +712,140 @@ TEST_F(TestFixedSizeBinaryKernels, BinaryLength) { "[6, null, 6]"); } +TEST_F(TestFixedSizeBinaryKernels, BinarySliceEmpty) { + SliceOptions options{2, 4}; + CheckScalarUnary("binary_slice", ArrayFromJSON(fixed_size_binary(0), R"([""])"), + ArrayFromJSON(fixed_size_binary(0), R"([""])"), &options); + + CheckScalarUnary("binary_slice", + ArrayFromJSON(fixed_size_binary(0), R"(["", null, ""])"), + ArrayFromJSON(fixed_size_binary(0), R"(["", null, ""])"), &options); + + CheckUnary("binary_slice", R"([null, null])", fixed_size_binary(2), R"([null, null])", + &options); +} + +TEST_F(TestFixedSizeBinaryKernels, BinarySliceBasic) { + SliceOptions options{2, 4}; + CheckUnary("binary_slice", R"(["abcdef", null, "foobaz"])", fixed_size_binary(2), + R"(["cd", null, "ob"])", &options); + + SliceOptions options_edgecase_1{-3, 1}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(0), + R"(["", ""])", &options_edgecase_1); + + SliceOptions options_edgecase_2{-10, -3}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz", null])", fixed_size_binary(3), + R"(["abc", "foo", null])", &options_edgecase_2); + + auto input = ArrayFromJSON(this->type(), R"(["foobaz"])"); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + testing::HasSubstr("Function 'binary_slice' cannot be called without options"), + CallFunction("binary_slice", {input})); + + SliceOptions options_invalid{2, 4, 0}; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("Slice step cannot be zero"), + CallFunction("binary_slice", {input}, &options_invalid)); +} + +TEST_F(TestFixedSizeBinaryKernels, BinarySlicePosPos) { + SliceOptions options_step{1, 5, 2}; + CheckUnary("binary_slice", R"([null, "abcdef", "foobaz"])", fixed_size_binary(2), + R"([null, "bd", "ob"])", &options_step); + + SliceOptions options_step_neg{5, 0, -2}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(3), + R"(["fdb", "zbo"])", &options_step_neg); +} + +TEST_F(TestFixedSizeBinaryKernels, BinarySlicePosNeg) { + SliceOptions options{2, -1}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(3), + R"(["cde", "oba"])", &options); + + SliceOptions options_step{1, -1, 2}; + CheckUnary("binary_slice", R"(["abcdef", null, "foobaz"])", fixed_size_binary(2), + R"(["bd", null, "ob"])", &options_step); + + SliceOptions options_step_neg{5, -4, -2}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(2), + R"(["fd", "zb"])", &options_step_neg); + + options_step_neg.stop = -6; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(3), + R"(["fdb", "zbo"])", &options_step_neg); +} + +TEST_F(TestFixedSizeBinaryKernels, BinarySliceNegNeg) { + SliceOptions options{-2, -1}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(1), + R"(["e", "a"])", &options); + + SliceOptions options_step{-4, -1, 2}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz", null, null])", fixed_size_binary(2), + R"(["ce", "oa", null, null])", &options_step); + + SliceOptions options_step_neg{-1, -3, -2}; + CheckUnary("binary_slice", R"([null, "abcdef", null, "foobaz"])", fixed_size_binary(1), + R"([null, "f", null, "z"])", &options_step_neg); + + options_step_neg.stop = -4; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(2), + R"(["fd", "zb"])", &options_step_neg); +} + +TEST_F(TestFixedSizeBinaryKernels, BinarySliceNegPos) { + SliceOptions options{-2, 4}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(0), + R"(["", ""])", &options); + + SliceOptions options_step{-4, 5, 2}; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(2), + R"(["ce", "oa"])", &options_step); + + SliceOptions options_step_neg{-1, 1, -2}; + CheckUnary("binary_slice", R"([null, "abcdef", "foobaz", null])", fixed_size_binary(2), + R"([null, "fd", "zb", null])", &options_step_neg); + + options_step_neg.stop = 0; + CheckUnary("binary_slice", R"(["abcdef", "foobaz"])", fixed_size_binary(3), + R"(["fdb", "zbo"])", &options_step_neg); +} + +TEST_F(TestFixedSizeBinaryKernels, BinarySliceConsistentyWithVarLenBinary) { + std::string source_str = "abcdef"; + for (size_t str_len = 0; str_len < source_str.size(); ++str_len) { + auto input_str = source_str.substr(0, str_len); + auto fixed_input = ArrayFromJSON(fixed_size_binary(static_cast(str_len)), + R"([")" + input_str + R"("])"); + auto varlen_input = ArrayFromJSON(binary(), R"([")" + input_str + R"("])"); + for (auto start = -6; start <= 6; ++start) { + for (auto stop = -6; stop <= 6; ++stop) { + for (auto step = -3; step <= 4; ++step) { + if (step == 0) { + continue; + } + SliceOptions options{start, stop, step}; + auto expected = + CallFunction("binary_slice", {varlen_input}, &options).ValueOrDie(); + auto actual = + CallFunction("binary_slice", {fixed_input}, &options).ValueOrDie(); + actual = Cast(actual, binary()).ValueOrDie(); + ASSERT_OK(actual.make_array()->ValidateFull()); + AssertDatumsEqual(expected, actual); + } + } + } + } +} + TEST_F(TestFixedSizeBinaryKernels, BinaryReplaceSlice) { ReplaceSliceOptions options{0, 1, "XX"}; CheckUnary("binary_replace_slice", "[]", fixed_size_binary(7), "[]", &options); - CheckUnary("binary_replace_slice", R"([null, "abcdef"])", fixed_size_binary(7), - R"([null, "XXbcdef"])", &options); + CheckUnary("binary_replace_slice", R"(["foobaz", null, "abcdef"])", + fixed_size_binary(7), R"(["XXoobaz", null, "XXbcdef"])", &options); ReplaceSliceOptions options_shrink{0, 2, ""}; CheckUnary("binary_replace_slice", R"([null, "abcdef"])", fixed_size_binary(4), @@ -731,8 +860,8 @@ TEST_F(TestFixedSizeBinaryKernels, BinaryReplaceSlice) { R"([null, "abXXef"])", &options_middle); ReplaceSliceOptions options_neg_start{-3, -2, "XX"}; - CheckUnary("binary_replace_slice", R"([null, "abcdef"])", fixed_size_binary(7), - R"([null, "abcXXef"])", &options_neg_start); + CheckUnary("binary_replace_slice", R"(["foobaz", null, "abcdef"])", + fixed_size_binary(7), R"(["fooXXaz", null, "abcXXef"])", &options_neg_start); ReplaceSliceOptions options_neg_end{2, -2, "XX"}; CheckUnary("binary_replace_slice", R"([null, "abcdef"])", fixed_size_binary(6), @@ -807,7 +936,7 @@ TEST_F(TestFixedSizeBinaryKernels, CountSubstringIgnoreCase) { offset_type(), "[0, null, 0, 1, 1, 1, 2, 2, 1]", &options); MatchSubstringOptions options_empty{"", /*ignore_case=*/true}; - CheckUnary("count_substring", R"([" ", null, "abcABc"])", offset_type(), + CheckUnary("count_substring", R"([" ", null, "abcdef"])", offset_type(), "[7, null, 7]", &options_empty); } @@ -2382,5 +2511,4 @@ TEST(TestStringKernels, UnicodeLibraryAssumptions) { } #endif -} // namespace compute -} // namespace arrow +} // namespace arrow::compute diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 7c5a134d330ac..d1eb605c71881 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -561,7 +561,8 @@ def test_slice_compatibility(): def test_binary_slice_compatibility(): - arr = pa.array([b"", b"a", b"a\xff", b"ab\x00", b"abc\xfb", b"ab\xf2de"]) + data = [b"", b"a", b"a\xff", b"ab\x00", b"abc\xfb", b"ab\xf2de"] + arr = pa.array(data) for start, stop, step in itertools.product(range(-6, 6), range(-6, 6), range(-3, 4)): @@ -574,6 +575,13 @@ def test_binary_slice_compatibility(): assert expected.equals(result) # Positional options assert pc.binary_slice(arr, start, stop, step) == result + # Fixed size binary input / output + for item in data: + fsb_scalar = pa.scalar(item, type=pa.binary(len(item))) + expected = item[start:stop:step] + actual = pc.binary_slice(fsb_scalar, start, stop, step) + assert actual.type == pa.binary(len(expected)) + assert actual.as_py() == expected def test_split_pattern(): From 6fe7480125b7fdb3a000a27fcc9cf464697b8a60 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 11 Jan 2024 10:00:49 -0800 Subject: [PATCH 08/92] GH-37164: [Python] Attach Python stacktrace to errors in `ConvertPyError` (#39380) ### Rationale for this change Users might define Python generators that are used in RecordBatchReaders and then exported through the C Data Interface. However, if an error occurs in their generator, the stacktrace and message are currently swallowed in the current `ConvertPyError` implementation, which only provides the type of error. This makes debugging code that passed RBRs difficult. ### What changes are included in this PR? Changes `ConvertPyError` to provide the fully formatted traceback in the error message. ### Are these changes tested? Yes, added one test to validate the errors messages are propagated. ### Are there any user-facing changes? This is a minor change in the error reporting behavior, which will provide more information. * Closes: #37164 Authored-by: Will Jones Signed-off-by: Antoine Pitrou --- python/pyarrow/src/arrow/python/common.cc | 49 +++++++++++++++++-- .../pyarrow/src/arrow/python/python_test.cc | 17 +++++-- python/pyarrow/tests/test_cffi.py | 34 +++++++++++++ 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/python/pyarrow/src/arrow/python/common.cc b/python/pyarrow/src/arrow/python/common.cc index 6fe2ed4dae321..2f44a9122f024 100644 --- a/python/pyarrow/src/arrow/python/common.cc +++ b/python/pyarrow/src/arrow/python/common.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include "arrow/memory_pool.h" @@ -90,9 +91,15 @@ class PythonErrorDetail : public StatusDetail { std::string ToString() const override { // This is simple enough not to need the GIL - const auto ty = reinterpret_cast(exc_type_.obj()); - // XXX Should we also print traceback? - return std::string("Python exception: ") + ty->tp_name; + Result result = FormatImpl(); + + if (result.ok()) { + return result.ValueOrDie(); + } else { + // Fallback to just the exception type + const auto ty = reinterpret_cast(exc_type_.obj()); + return std::string("Python exception: ") + ty->tp_name; + } } void RestorePyError() const { @@ -131,6 +138,42 @@ class PythonErrorDetail : public StatusDetail { } protected: + Result FormatImpl() const { + PyAcquireGIL lock; + + // Use traceback.format_exception() + OwnedRef traceback_module; + RETURN_NOT_OK(internal::ImportModule("traceback", &traceback_module)); + + OwnedRef fmt_exception; + RETURN_NOT_OK(internal::ImportFromModule(traceback_module.obj(), "format_exception", + &fmt_exception)); + + OwnedRef formatted; + formatted.reset(PyObject_CallFunctionObjArgs(fmt_exception.obj(), exc_type_.obj(), + exc_value_.obj(), exc_traceback_.obj(), + NULL)); + RETURN_IF_PYERROR(); + + std::stringstream ss; + ss << "Python exception: "; + Py_ssize_t num_lines = PySequence_Length(formatted.obj()); + RETURN_IF_PYERROR(); + + for (Py_ssize_t i = 0; i < num_lines; ++i) { + Py_ssize_t line_size; + + PyObject* line = PySequence_GetItem(formatted.obj(), i); + RETURN_IF_PYERROR(); + + const char* data = PyUnicode_AsUTF8AndSize(line, &line_size); + RETURN_IF_PYERROR(); + + ss << std::string_view(data, line_size); + } + return ss.str(); + } + PythonErrorDetail() = default; OwnedRefNoGIL exc_type_, exc_value_, exc_traceback_; diff --git a/python/pyarrow/src/arrow/python/python_test.cc b/python/pyarrow/src/arrow/python/python_test.cc index 01ab8a3038099..746bf410911f9 100644 --- a/python/pyarrow/src/arrow/python/python_test.cc +++ b/python/pyarrow/src/arrow/python/python_test.cc @@ -174,10 +174,14 @@ Status TestOwnedRefNoGILMoves() { } } -std::string FormatPythonException(const std::string& exc_class_name) { +std::string FormatPythonException(const std::string& exc_class_name, + const std::string& exc_value) { std::stringstream ss; ss << "Python exception: "; ss << exc_class_name; + ss << ": "; + ss << exc_value; + ss << "\n"; return ss.str(); } @@ -205,7 +209,8 @@ Status TestCheckPyErrorStatus() { } PyErr_SetString(PyExc_TypeError, "some error"); - ASSERT_OK(check_error(st, "some error", FormatPythonException("TypeError"))); + ASSERT_OK( + check_error(st, "some error", FormatPythonException("TypeError", "some error"))); ASSERT_TRUE(st.IsTypeError()); PyErr_SetString(PyExc_ValueError, "some error"); @@ -223,7 +228,8 @@ Status TestCheckPyErrorStatus() { } PyErr_SetString(PyExc_NotImplementedError, "some error"); - ASSERT_OK(check_error(st, "some error", FormatPythonException("NotImplementedError"))); + ASSERT_OK(check_error(st, "some error", + FormatPythonException("NotImplementedError", "some error"))); ASSERT_TRUE(st.IsNotImplemented()); // No override if a specific status code is given @@ -246,7 +252,8 @@ Status TestCheckPyErrorStatusNoGIL() { lock.release(); ASSERT_TRUE(st.IsUnknownError()); ASSERT_EQ(st.message(), "zzzt"); - ASSERT_EQ(st.detail()->ToString(), FormatPythonException("ZeroDivisionError")); + ASSERT_EQ(st.detail()->ToString(), + FormatPythonException("ZeroDivisionError", "zzzt")); return Status::OK(); } } @@ -257,7 +264,7 @@ Status TestRestorePyErrorBasics() { ASSERT_FALSE(PyErr_Occurred()); ASSERT_TRUE(st.IsUnknownError()); ASSERT_EQ(st.message(), "zzzt"); - ASSERT_EQ(st.detail()->ToString(), FormatPythonException("ZeroDivisionError")); + ASSERT_EQ(st.detail()->ToString(), FormatPythonException("ZeroDivisionError", "zzzt")); RestorePyError(st); ASSERT_TRUE(PyErr_Occurred()); diff --git a/python/pyarrow/tests/test_cffi.py b/python/pyarrow/tests/test_cffi.py index a9c17cc100cb4..ff81b06440f03 100644 --- a/python/pyarrow/tests/test_cffi.py +++ b/python/pyarrow/tests/test_cffi.py @@ -414,6 +414,40 @@ def test_export_import_batch_reader(reader_factory): pa.RecordBatchReader._import_from_c(ptr_stream) +@needs_cffi +def test_export_import_exception_reader(): + # See: https://github.com/apache/arrow/issues/37164 + c_stream = ffi.new("struct ArrowArrayStream*") + ptr_stream = int(ffi.cast("uintptr_t", c_stream)) + + gc.collect() # Make sure no Arrow data dangles in a ref cycle + old_allocated = pa.total_allocated_bytes() + + def gen(): + if True: + try: + raise ValueError('foo') + except ValueError as e: + raise NotImplementedError('bar') from e + else: + yield from make_batches() + + original = pa.RecordBatchReader.from_batches(make_schema(), gen()) + original._export_to_c(ptr_stream) + + reader = pa.RecordBatchReader._import_from_c(ptr_stream) + with pytest.raises(OSError) as exc_info: + reader.read_next_batch() + + # inner *and* outer exception should be present + assert 'ValueError: foo' in str(exc_info.value) + assert 'NotImplementedError: bar' in str(exc_info.value) + # Stacktrace containing line of the raise statement + assert 'raise ValueError(\'foo\')' in str(exc_info.value) + + assert pa.total_allocated_bytes() == old_allocated + + @needs_cffi def test_imported_batch_reader_error(): c_stream = ffi.new("struct ArrowArrayStream*") From 8149c390276c2f4d4e0031cd162b4498825f9062 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 11 Jan 2024 19:40:45 +0100 Subject: [PATCH 09/92] GH-39560: [C++][Parquet] Add integration test for BYTE_STREAM_SPLIT (#39570) ### Rationale for this change In https://github.com/apache/parquet-testing/pull/45 , an integration file for BYTE_STREAM_SPLIT was added to the parquet-testing repo. ### What changes are included in this PR? Add a test reading that file and ensuring the decoded values are as expected. ### Are these changes tested? By definition. ### Are there any user-facing changes? No. * Closes: #39560 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/parquet/reader_test.cc | 53 ++++++++++++++++++++++++++++++++-- cpp/submodules/parquet-testing | 2 +- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/cpp/src/parquet/reader_test.cc b/cpp/src/parquet/reader_test.cc index 2c2b62f5d12f6..551f62798e3b5 100644 --- a/cpp/src/parquet/reader_test.cc +++ b/cpp/src/parquet/reader_test.cc @@ -120,11 +120,27 @@ std::string concatenated_gzip_members() { return data_file("concatenated_gzip_members.parquet"); } +std::string byte_stream_split() { return data_file("byte_stream_split.zstd.parquet"); } + +template +std::vector ReadColumnValues(ParquetFileReader* file_reader, int row_group, + int column, int64_t expected_values_read) { + auto column_reader = checked_pointer_cast>( + file_reader->RowGroup(row_group)->Column(column)); + std::vector values(expected_values_read); + int64_t values_read; + auto levels_read = column_reader->ReadBatch(expected_values_read, nullptr, nullptr, + values.data(), &values_read); + EXPECT_EQ(expected_values_read, levels_read); + EXPECT_EQ(expected_values_read, values_read); + return values; +} + // TODO: Assert on definition and repetition levels -template +template void AssertColumnValues(std::shared_ptr> col, int64_t batch_size, int64_t expected_levels_read, - std::vector& expected_values, + const std::vector& expected_values, int64_t expected_values_read) { std::vector values(batch_size); int64_t values_read; @@ -1412,7 +1428,6 @@ TEST_P(TestCodec, LargeFileValues) { // column 0 ("a") auto col = checked_pointer_cast(group->Column(0)); - std::vector values(kNumRows); int64_t values_read; auto levels_read = @@ -1474,6 +1489,38 @@ TEST(TestFileReader, TestOverflowInt16PageOrdinal) { } } +#ifdef ARROW_WITH_ZSTD +TEST(TestByteStreamSplit, FloatIntegrationFile) { + auto file_path = byte_stream_split(); + auto file = ParquetFileReader::OpenFile(file_path); + + const int64_t kNumRows = 300; + + ASSERT_EQ(kNumRows, file->metadata()->num_rows()); + ASSERT_EQ(2, file->metadata()->num_columns()); + ASSERT_EQ(1, file->metadata()->num_row_groups()); + + // column 0 ("f32") + { + auto values = + ReadColumnValues(file.get(), /*row_group=*/0, /*column=*/0, kNumRows); + ASSERT_EQ(values[0], 1.7640524f); + ASSERT_EQ(values[1], 0.4001572f); + ASSERT_EQ(values[kNumRows - 2], -0.39944902f); + ASSERT_EQ(values[kNumRows - 1], 0.37005588f); + } + // column 1 ("f64") + { + auto values = + ReadColumnValues(file.get(), /*row_group=*/0, /*column=*/1, kNumRows); + ASSERT_EQ(values[0], -1.3065268517353166); + ASSERT_EQ(values[1], 1.658130679618188); + ASSERT_EQ(values[kNumRows - 2], -0.9301565025243212); + ASSERT_EQ(values[kNumRows - 1], -0.17858909208732915); + } +} +#endif // ARROW_WITH_ZSTD + struct PageIndexReaderParam { std::vector row_group_indices; std::vector column_indices; diff --git a/cpp/submodules/parquet-testing b/cpp/submodules/parquet-testing index d69d979223e88..4cb3cff24c965 160000 --- a/cpp/submodules/parquet-testing +++ b/cpp/submodules/parquet-testing @@ -1 +1 @@ -Subproject commit d69d979223e883faef9dc6fe3cf573087243c28a +Subproject commit 4cb3cff24c965fb329cdae763eabce47395a68a0 From 1a622ec18c154157341ae2562dda3b0df26550f8 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 10:50:02 -0800 Subject: [PATCH 10/92] GH-38997: [Java] Modularize format and vector (#38995) ### Rationale for this change This PR depends on #39011 . Make arrow-vector and arrow-format JPMS modules for improved compatibility with newer JDKs and tools that require modules. ### What changes are included in this PR? * add module-info.java files for vector and format. ### Are these changes tested? Yes, these modules are run as JPMS modules in JDK9+ unit tests. ### Are there any user-facing changes? Yes, some test classes have moved and users can now run these as JPMS modules. * Closes: #38997 --- .../apache/arrow/dataset/TestAllTypes.java | 2 +- .../apache/arrow/flight/FlightTestUtil.java | 2 +- java/format/src/main/java/module-info.java | 21 ++++++++ java/vector/pom.xml | 4 ++ .../src/main/codegen/includes/vv_imports.ftl | 3 -- java/vector/src/main/java/module-info.java | 50 +++++++++++++++++++ .../{ => vector}/util/ArrowTestDataUtil.java | 2 +- .../{ => vector}/util/TestSchemaUtil.java | 2 +- 8 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 java/format/src/main/java/module-info.java create mode 100644 java/vector/src/main/java/module-info.java rename java/vector/src/test/java/org/apache/arrow/{ => vector}/util/ArrowTestDataUtil.java (97%) rename java/vector/src/test/java/org/apache/arrow/{ => vector}/util/TestSchemaUtil.java (98%) diff --git a/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java b/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java index 7be49079e7450..5293aca0c329b 100644 --- a/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java +++ b/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java @@ -32,7 +32,6 @@ import org.apache.arrow.dataset.file.DatasetFileWriter; import org.apache.arrow.dataset.file.FileFormat; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.util.ArrowTestDataUtil; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateMilliVector; @@ -76,6 +75,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.ArrowTestDataUtil; import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; import org.apache.arrow.vector.util.Text; import org.junit.ClassRule; diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java index 64f70856a3b05..f9def74b56d1b 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Random; -import org.apache.arrow.util.ArrowTestDataUtil; +import org.apache.arrow.vector.util.ArrowTestDataUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.function.Executable; diff --git a/java/format/src/main/java/module-info.java b/java/format/src/main/java/module-info.java new file mode 100644 index 0000000000000..bda779c91afbc --- /dev/null +++ b/java/format/src/main/java/module-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.format { + exports org.apache.arrow.flatbuf; + requires transitive flatbuffers.java; +} diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 17d8f312a52a5..a4292449c9cb2 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -30,6 +30,10 @@ org.apache.arrow arrow-memory-core + + org.immutables + value + com.fasterxml.jackson.core jackson-core diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl index c9a8820b258b1..f4c72a1a6cbae 100644 --- a/java/vector/src/main/codegen/includes/vv_imports.ftl +++ b/java/vector/src/main/codegen/includes/vv_imports.ftl @@ -48,9 +48,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.ByteBuffer; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; import java.math.BigDecimal; import java.math.BigInteger; import java.time.Duration; diff --git a/java/vector/src/main/java/module-info.java b/java/vector/src/main/java/module-info.java new file mode 100644 index 0000000000000..20f7094715f4d --- /dev/null +++ b/java/vector/src/main/java/module-info.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.vector { + exports org.apache.arrow.vector; + exports org.apache.arrow.vector.compare; + exports org.apache.arrow.vector.compare.util; + exports org.apache.arrow.vector.complex; + exports org.apache.arrow.vector.complex.impl; + exports org.apache.arrow.vector.complex.reader; + exports org.apache.arrow.vector.complex.writer; + exports org.apache.arrow.vector.compression; + exports org.apache.arrow.vector.dictionary; + exports org.apache.arrow.vector.holders; + exports org.apache.arrow.vector.ipc; + exports org.apache.arrow.vector.ipc.message; + exports org.apache.arrow.vector.table; + exports org.apache.arrow.vector.types; + exports org.apache.arrow.vector.types.pojo; + exports org.apache.arrow.vector.util; + exports org.apache.arrow.vector.validate; + + opens org.apache.arrow.vector.types.pojo to com.fasterxml.jackson.databind; + + requires com.fasterxml.jackson.annotation; + requires com.fasterxml.jackson.core; + requires com.fasterxml.jackson.databind; + requires com.fasterxml.jackson.datatype.jsr310; + requires flatbuffers.java; + requires jdk.unsupported; + requires org.apache.arrow.format; + requires org.apache.arrow.memory.core; + requires org.apache.commons.codec; + requires org.eclipse.collections.impl; + requires org.slf4j; +} diff --git a/java/vector/src/test/java/org/apache/arrow/util/ArrowTestDataUtil.java b/java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java similarity index 97% rename from java/vector/src/test/java/org/apache/arrow/util/ArrowTestDataUtil.java rename to java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java index 120c0adc884ed..1c525c0c271ac 100644 --- a/java/vector/src/test/java/org/apache/arrow/util/ArrowTestDataUtil.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.util; +package org.apache.arrow.vector.util; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/java/vector/src/test/java/org/apache/arrow/util/TestSchemaUtil.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestSchemaUtil.java similarity index 98% rename from java/vector/src/test/java/org/apache/arrow/util/TestSchemaUtil.java rename to java/vector/src/test/java/org/apache/arrow/vector/util/TestSchemaUtil.java index cefff83823289..52b6584086832 100644 --- a/java/vector/src/test/java/org/apache/arrow/util/TestSchemaUtil.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestSchemaUtil.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.util; +package org.apache.arrow.vector.util; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; From c78a1aeb2e328cfee713f615a5e52784866725a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 12 Jan 2024 02:12:02 +0100 Subject: [PATCH 11/92] GH-38470: [CI][Integration] Install jpype and build JNI c-data to run integration tests (#39502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Integration verification tasks are currently failing on CI. ### What changes are included in this PR? Install jpype and build JNI c-data to run integration tests ### Are these changes tested? Yes via archery ### Are there any user-facing changes? No * Closes: #38470 Lead-authored-by: Raúl Cumplido Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .../archery/integration/tester_java.py | 14 +++++- dev/archery/setup.py | 1 + dev/release/verify-release-candidate.sh | 47 +++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index 032ac13e74ec2..857fe0c50af06 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -40,7 +40,6 @@ def load_version_from_pom(): _JAVA_OPTS = [ "-Dio.netty.tryReflectionSetAccessible=true", "-Darrow.struct.conflict.policy=CONFLICT_APPEND", - "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED", # GH-39113: avoid failures accessing files in `/tmp/hsperfdata_...` "-XX:-UsePerfData", ] @@ -83,13 +82,24 @@ def setup_jpype(): import jpype jar_path = f"{_ARROW_TOOLS_JAR}:{_ARROW_C_DATA_JAR}" # XXX Didn't manage to tone down the logging level here (DEBUG -> INFO) + java_opts = _JAVA_OPTS[:] + proc = subprocess.run( + ['java', '--add-opens'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True) + if 'Unrecognized option: --add-opens' not in proc.stderr: + # Java 9+ + java_opts.append( + '--add-opens=java.base/java.nio=' + 'org.apache.arrow.memory.core,ALL-UNNAMED') jpype.startJVM(jpype.getDefaultJVMPath(), "-Djava.class.path=" + jar_path, # This flag is too heavy for IPC and Flight tests "-Darrow.memory.debug.allocator=true", # Reduce internal use of signals by the JVM "-Xrs", - *_JAVA_OPTS) + *java_opts) class _CDataBase: diff --git a/dev/archery/setup.py b/dev/archery/setup.py index e2c89ae204bd6..2ecc72e04e8aa 100755 --- a/dev/archery/setup.py +++ b/dev/archery/setup.py @@ -35,6 +35,7 @@ 'setuptools_scm'], 'docker': ['ruamel.yaml', 'python-dotenv'], 'integration': ['cffi'], + 'integration-java': ['jpype1'], 'lint': ['numpydoc==1.1.0', 'autopep8', 'flake8==6.1.0', 'cython-lint', 'cmake_format==0.6.13'], 'numpydoc': ['numpydoc==1.1.0'], diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index ebdb493f8006e..f12a5dc8b8964 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -568,10 +568,55 @@ test_package_java() { maybe_setup_conda maven openjdk pushd java + + if [ ${TEST_INTEGRATION_JAVA} -gt 0 ]; then + # Build JNI for C data interface + local -a cmake_options=() + # Enable only C data interface. + cmake_options+=(-DARROW_JAVA_JNI_ENABLE_C=ON) + cmake_options+=(-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF) + # Disable Testing because GTest might not be present. + cmake_options+=(-DBUILD_TESTING=OFF) + if [ ! -z "${CMAKE_GENERATOR}" ]; then + cmake_options+=(-G "${CMAKE_GENERATOR}") + fi + local build_dir="${ARROW_TMPDIR}/java-jni-build" + local install_dir="${ARROW_TMPDIR}/java-jni-install" + local dist_dir="${ARROW_TMPDIR}/java-jni-dist" + cmake \ + -S . \ + -B "${build_dir}" \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-release} \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ + -DCMAKE_PREFIX_PATH="${ARROW_HOME}" \ + "${cmake_options[@]}" + cmake --build "${build_dir}" + cmake --install "${build_dir}" + + local normalized_arch=$(arch) + case ${normalized_arch} in + aarch64|arm64) + normalized_arch=aarch_64 + ;; + i386) + normalized_arch=x86_64 + ;; + esac + mkdir -p ${dist_dir}/${normalized_arch}/ + mv ${install_dir}/lib/* ${dist_dir}/${normalized_arch}/ + mvn install \ + -Darrow.c.jni.dist.dir=${dist_dir} \ + -Parrow-c-data + fi + if [ ${TEST_JAVA} -gt 0 ]; then mvn test fi + + # Build jars mvn package + popd } @@ -632,6 +677,7 @@ test_and_install_cpp() { -DARROW_JSON=ON \ -DARROW_ORC=ON \ -DARROW_PARQUET=ON \ + -DARROW_SUBSTRAIT=ON \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-ON} \ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ @@ -904,6 +950,7 @@ test_integration() { maybe_setup_virtualenv pip install -e dev/archery[integration] + pip install -e dev/archery[integration-java] JAVA_DIR=$ARROW_SOURCE_DIR/java CPP_BUILD_DIR=$ARROW_TMPDIR/cpp-build From 3cc04f1e8389deea18b88eedc5b4e3458467d9c6 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 12 Jan 2024 05:08:40 +0100 Subject: [PATCH 12/92] GH-39523: [R] Don't override explicitly set NOT_CRAN=false when on dev version (#39524) ### Rationale for this change The default linux build used in the lto job should not build with aws/gcs. A change in the build system changed this. ### What changes are included in this PR? Revert to old behavior by not overriding explicitly set `NOT_CRAN=false`. ### Are these changes tested? CI ### Are there any user-facing changes? No * Closes: #39523 Lead-authored-by: Jacob Wujciak-Jens Co-authored-by: Sutou Kouhei Signed-off-by: Jacob Wujciak-Jens --- r/tools/nixlibs.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index fe8de284b16b0..9027aa227a074 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -860,7 +860,8 @@ if (is_release) { VERSION <- VERSION[1, 1:3] arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/") } else { - not_cran <- TRUE + # Don't override explictily set NOT_CRAN env var, as it is used in CI. + not_cran <- !env_is("NOT_CRAN", "false") arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/") } From 22b42b0e7718c91cc80390090f64fda312ff93a0 Mon Sep 17 00:00:00 2001 From: Alexander Blazhkov <68013846+thermo911@users.noreply.github.com> Date: Fri, 12 Jan 2024 22:56:04 +0300 Subject: [PATCH 13/92] GH-39558: [Java] Add SQL_ALL_TABLES_ARE_SELECTABLE, SQL_NULL_ORDERING and SQL_MAX_COLUMNS_IN_TABLE support to SqlInfoBuilder (#39561) This PR adds ability to specify `SQL_ALL_TABLES_ARE_SELECTABLE` and `SQL_NULL_ORDERING` metadata in `org.apache.arrow.flight.sql.SqlInfoBuilder`. ### Rationale for this change Without this change it is impossible to specify whether all tables are selectable, supported null ordering and maximum number of columns in table using `SqlInfoBuilder`. ### What changes are included in this PR? In this PR two methods are added to `SqlInfoBuilder`: - `withSqlAllTablesAreSelectable` accepting boolean parameter that specifies whether all tables are selectable - `withSqlNullOrdering` accepting `org.apache.arrow.flight.sql.impl.FlightSql.SqlNullOrdering` value that specifies supported null ordering - `withSqlMaxColumnsInTable` accepting long parameter that specifies maximum number of columns in table ### Are these changes tested? To ensure correctness `org.apache.arrow.flight.TestFlightSql#testGetSqlInfoResultsWithManyArgs` test is added). ### Are there any user-facing changes? This PR does not contain any breaking changes of user API. * Closes: #39558 Authored-by: Alexander Blazhkov Signed-off-by: David Li --- .../arrow/flight/sql/SqlInfoBuilder.java | 31 ++++++++++++ .../apache/arrow/flight/TestFlightSql.java | 48 ++++++++++--------- .../flight/sql/example/FlightSqlExample.java | 5 +- 3 files changed, 60 insertions(+), 24 deletions(-) diff --git a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/SqlInfoBuilder.java b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/SqlInfoBuilder.java index 18793f9b905fe..251a709f63965 100644 --- a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/SqlInfoBuilder.java +++ b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/SqlInfoBuilder.java @@ -32,6 +32,7 @@ import java.util.function.ObjIntConsumer; import org.apache.arrow.flight.sql.impl.FlightSql.SqlInfo; +import org.apache.arrow.flight.sql.impl.FlightSql.SqlNullOrdering; import org.apache.arrow.flight.sql.impl.FlightSql.SqlOuterJoinsSupportLevel; import org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportedCaseSensitivity; import org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportedElementActions; @@ -502,6 +503,26 @@ public SqlInfoBuilder withSqlQuotedIdentifierCase(final SqlSupportedCaseSensitiv return withBitIntProvider(SqlInfo.SQL_QUOTED_IDENTIFIER_CASE_VALUE, value.getNumber()); } + /** + * Sets a value for {@link SqlInfo#SQL_ALL_TABLES_ARE_SELECTABLE} in the builder. + * + * @param value the value for {@link SqlInfo#SQL_ALL_TABLES_ARE_SELECTABLE} to be set. + * @return the SqlInfoBuilder itself. + */ + public SqlInfoBuilder withSqlAllTablesAreSelectable(final boolean value) { + return withBooleanProvider(SqlInfo.SQL_ALL_TABLES_ARE_SELECTABLE_VALUE, value); + } + + /** + * Sets a value for {@link SqlInfo#SQL_NULL_ORDERING} in the builder. + * + * @param value the value for {@link SqlInfo#SQL_NULL_ORDERING} to be set. + * @return the SqlInfoBuilder itself. + */ + public SqlInfoBuilder withSqlNullOrdering(final SqlNullOrdering value) { + return withBitIntProvider(SqlInfo.SQL_NULL_ORDERING_VALUE, value.getNumber()); + } + /** * Sets a value SqlInf @link SqlInfo#SQL_MAX_BINARY_LITERAL_LENGTH} in the builder. * @@ -572,6 +593,16 @@ public SqlInfoBuilder withSqlMaxColumnsInSelect(final long value) { return withBitIntProvider(SqlInfo.SQL_MAX_COLUMNS_IN_SELECT_VALUE, value); } + /** + * Sets a value for {@link SqlInfo#SQL_MAX_COLUMNS_IN_TABLE} in the builder. + * + * @param value the value for {@link SqlInfo#SQL_MAX_COLUMNS_IN_TABLE} to be set. + * @return the SqlInfoBuilder itself. + */ + public SqlInfoBuilder withSqlMaxColumnsInTable(final long value) { + return withBitIntProvider(SqlInfo.SQL_MAX_COLUMNS_IN_TABLE_VALUE, value); + } + /** * Sets a value for {@link SqlInfo#SQL_MAX_CONNECTIONS} in the builder. * diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java index 7635b80ecd0fd..948364a920004 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java @@ -106,6 +106,12 @@ public static void setUp() throws Exception { .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION_VALUE), "10.14.2.0 - (1828579)"); GET_SQL_INFO_EXPECTED_RESULTS_MAP .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_READ_ONLY_VALUE), "false"); + GET_SQL_INFO_EXPECTED_RESULTS_MAP + .put(Integer.toString(FlightSql.SqlInfo.SQL_ALL_TABLES_ARE_SELECTABLE_VALUE), "true"); + GET_SQL_INFO_EXPECTED_RESULTS_MAP + .put( + Integer.toString(FlightSql.SqlInfo.SQL_NULL_ORDERING_VALUE), + Integer.toString(FlightSql.SqlNullOrdering.SQL_NULLS_SORTED_AT_END_VALUE)); GET_SQL_INFO_EXPECTED_RESULTS_MAP .put(Integer.toString(FlightSql.SqlInfo.SQL_DDL_CATALOG_VALUE), "false"); GET_SQL_INFO_EXPECTED_RESULTS_MAP @@ -122,6 +128,8 @@ public static void setUp() throws Exception { .put( Integer.toString(FlightSql.SqlInfo.SQL_QUOTED_IDENTIFIER_CASE_VALUE), Integer.toString(SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_CASE_INSENSITIVE_VALUE)); + GET_SQL_INFO_EXPECTED_RESULTS_MAP + .put(Integer.toString(FlightSql.SqlInfo.SQL_MAX_COLUMNS_IN_TABLE_VALUE), "42"); } @AfterAll @@ -135,12 +143,15 @@ private static List> getNonConformingResultsForGetSqlInfo(final Lis FlightSql.SqlInfo.FLIGHT_SQL_SERVER_VERSION, FlightSql.SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION, FlightSql.SqlInfo.FLIGHT_SQL_SERVER_READ_ONLY, + FlightSql.SqlInfo.SQL_ALL_TABLES_ARE_SELECTABLE, + FlightSql.SqlInfo.SQL_NULL_ORDERING, FlightSql.SqlInfo.SQL_DDL_CATALOG, FlightSql.SqlInfo.SQL_DDL_SCHEMA, FlightSql.SqlInfo.SQL_DDL_TABLE, FlightSql.SqlInfo.SQL_IDENTIFIER_CASE, FlightSql.SqlInfo.SQL_IDENTIFIER_QUOTE_CHAR, - FlightSql.SqlInfo.SQL_QUOTED_IDENTIFIER_CASE); + FlightSql.SqlInfo.SQL_QUOTED_IDENTIFIER_CASE, + FlightSql.SqlInfo.SQL_MAX_COLUMNS_IN_TABLE); } private static List> getNonConformingResultsForGetSqlInfo( @@ -152,6 +163,7 @@ private static List> getNonConformingResultsForGetSqlInfo( final List result = results.get(index); final String providedName = result.get(0); final String expectedName = Integer.toString(args[index].getNumber()); + System.err.println(expectedName); if (!(GET_SQL_INFO_EXPECTED_RESULTS_MAP.get(providedName).equals(result.get(1)) && providedName.equals(expectedName))) { nonConformingResults.add(result); @@ -603,31 +615,21 @@ public void testGetSqlInfoResultsWithSingleArg() throws Exception { } @Test - public void testGetSqlInfoResultsWithTwoArgs() throws Exception { - final FlightSql.SqlInfo[] args = { - FlightSql.SqlInfo.FLIGHT_SQL_SERVER_NAME, - FlightSql.SqlInfo.FLIGHT_SQL_SERVER_VERSION}; - final FlightInfo info = sqlClient.getSqlInfo(args); - try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { - Assertions.assertAll( - () -> MatcherAssert.assertThat( - stream.getSchema(), - is(FlightSqlProducer.Schemas.GET_SQL_INFO_SCHEMA) - ), - () -> MatcherAssert.assertThat( - getNonConformingResultsForGetSqlInfo(getResults(stream), args), - is(emptyList()) - ) - ); - } - } - - @Test - public void testGetSqlInfoResultsWithThreeArgs() throws Exception { + public void testGetSqlInfoResultsWithManyArgs() throws Exception { final FlightSql.SqlInfo[] args = { FlightSql.SqlInfo.FLIGHT_SQL_SERVER_NAME, FlightSql.SqlInfo.FLIGHT_SQL_SERVER_VERSION, - FlightSql.SqlInfo.SQL_IDENTIFIER_QUOTE_CHAR}; + FlightSql.SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION, + FlightSql.SqlInfo.FLIGHT_SQL_SERVER_READ_ONLY, + FlightSql.SqlInfo.SQL_ALL_TABLES_ARE_SELECTABLE, + FlightSql.SqlInfo.SQL_NULL_ORDERING, + FlightSql.SqlInfo.SQL_DDL_CATALOG, + FlightSql.SqlInfo.SQL_DDL_SCHEMA, + FlightSql.SqlInfo.SQL_DDL_TABLE, + FlightSql.SqlInfo.SQL_IDENTIFIER_CASE, + FlightSql.SqlInfo.SQL_IDENTIFIER_QUOTE_CHAR, + FlightSql.SqlInfo.SQL_QUOTED_IDENTIFIER_CASE, + FlightSql.SqlInfo.SQL_MAX_COLUMNS_IN_TABLE}; final FlightInfo info = sqlClient.getSqlInfo(args); try (final FlightStream stream = sqlClient.getStream(info.getEndpoints().get(0).getTicket())) { Assertions.assertAll( diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java index 3cc8f4a1c1bd5..11f38ded5fcdd 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java @@ -237,7 +237,10 @@ public FlightSqlExample(final Location location) { SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_UPPERCASE : metaData.storesLowerCaseQuotedIdentifiers() ? SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_LOWERCASE : - SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_UNKNOWN); + SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_UNKNOWN) + .withSqlAllTablesAreSelectable(true) + .withSqlNullOrdering(SqlNullOrdering.SQL_NULLS_SORTED_AT_END) + .withSqlMaxColumnsInTable(42); } catch (SQLException e) { throw new RuntimeException(e); } From 87ed8ac432dc7e77fa0275a24e3edda9b0bc2a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 12 Jan 2024 22:25:09 +0100 Subject: [PATCH 14/92] GH-39564: [CI][Java] Set correct version on Java BOM (#39580) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change The version set currently on the maintenance branch is incorrect for Java BOM. ### What changes are included in this PR? Suggested changes to set specifically version for BOM and maven. ### Are these changes tested? I will trigger java-jars via archery but I think this is currently only reproducible on the maintenance branch. So we will have to merge and validate there. ### Are there any user-facing changes? No * Closes: #39564 Authored-by: Raúl Cumplido Signed-off-by: Raúl Cumplido --- dev/tasks/java-jars/github.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index fbce12ee427e1..7ee68e77ee637 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -236,6 +236,8 @@ jobs: set -e pushd arrow/java mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} + mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f bom + mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f maven popd arrow/ci/scripts/java_full_build.sh \ $GITHUB_WORKSPACE/arrow \ From e6323646558ee01234ce58af273c5a834745f298 Mon Sep 17 00:00:00 2001 From: abandy Date: Sat, 13 Jan 2024 17:02:06 -0500 Subject: [PATCH 15/92] GH-39519: [Swift] Fix null count when using reader (#39520) Currently the reader is not properly setting the null count when building an array from a stream. This PR adds a fix for this. * Closes: #39519 Authored-by: Alva Bandy Signed-off-by: Sutou Kouhei --- swift/Arrow/Sources/Arrow/ArrowReader.swift | 12 ++- .../Sources/Arrow/ArrowReaderHelper.swift | 82 +++++++++++-------- swift/Arrow/Tests/ArrowTests/IPCTests.swift | 40 +++++++-- .../Tests/ArrowTests/RecordBatchTests.swift | 9 +- 4 files changed, 96 insertions(+), 47 deletions(-) diff --git a/swift/Arrow/Sources/Arrow/ArrowReader.swift b/swift/Arrow/Sources/Arrow/ArrowReader.swift index d9dc1bdb470e6..237f22dc979e3 100644 --- a/swift/Arrow/Sources/Arrow/ArrowReader.swift +++ b/swift/Arrow/Sources/Arrow/ArrowReader.swift @@ -57,15 +57,17 @@ public class ArrowReader { private func loadPrimitiveData(_ loadInfo: DataLoadInfo) -> Result { do { let node = loadInfo.recordBatch.nodes(at: loadInfo.nodeIndex)! + let nullLength = UInt(ceil(Double(node.length) / 8)) try validateBufferIndex(loadInfo.recordBatch, index: loadInfo.bufferIndex) let nullBuffer = loadInfo.recordBatch.buffers(at: loadInfo.bufferIndex)! let arrowNullBuffer = makeBuffer(nullBuffer, fileData: loadInfo.fileData, - length: UInt(node.nullCount), messageOffset: loadInfo.messageOffset) + length: nullLength, messageOffset: loadInfo.messageOffset) try validateBufferIndex(loadInfo.recordBatch, index: loadInfo.bufferIndex + 1) let valueBuffer = loadInfo.recordBatch.buffers(at: loadInfo.bufferIndex + 1)! let arrowValueBuffer = makeBuffer(valueBuffer, fileData: loadInfo.fileData, length: UInt(node.length), messageOffset: loadInfo.messageOffset) - return makeArrayHolder(loadInfo.field, buffers: [arrowNullBuffer, arrowValueBuffer]) + return makeArrayHolder(loadInfo.field, buffers: [arrowNullBuffer, arrowValueBuffer], + nullCount: UInt(node.nullCount)) } catch let error as ArrowError { return .failure(error) } catch { @@ -76,10 +78,11 @@ public class ArrowReader { private func loadVariableData(_ loadInfo: DataLoadInfo) -> Result { let node = loadInfo.recordBatch.nodes(at: loadInfo.nodeIndex)! do { + let nullLength = UInt(ceil(Double(node.length) / 8)) try validateBufferIndex(loadInfo.recordBatch, index: loadInfo.bufferIndex) let nullBuffer = loadInfo.recordBatch.buffers(at: loadInfo.bufferIndex)! let arrowNullBuffer = makeBuffer(nullBuffer, fileData: loadInfo.fileData, - length: UInt(node.nullCount), messageOffset: loadInfo.messageOffset) + length: nullLength, messageOffset: loadInfo.messageOffset) try validateBufferIndex(loadInfo.recordBatch, index: loadInfo.bufferIndex + 1) let offsetBuffer = loadInfo.recordBatch.buffers(at: loadInfo.bufferIndex + 1)! let arrowOffsetBuffer = makeBuffer(offsetBuffer, fileData: loadInfo.fileData, @@ -88,7 +91,8 @@ public class ArrowReader { let valueBuffer = loadInfo.recordBatch.buffers(at: loadInfo.bufferIndex + 2)! let arrowValueBuffer = makeBuffer(valueBuffer, fileData: loadInfo.fileData, length: UInt(node.length), messageOffset: loadInfo.messageOffset) - return makeArrayHolder(loadInfo.field, buffers: [arrowNullBuffer, arrowOffsetBuffer, arrowValueBuffer]) + return makeArrayHolder(loadInfo.field, buffers: [arrowNullBuffer, arrowOffsetBuffer, arrowValueBuffer], + nullCount: UInt(node.nullCount)) } catch let error as ArrowError { return .failure(error) } catch { diff --git a/swift/Arrow/Sources/Arrow/ArrowReaderHelper.swift b/swift/Arrow/Sources/Arrow/ArrowReaderHelper.swift index fa52160478f24..7b3ec04b3aa36 100644 --- a/swift/Arrow/Sources/Arrow/ArrowReaderHelper.swift +++ b/swift/Arrow/Sources/Arrow/ArrowReaderHelper.swift @@ -18,10 +18,11 @@ import FlatBuffers import Foundation -private func makeBinaryHolder(_ buffers: [ArrowBuffer]) -> Result { +private func makeBinaryHolder(_ buffers: [ArrowBuffer], + nullCount: UInt) -> Result { do { let arrowData = try ArrowData(ArrowType(ArrowType.ArrowBinary), buffers: buffers, - nullCount: buffers[0].length, stride: MemoryLayout.stride) + nullCount: nullCount, stride: MemoryLayout.stride) return .success(ArrowArrayHolder(BinaryArray(arrowData))) } catch let error as ArrowError { return .failure(error) @@ -30,10 +31,11 @@ private func makeBinaryHolder(_ buffers: [ArrowBuffer]) -> Result Result { +private func makeStringHolder(_ buffers: [ArrowBuffer], + nullCount: UInt) -> Result { do { let arrowData = try ArrowData(ArrowType(ArrowType.ArrowString), buffers: buffers, - nullCount: buffers[0].length, stride: MemoryLayout.stride) + nullCount: nullCount, stride: MemoryLayout.stride) return .success(ArrowArrayHolder(StringArray(arrowData))) } catch let error as ArrowError { return .failure(error) @@ -43,30 +45,32 @@ private func makeStringHolder(_ buffers: [ArrowBuffer]) -> Result Result { switch floatType.precision { case .single: - return makeFixedHolder(Float.self, buffers: buffers, arrowType: ArrowType.ArrowFloat) + return makeFixedHolder(Float.self, buffers: buffers, arrowType: ArrowType.ArrowFloat, nullCount: nullCount) case .double: - return makeFixedHolder(Double.self, buffers: buffers, arrowType: ArrowType.ArrowDouble) + return makeFixedHolder(Double.self, buffers: buffers, arrowType: ArrowType.ArrowDouble, nullCount: nullCount) default: return .failure(.unknownType("Float precision \(floatType.precision) currently not supported")) } } private func makeDateHolder(_ dateType: org_apache_arrow_flatbuf_Date, - buffers: [ArrowBuffer] + buffers: [ArrowBuffer], + nullCount: UInt ) -> Result { do { if dateType.unit == .day { let arrowData = try ArrowData(ArrowType(ArrowType.ArrowString), buffers: buffers, - nullCount: buffers[0].length, stride: MemoryLayout.stride) + nullCount: nullCount, stride: MemoryLayout.stride) return .success(ArrowArrayHolder(Date32Array(arrowData))) } let arrowData = try ArrowData(ArrowType(ArrowType.ArrowString), buffers: buffers, - nullCount: buffers[0].length, stride: MemoryLayout.stride) + nullCount: nullCount, stride: MemoryLayout.stride) return .success(ArrowArrayHolder(Date64Array(arrowData))) } catch let error as ArrowError { return .failure(error) @@ -76,19 +80,20 @@ private func makeDateHolder(_ dateType: org_apache_arrow_flatbuf_Date, } private func makeTimeHolder(_ timeType: org_apache_arrow_flatbuf_Time, - buffers: [ArrowBuffer] + buffers: [ArrowBuffer], + nullCount: UInt ) -> Result { do { if timeType.unit == .second || timeType.unit == .millisecond { let arrowUnit: ArrowTime32Unit = timeType.unit == .second ? .seconds : .milliseconds let arrowData = try ArrowData(ArrowTypeTime32(arrowUnit), buffers: buffers, - nullCount: buffers[0].length, stride: MemoryLayout.stride) + nullCount: nullCount, stride: MemoryLayout.stride) return .success(ArrowArrayHolder(FixedArray(arrowData))) } let arrowUnit: ArrowTime64Unit = timeType.unit == .microsecond ? .microseconds : .nanoseconds let arrowData = try ArrowData(ArrowTypeTime64(arrowUnit), buffers: buffers, - nullCount: buffers[0].length, stride: MemoryLayout.stride) + nullCount: nullCount, stride: MemoryLayout.stride) return .success(ArrowArrayHolder(FixedArray(arrowData))) } catch let error as ArrowError { return .failure(error) @@ -97,10 +102,11 @@ private func makeTimeHolder(_ timeType: org_apache_arrow_flatbuf_Time, } } -private func makeBoolHolder(_ buffers: [ArrowBuffer]) -> Result { +private func makeBoolHolder(_ buffers: [ArrowBuffer], + nullCount: UInt) -> Result { do { let arrowData = try ArrowData(ArrowType(ArrowType.ArrowBool), buffers: buffers, - nullCount: buffers[0].length, stride: MemoryLayout.stride) + nullCount: nullCount, stride: MemoryLayout.stride) return .success(ArrowArrayHolder(BoolArray(arrowData))) } catch let error as ArrowError { return .failure(error) @@ -111,11 +117,12 @@ private func makeBoolHolder(_ buffers: [ArrowBuffer]) -> Result( _: T.Type, buffers: [ArrowBuffer], - arrowType: ArrowType.Info + arrowType: ArrowType.Info, + nullCount: UInt ) -> Result { do { let arrowData = try ArrowData(ArrowType(arrowType), buffers: buffers, - nullCount: buffers[0].length, stride: MemoryLayout.stride) + nullCount: nullCount, stride: MemoryLayout.stride) return .success(ArrowArrayHolder(FixedArray(arrowData))) } catch let error as ArrowError { return .failure(error) @@ -124,9 +131,10 @@ private func makeFixedHolder( } } -func makeArrayHolder( // swiftlint:disable:this cyclomatic_complexity +func makeArrayHolder( // swiftlint:disable:this cyclomatic_complexity function_body_length _ field: org_apache_arrow_flatbuf_Field, - buffers: [ArrowBuffer] + buffers: [ArrowBuffer], + nullCount: UInt ) -> Result { let type = field.typeType switch type { @@ -135,45 +143,53 @@ func makeArrayHolder( // swiftlint:disable:this cyclomatic_complexity let bitWidth = intType.bitWidth if bitWidth == 8 { if intType.isSigned { - return makeFixedHolder(Int8.self, buffers: buffers, arrowType: ArrowType.ArrowInt8) + return makeFixedHolder(Int8.self, buffers: buffers, + arrowType: ArrowType.ArrowInt8, nullCount: nullCount) } else { - return makeFixedHolder(UInt8.self, buffers: buffers, arrowType: ArrowType.ArrowUInt8) + return makeFixedHolder(UInt8.self, buffers: buffers, + arrowType: ArrowType.ArrowUInt8, nullCount: nullCount) } } else if bitWidth == 16 { if intType.isSigned { - return makeFixedHolder(Int16.self, buffers: buffers, arrowType: ArrowType.ArrowInt16) + return makeFixedHolder(Int16.self, buffers: buffers, + arrowType: ArrowType.ArrowInt16, nullCount: nullCount) } else { - return makeFixedHolder(UInt16.self, buffers: buffers, arrowType: ArrowType.ArrowUInt16) + return makeFixedHolder(UInt16.self, buffers: buffers, + arrowType: ArrowType.ArrowUInt16, nullCount: nullCount) } } else if bitWidth == 32 { if intType.isSigned { - return makeFixedHolder(Int32.self, buffers: buffers, arrowType: ArrowType.ArrowInt32) + return makeFixedHolder(Int32.self, buffers: buffers, + arrowType: ArrowType.ArrowInt32, nullCount: nullCount) } else { - return makeFixedHolder(UInt32.self, buffers: buffers, arrowType: ArrowType.ArrowUInt32) + return makeFixedHolder(UInt32.self, buffers: buffers, + arrowType: ArrowType.ArrowUInt32, nullCount: nullCount) } } else if bitWidth == 64 { if intType.isSigned { - return makeFixedHolder(Int64.self, buffers: buffers, arrowType: ArrowType.ArrowInt64) + return makeFixedHolder(Int64.self, buffers: buffers, + arrowType: ArrowType.ArrowInt64, nullCount: nullCount) } else { - return makeFixedHolder(UInt64.self, buffers: buffers, arrowType: ArrowType.ArrowUInt64) + return makeFixedHolder(UInt64.self, buffers: buffers, + arrowType: ArrowType.ArrowUInt64, nullCount: nullCount) } } return .failure(.unknownType("Int width \(bitWidth) currently not supported")) case .bool: - return makeBoolHolder(buffers) + return makeBoolHolder(buffers, nullCount: nullCount) case .floatingpoint: let floatType = field.type(type: org_apache_arrow_flatbuf_FloatingPoint.self)! - return makeFloatHolder(floatType, buffers: buffers) + return makeFloatHolder(floatType, buffers: buffers, nullCount: nullCount) case .utf8: - return makeStringHolder(buffers) + return makeStringHolder(buffers, nullCount: nullCount) case .binary: - return makeBinaryHolder(buffers) + return makeBinaryHolder(buffers, nullCount: nullCount) case .date: let dateType = field.type(type: org_apache_arrow_flatbuf_Date.self)! - return makeDateHolder(dateType, buffers: buffers) + return makeDateHolder(dateType, buffers: buffers, nullCount: nullCount) case .time: let timeType = field.type(type: org_apache_arrow_flatbuf_Time.self)! - return makeTimeHolder(timeType, buffers: buffers) + return makeTimeHolder(timeType, buffers: buffers, nullCount: nullCount) default: return .failure(.unknownType("Type \(type) currently not supported")) } diff --git a/swift/Arrow/Tests/ArrowTests/IPCTests.swift b/swift/Arrow/Tests/ArrowTests/IPCTests.swift index 59cad94ef4da5..103c3b24c7b93 100644 --- a/swift/Arrow/Tests/ArrowTests/IPCTests.swift +++ b/swift/Arrow/Tests/ArrowTests/IPCTests.swift @@ -64,14 +64,16 @@ func makeSchema() -> ArrowSchema { return schemaBuilder.addField("col1", type: ArrowType(ArrowType.ArrowUInt8), isNullable: true) .addField("col2", type: ArrowType(ArrowType.ArrowString), isNullable: false) .addField("col3", type: ArrowType(ArrowType.ArrowDate32), isNullable: false) + .addField("col4", type: ArrowType(ArrowType.ArrowInt32), isNullable: false) + .addField("col5", type: ArrowType(ArrowType.ArrowFloat), isNullable: false) .finish() } func makeRecordBatch() throws -> RecordBatch { let uint8Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() uint8Builder.append(10) - uint8Builder.append(22) - uint8Builder.append(33) + uint8Builder.append(nil) + uint8Builder.append(nil) uint8Builder.append(44) let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() stringBuilder.append("test10") @@ -85,13 +87,28 @@ func makeRecordBatch() throws -> RecordBatch { date32Builder.append(date2) date32Builder.append(date1) date32Builder.append(date2) - let intHolder = ArrowArrayHolder(try uint8Builder.finish()) + let int32Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + int32Builder.append(1) + int32Builder.append(2) + int32Builder.append(3) + int32Builder.append(4) + let floatBuilder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + floatBuilder.append(211.112) + floatBuilder.append(322.223) + floatBuilder.append(433.334) + floatBuilder.append(544.445) + + let uint8Holder = ArrowArrayHolder(try uint8Builder.finish()) let stringHolder = ArrowArrayHolder(try stringBuilder.finish()) let date32Holder = ArrowArrayHolder(try date32Builder.finish()) + let int32Holder = ArrowArrayHolder(try int32Builder.finish()) + let floatHolder = ArrowArrayHolder(try floatBuilder.finish()) let result = RecordBatch.Builder() - .addColumn("col1", arrowArray: intHolder) + .addColumn("col1", arrowArray: uint8Holder) .addColumn("col2", arrowArray: stringHolder) .addColumn("col3", arrowArray: date32Holder) + .addColumn("col4", arrowArray: int32Holder) + .addColumn("col5", arrowArray: floatHolder) .finish() switch result { case .success(let recordBatch): @@ -182,15 +199,20 @@ final class IPCFileReaderTests: XCTestCase { XCTAssertEqual(recordBatches.count, 1) for recordBatch in recordBatches { XCTAssertEqual(recordBatch.length, 4) - XCTAssertEqual(recordBatch.columns.count, 3) - XCTAssertEqual(recordBatch.schema.fields.count, 3) + XCTAssertEqual(recordBatch.columns.count, 5) + XCTAssertEqual(recordBatch.schema.fields.count, 5) XCTAssertEqual(recordBatch.schema.fields[0].name, "col1") XCTAssertEqual(recordBatch.schema.fields[0].type.info, ArrowType.ArrowUInt8) XCTAssertEqual(recordBatch.schema.fields[1].name, "col2") XCTAssertEqual(recordBatch.schema.fields[1].type.info, ArrowType.ArrowString) XCTAssertEqual(recordBatch.schema.fields[2].name, "col3") XCTAssertEqual(recordBatch.schema.fields[2].type.info, ArrowType.ArrowDate32) + XCTAssertEqual(recordBatch.schema.fields[3].name, "col4") + XCTAssertEqual(recordBatch.schema.fields[3].type.info, ArrowType.ArrowInt32) + XCTAssertEqual(recordBatch.schema.fields[4].name, "col5") + XCTAssertEqual(recordBatch.schema.fields[4].type.info, ArrowType.ArrowFloat) let columns = recordBatch.columns + XCTAssertEqual(columns[0].nullCount, 2) let dateVal = "\((columns[2].array as! AsString).asString(0))" // swiftlint:disable:this force_cast XCTAssertEqual(dateVal, "2014-09-10 00:00:00 +0000") @@ -227,13 +249,17 @@ final class IPCFileReaderTests: XCTestCase { case .success(let result): XCTAssertNotNil(result.schema) let schema = result.schema! - XCTAssertEqual(schema.fields.count, 3) + XCTAssertEqual(schema.fields.count, 5) XCTAssertEqual(schema.fields[0].name, "col1") XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowUInt8) XCTAssertEqual(schema.fields[1].name, "col2") XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString) XCTAssertEqual(schema.fields[2].name, "col3") XCTAssertEqual(schema.fields[2].type.info, ArrowType.ArrowDate32) + XCTAssertEqual(schema.fields[3].name, "col4") + XCTAssertEqual(schema.fields[3].type.info, ArrowType.ArrowInt32) + XCTAssertEqual(schema.fields[4].name, "col5") + XCTAssertEqual(schema.fields[4].type.info, ArrowType.ArrowFloat) case.failure(let error): throw error } diff --git a/swift/Arrow/Tests/ArrowTests/RecordBatchTests.swift b/swift/Arrow/Tests/ArrowTests/RecordBatchTests.swift index ab6cad1b5e409..8820f1cdb1a91 100644 --- a/swift/Arrow/Tests/ArrowTests/RecordBatchTests.swift +++ b/swift/Arrow/Tests/ArrowTests/RecordBatchTests.swift @@ -23,9 +23,11 @@ final class RecordBatchTests: XCTestCase { let uint8Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() uint8Builder.append(10) uint8Builder.append(22) + uint8Builder.append(nil) let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() stringBuilder.append("test10") stringBuilder.append("test22") + stringBuilder.append("test33") let intHolder = ArrowArrayHolder(try uint8Builder.finish()) let stringHolder = ArrowArrayHolder(try stringBuilder.finish()) @@ -39,15 +41,16 @@ final class RecordBatchTests: XCTestCase { XCTAssertEqual(schema.fields.count, 2) XCTAssertEqual(schema.fields[0].name, "col1") XCTAssertEqual(schema.fields[0].type.info, ArrowType.ArrowUInt8) - XCTAssertEqual(schema.fields[0].isNullable, false) + XCTAssertEqual(schema.fields[0].isNullable, true) XCTAssertEqual(schema.fields[1].name, "col2") XCTAssertEqual(schema.fields[1].type.info, ArrowType.ArrowString) XCTAssertEqual(schema.fields[1].isNullable, false) XCTAssertEqual(recordBatch.columns.count, 2) let col1: ArrowArray = recordBatch.data(for: 0) let col2: ArrowArray = recordBatch.data(for: 1) - XCTAssertEqual(col1.length, 2) - XCTAssertEqual(col2.length, 2) + XCTAssertEqual(col1.length, 3) + XCTAssertEqual(col2.length, 3) + XCTAssertEqual(col1.nullCount, 1) case .failure(let error): throw error } From 7e703aae55c150c3556fe0dc972b575460cfb86b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sun, 14 Jan 2024 15:43:00 +0100 Subject: [PATCH 16/92] GH-39588: [CI][Go] Add CGO_ENABLED=1 to cdata_integration build to fix macOS build with conda (#39589) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change CI job has been failing since we added integration tests. ### What changes are included in this PR? Add `CGO_ENABLED=1` to go build cdata_integration on the verification script. ### Are these changes tested? Yes via archery. ### Are there any user-facing changes? No * Closes: #39588 Authored-by: Raúl Cumplido Signed-off-by: Raúl Cumplido --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index f12a5dc8b8964..ab5c476768ed5 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -935,7 +935,7 @@ test_go() { go_lib="arrow_go_integration.dll" ;; esac - go build -buildvcs=false -tags cdata_integration,assert -buildmode=c-shared -o ${go_lib} . + CGO_ENABLED=1 go build -buildvcs=false -tags cdata_integration,assert -buildmode=c-shared -o ${go_lib} . popd fi go clean -modcache From d7bc55542e6187a34c27f845de0bea78f6061de2 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 15 Jan 2024 05:58:05 -0600 Subject: [PATCH 17/92] MINOR: [R] Clean up docs (#39591) I noticed a few docs that needed cleaning up when running make commands Authored-by: Jonathan Keane Signed-off-by: Dewey Dunnington --- r/R/dplyr-funcs-doc.R | 4 ++-- r/inst/NOTICE.txt | 2 +- r/man/acero.Rd | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/r/R/dplyr-funcs-doc.R b/r/R/dplyr-funcs-doc.R index 492729df8c12a..2042f800142b7 100644 --- a/r/R/dplyr-funcs-doc.R +++ b/r/R/dplyr-funcs-doc.R @@ -21,7 +21,7 @@ #' #' The `arrow` package contains methods for 37 `dplyr` table functions, many of #' which are "verbs" that do transformations to one or more tables. -#' The package also has mappings of 211 R functions to the corresponding +#' The package also has mappings of 212 R functions to the corresponding #' functions in the Arrow compute library. These allow you to write code inside #' of `dplyr` methods that call R functions, including many in packages like #' `stringr` and `lubridate`, and they will get translated to Arrow and run @@ -83,7 +83,7 @@ #' Functions can be called either as `pkg::fun()` or just `fun()`, i.e. both #' `str_sub()` and `stringr::str_sub()` work. #' -#' In addition to these functions, you can call any of Arrow's 254 compute +#' In addition to these functions, you can call any of Arrow's 262 compute #' functions directly. Arrow has many functions that don't map to an existing R #' function. In other cases where there is an R function mapping, you can still #' call the Arrow function directly if you don't want the adaptations that the R diff --git a/r/inst/NOTICE.txt b/r/inst/NOTICE.txt index a609791374c28..2089c6fb20358 100644 --- a/r/inst/NOTICE.txt +++ b/r/inst/NOTICE.txt @@ -1,5 +1,5 @@ Apache Arrow -Copyright 2016-2019 The Apache Software Foundation +Copyright 2016-2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/r/man/acero.Rd b/r/man/acero.Rd index 12afdc23138ac..365795d9fc65c 100644 --- a/r/man/acero.Rd +++ b/r/man/acero.Rd @@ -9,7 +9,7 @@ \description{ The \code{arrow} package contains methods for 37 \code{dplyr} table functions, many of which are "verbs" that do transformations to one or more tables. -The package also has mappings of 211 R functions to the corresponding +The package also has mappings of 212 R functions to the corresponding functions in the Arrow compute library. These allow you to write code inside of \code{dplyr} methods that call R functions, including many in packages like \code{stringr} and \code{lubridate}, and they will get translated to Arrow and run @@ -71,7 +71,7 @@ can assume that the function works in Acero just as it does in R. Functions can be called either as \code{pkg::fun()} or just \code{fun()}, i.e. both \code{str_sub()} and \code{stringr::str_sub()} work. -In addition to these functions, you can call any of Arrow's 254 compute +In addition to these functions, you can call any of Arrow's 262 compute functions directly. Arrow has many functions that don't map to an existing R function. In other cases where there is an R function mapping, you can still call the Arrow function directly if you don't want the adaptations that the R From 7acbaf45ce2d5be31e70b552d1a24476c67383e6 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 15 Jan 2024 15:02:45 +0100 Subject: [PATCH 18/92] GH-39504: [Docs] Update footer in main sphinx docs with correct attribution (#39505) * Closes: #39504 Lead-authored-by: Joris Van den Bossche Co-authored-by: Alenka Frim Signed-off-by: AlenkaF --- docs/source/conf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index cde0c2b31f8fd..5af7b7955fdde 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -201,7 +201,12 @@ # General information about the project. project = u'Apache Arrow' -copyright = f'2016-{datetime.datetime.now().year} Apache Software Foundation' +copyright = ( + f"2016-{datetime.datetime.now().year} Apache Software Foundation.\n" + "Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow " + "project logo are either registered trademarks or trademarks of The Apache " + "Software Foundation in the United States and other countries" +) author = u'Apache Software Foundation' # The version info for the project you're documenting, acts as replacement for From 1f83c8e9a1c8f862a13e2521f863978cb3c6683d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 10:32:02 -0800 Subject: [PATCH 19/92] MINOR: [C#] Bump Google.Protobuf from 3.25.1 to 3.25.2 in /csharp (#39617) Bumps [Google.Protobuf](https://github.com/protocolbuffers/protobuf) from 3.25.1 to 3.25.2.
Commits
  • a9b006b Updating version.json and repo version numbers to: 25.2
  • 59eebe3 Merge pull request #14907 from mkruskal-google/cmake-install-fix
  • 05ad652 Only substitute prefixes during installation setup.
  • c709a34 Merge pull request #14800 from mkruskal-google/leak-patch
  • 3d5c709 Register a shutdown delete for C++ feature defaults
  • c529459 Merge pull request #14765 from protocolbuffers/25.x-202311152135
  • 6ac0447 Updating version.json and repo version numbers to: 25.2-dev
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Google.Protobuf&package-manager=nuget&previous-version=3.25.1&new-version=3.25.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj index aae26273ac282..68c3e47e01902 100644 --- a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj +++ b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj @@ -5,7 +5,7 @@ - + From 31d02314dabb2f6e422eda7951cb90905e78011f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 10:33:04 -0800 Subject: [PATCH 20/92] MINOR: [C#] Bump xunit from 2.6.5 to 2.6.6 in /csharp (#39618) Bumps [xunit](https://github.com/xunit/xunit) from 2.6.5 to 2.6.6.
Commits
  • f03fe09 v2.6.6
  • 280a0cb Update build to use .NET SDK 8 and C# 12 (#2863)
  • fdf75ab #2334: Add assembly-level support for BeforeAfterTestAttribute (v2)
  • ba06476 Add targetFramework for xunit.assert.nuspec
  • 9db7d30 Test out additional target framework dependencies in .nuspec files
  • 27e91e3 Bump up to 2.6.6-pre
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=xunit&package-manager=nuget&previous-version=2.6.5&new-version=2.6.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- .../Apache.Arrow.Compression.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Sql.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj | 2 +- csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj index 7a93d8f92635b..8ed7a93bdcf27 100644 --- a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj +++ b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj index 7799577535ded..b5e7170a8c31d 100644 --- a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj index 972aa178eabe8..a7c52846fd9a4 100644 --- a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj index afb636123b37b..d8a92ff756751 100644 --- a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj +++ b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj @@ -15,7 +15,7 @@ - + all runtime; build; native; contentfiles; analyzers From 809244e4cb3dadc8742a891342feda4ef1b27e1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:51:55 -0500 Subject: [PATCH 21/92] MINOR: [Java] Bump org.apache.maven.plugins:maven-jar-plugin from 2.4 to 3.3.0 in /java (#39612) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.maven.plugins:maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 2.4 to 3.3.0.
Release notes

Sourced from org.apache.maven.plugins:maven-jar-plugin's releases.

3.3.0

🚀 New features and improvements

🐛 Bug Fixes

📦 Dependency updates

📝 Documentation updates

  • Restore mavenArchiverVersion property used in the site (#51) @​jorsol
  • (doc) Updated create-test-jar.apt.vm removing 'and' in Maven site Create Test JAR documentation (#34) @​focbenz

👻 Maintenance

3.2.2

What's Changed

New Contributors

... (truncated)

Commits
  • d68df4b [maven-release-plugin] prepare release maven-jar-plugin-3.3.0
  • fb2299a Restore mavenArchiverVersion property used in the site
  • 1204127 [MJAR-290] - Update Plexus Utils to 3.4.2
  • 5fd2fc9 [MJAR-291] - Upgrade Parent to 37
  • 56344da use shared action v3 (#49)
  • 4148491 Code simplifications in AbstractMojo (#47)
  • 46c017d [MJAR-275] - Fix outputTimestamp not applied to module-info; breaks reproduci...
  • c02be20 [MJAR-278] Update plugin (requires Maven 3.2.5+) (#19)
  • b6fe3eb Bump junit from 4.11 to 4.13.2 in /src/it/MJAR-228
  • 78a28dd Ignore Maven Core updates
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.maven.plugins:maven-jar-plugin&package-manager=maven&previous-version=2.4&new-version=3.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/maven/module-info-compiler-maven-plugin/pom.xml | 2 +- java/performance/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index 46c0d563f4eb9..70d1993b33c6e 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -84,7 +84,7 @@ maven-jar-plugin - 3.0.2 + 3.3.0 maven-install-plugin diff --git a/java/performance/pom.xml b/java/performance/pom.xml index 5e0b6c1b54541..eff3240890beb 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -191,7 +191,7 @@ maven-jar-plugin - 2.4 + 3.3.0 maven-javadoc-plugin From b59082a72094f112f95914a1d8ad469db87a34c9 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Tue, 16 Jan 2024 01:21:15 +0100 Subject: [PATCH 22/92] MINOR: [CI] Update allowed_roles for crossbow submission (#39610) ### Rationale for this change Improved security, this aligns the permissions with the current default repo setting of required approval for all contributors. ### What changes are included in this PR? Only committers (members,owner and collaborator of ASF org) can submit a crossbow job. ### Are these changes tested? Not possible. Authored-by: Jacob Wujciak-Jens Signed-off-by: Sutou Kouhei --- dev/archery/archery/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py index 68b24dc08d71b..4e5104362254c 100644 --- a/dev/archery/archery/bot.py +++ b/dev/archery/archery/bot.py @@ -280,7 +280,7 @@ def handle_issue_comment(self, command, payload): # https://developer.github.com/v4/enum/commentauthorassociation/ # Checking privileges here enables the bot to respond # without relying on the handler. - allowed_roles = {'OWNER', 'MEMBER', 'CONTRIBUTOR', 'COLLABORATOR'} + allowed_roles = {'OWNER', 'MEMBER', 'COLLABORATOR'} if payload['comment']['author_association'] not in allowed_roles: raise EventError( "Only contributors can submit requests to this bot. " From 697b70ee0fa1f3015a45afaed725988aab4172f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:21:45 +0900 Subject: [PATCH 23/92] MINOR: [Java] Bump commons-codec:commons-codec from 1.15 to 1.16.0 in /java (#39611) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.15 to 1.16.0.
Changelog

Sourced from commons-codec:commons-codec's changelog.

          Apache Commons Codec 1.16.0

The Apache Commons Codec package contains simple encoder and decoders for various formats such as Base64 and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.

Feature and fix release.

Changes in this version include:

New features: o CODEC-296: Add support for Blake3 family of hashes. Thanks to Matt Sicker. o Add github/codeql-action.

Fixed Bugs: o CODEC-295: Minor improvements #67. Thanks to Arturo Bernal. o Remove duplicated words from Javadocs. Thanks to James Gan. o CODEC-301: Simplify assertion #84. Thanks to Alexander Pinske, Alex Herbert. o CODEC-300: Simplify assertion #84. Thanks to Arturo Bernal. o CODEC-298: Use Standard Charset object #82. Thanks to Arturo Bernal. o Use String.contains() functions #125. Thanks to Arturo Bernal. o Avoid use toString() or substring() in favor of a simplified expression #126. Thanks to Arturo Bernal. o CODEC-305: Fix byte-skipping in Base16 decoding #135. Thanks to Florian. o Fix several typos, improve writing in some javadocs #139. Thanks to Marc Wrobel. o BaseNCodecOutputStream.eof() should not throw IOException. Thanks to Gary Gregory. o Javadoc improvements and cleanups. Thanks to Gary Gregory. o Deprecate BaseNCodec.isWhiteSpace(byte) and use Character.isWhitespace(int). Thanks to Gary Gregory.

Changes: o Bump actions/cache from v2 to v3.0.10 #75, #99, #119, #138, #149, #152. Thanks to Dependabot, Gary Gregory. o Bump actions/setup-java from v1.4.1 to 3.5.1 #60, #62, #121. Thanks to Dependabot, Gary Gregory. o Bump actions/checkout from 2.3.2 to 3.1.0 #65, #98, #114, #153. Thanks to Dependabot, Gary Gregory. o Bump commons-parent from 52 to 58, #147, #165, #170. Thanks to Dependabot, Gary Gregory. o CODEC-285: Bump junit from 4.13.1 to 5.9.1 #76, #39, #140, #148. Thanks to Dependabot, John Patrick. o Bump Java 7 to 8. Thanks to Gary Gregory. o Bump japicmp-maven-plugin from 0.14.3 to 0.17.1. Thanks to Gary Gregory. o Bump jacoco-maven-plugin from 0.8.5 to 0.8.8 (Fixes Java 15 builds). Thanks to Gary Gregory. o Bump maven-surefire-plugin from 2.22.2 to 3.0.0-M7 #122, #134. Thanks to Gary Gregory. o Bump maven-javadoc-plugin from 3.2.0 to 3.4.1. Thanks to Gary Gregory. o Bump animal-sniffer-maven-plugin from 1.19 to 1.22. Thanks to Gary Gregory. o Bump maven-pmd-plugin from 3.13.0 to 3.19.0, #133, #142, #145. Thanks to Gary Gregory, Dependabot. o Bump pmd from 6.47.0 to 6.52.0. Thanks to Gary Gregory. o Bump maven-checkstyle-plugin from 2.17 to 3.2.0 #143. Thanks to Gary Gregory. o Bump checkstyle from 8.45.1 to 9.3 #97, #100, #101, #103. Thanks to Dependabot. o Bump taglist-maven-plugin from 2.4 to 3.0.0 #102. Thanks to Dependabot. o Bump jacoco-maven-plugin from 0.8.7 to 0.8.8. Thanks to Gary Gregory.

For complete information on Apache Commons Codec, including instructions on how to submit bug reports,

... (truncated)

Commits
  • 2614a4c Prepare for release candidate
  • 3a6cde0 Update POM version for Apache Commons Codec release 1.16.0
  • fa289a9 Prepare for release candidate
  • 393b8f7 Prepare for release candidate
  • 7bfa8af Prepare for release candidate
  • 18d83b1 Prepare for release candidate
  • a3d825d Package private method does not need Javadoc since tag
  • 4d9f462 New package private method does not need Javadoc since tag
  • af4d48a New package private method does not need Javadoc since tag
  • cc2808b Make new API Fluent
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commons-codec:commons-codec&package-manager=maven&previous-version=1.15&new-version=1.16.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/vector/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/vector/pom.xml b/java/vector/pom.xml index a4292449c9cb2..da26fc2982765 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -53,7 +53,7 @@ commons-codec commons-codec - 1.15 + 1.16.0 org.apache.arrow From f57feb6505d87b572bec77fd39876a0dd271118c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:23:59 +0900 Subject: [PATCH 24/92] MINOR: [Java] Bump org.jacoco:jacoco-maven-plugin from 0.8.7 to 0.8.11 in /java (#39615) Bumps [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.7 to 0.8.11.
Release notes

Sourced from org.jacoco:jacoco-maven-plugin's releases.

0.8.11

New Features

  • JaCoCo now officially supports Java 21 (GitHub #1520).
  • Experimental support for Java 22 class files (GitHub #1479).
  • Part of bytecode generated by the Java compilers for exhaustive switch expressions is filtered out during generation of report (GitHub #1472).
  • Part of bytecode generated by the Java compilers for record patterns is filtered out during generation of report (GitHub #1473).

Fixed bugs

  • Instrumentation should not cause VerifyError when the last local variable of method parameters is overridden in the method body to store a value of type long or double (GitHub #893).
  • Restore exec file compatibility with versions from 0.7.5 to 0.8.8 in case of class files with zero line numbers (GitHub #1492).

Non-functional Changes

  • jacoco-maven-plugin now requires at least Java 8 (GitHub #1466, #1468).
  • JaCoCo build now requires at least Maven 3.5.4 (GitHub #1467).
  • Maven 3.9.2 should not produce warnings for jacoco-maven-plugin (GitHub #1468).
  • JaCoCo build now requires JDK 17 (GitHub #1482).
  • JaCoCo now depends on ASM 9.6 (GitHub #1518).

0.8.10

Fixed bugs

  • Agent should not require configuration of permissions for SecurityManager outside of its codeBase (GitHub #1425).

0.8.9

New Features

  • JaCoCo now officially supports Java 19 and 20 (GitHub #1371, #1386).
  • Experimental support for Java 21 class files (GitHub #1386).
  • Add parameter to include the current project in the report-aggregate Maven goal (GitHub #1007).
  • Component accessors generated by the Java compilers for records are filtered out during generation of report. Contributed by Tesla Zhang (GitHub #1393).

Fixed bugs

  • Agent should not open java.lang package to unnamed module of the application class loader (GitHub #1334).

Non-functional Changes

  • JaCoCo now depends on ASM 9.5 (GitHub #1299, #1368, #1416).
  • JaCoCo build now requires JDK 11 (GitHub #1413).

0.8.8

New Features

  • JaCoCo now officially supports Java 17 and 18 (GitHub #1282, #1198).
  • Experimental support for Java 19 class files (GitHub #1264).
  • Part of bytecode generated by the Java compilers for assert statement is filtered out during generation of report (GitHub #1196).
  • Branch added by the Kotlin compiler version 1.6.0 and above for "unsafe" cast operator is filtered out during generation of report (GitHub #1266).
  • Improved support for multiple JaCoCo runtimes in the same VM (GitHub #1057).

Fixed bugs

  • Fixed NullPointerException during filtering (GitHub #1189).
  • Fix range for debug symbols of method parameters (GitHub #1246).

Non-functional Changes

  • JaCoCo now depends on ASM 9.2 (GitHub #1206).
  • Messages of exceptions occurring during analysis or instrumentation now include JaCoCo version (GitHub #1217).
Commits
  • f33756c Prepare release 0.8.11
  • 0670530 Upgrade animal-sniffer-maven-plugin to 1.23
  • 206e5be Restore exec file compatibility after upgrade of ASM to version 9.5 (#1492)
  • 36fc079 Update documentation: JDK version 21 is officially supported (#1520)
  • 7162917 Add validation tests for boolean expressions (#1505)
  • 4bc9267 Fix link to Bytecode Outline Plug-In (#1519)
  • ded62fc Upgrade ASM to 9.6 (#1518)
  • 6798260 Fix links to ASM website (#1515)
  • 4ba332f Fix misleading outdated javadoc (#1513)
  • 7ca0f0f Opcodes.RET should be processed by visitVarInsn instead of visitInsn (#...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.jacoco:jacoco-maven-plugin&package-manager=maven&previous-version=0.8.7&new-version=0.8.11)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 042488a5b949a..8a8d576320300 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -496,7 +496,7 @@ org.jacoco jacoco-maven-plugin - 0.8.7 + 0.8.11 From 0d128c6d01ccf247c2922752e589c763599ded09 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 15 Jan 2024 20:12:15 -0500 Subject: [PATCH 25/92] GH-39601: [R] Don't download cmake when TEST_OFFLINE_BUILD=true (#39602) See #39601 ### Are these changes tested? Existing CI should pass. This should also pass on macbuilder without downloading cmake, and if hardcoding `download_ok <- FALSE`, it should exit cleanly and informatively. ### Are there any user-facing changes? Define "user". * Closes: #39601 Authored-by: Neal Richardson Signed-off-by: Jacob Wujciak-Jens --- r/DESCRIPTION | 3 ++- r/tools/nixlibs.R | 38 ++++++++++++++++++++++---------- r/vignettes/developers/setup.Rmd | 3 +-- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/r/DESCRIPTION b/r/DESCRIPTION index b290a75f932d5..4acd21269cc49 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -27,7 +27,8 @@ URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/ BugReports: https://github.com/apache/arrow/issues Encoding: UTF-8 Language: en-US -SystemRequirements: C++17; for AWS S3 support on Linux, libcurl and openssl (optional) +SystemRequirements: C++17; for AWS S3 support on Linux, libcurl and openssl (optional); + cmake >= 3.16 (build-time only, and only for full source build) Biarch: true Imports: assertthat, diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 9027aa227a074..cb664388094b0 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -79,6 +79,10 @@ find_latest_nightly <- function(description_version, } try_download <- function(from_url, to_file, hush = quietly) { + if (!download_ok) { + # Don't even try + return(FALSE) + } # We download some fairly large files, so ensure the timeout is set appropriately. # This assumes a static library size of 100 MB (generous) and a download speed # of .3 MB/s (slow). This is to anticipate slower user connections or load on @@ -496,7 +500,7 @@ build_libarrow <- function(src_dir, dst_dir) { Sys.setenv(MAKEFLAGS = makeflags) } if (!quietly) { - lg("Building with MAKEFLAGS=", makeflags) + lg("Building with MAKEFLAGS=%s", makeflags) } # Check for libarrow build dependencies: # * cmake @@ -595,7 +599,6 @@ ensure_cmake <- function(cmake_minimum_required = "3.16") { if (is.null(cmake)) { # If not found, download it - lg("cmake", .indent = "****") CMAKE_VERSION <- Sys.getenv("CMAKE_VERSION", "3.26.4") if (on_macos) { postfix <- "-macos-universal.tar.gz" @@ -642,10 +645,7 @@ ensure_cmake <- function(cmake_minimum_required = "3.16") { bin_dir, "/cmake" ) - } else { - # Show which one we found - # Full source builds will always show "cmake" in the logs - lg("cmake: %s", cmake, .indent = "****") + lg("cmake %s", CMAKE_VERSION, .indent = "****") } cmake } @@ -653,6 +653,8 @@ ensure_cmake <- function(cmake_minimum_required = "3.16") { find_cmake <- function(paths = c( Sys.getenv("CMAKE"), Sys.which("cmake"), + # CRAN has it here, not on PATH + if (on_macos) "/Applications/CMake.app/Contents/bin/cmake", Sys.which("cmake3") ), version_required = "3.16") { @@ -660,10 +662,25 @@ find_cmake <- function(paths = c( # version_required should be a string or packageVersion; numeric version # can be misleading (e.g. 3.10 is actually 3.1) for (path in paths) { - if (nzchar(path) && cmake_version(path) >= version_required) { + if (nzchar(path) && file.exists(path)) { # Sys.which() returns a named vector, but that plays badly with c() later names(path) <- NULL - return(path) + found_version <- cmake_version(path) + if (found_version >= version_required) { + # Show which one we found + lg("cmake %s: %s", found_version, path, .indent = "****") + # Stop searching here + return(path) + } else { + # Keep trying + lg("Not using cmake found at %s", path, .indent = "****") + if (found_version > 0) { + lg("Version >= %s required; found %s", version_required, found_version, .indent = "*****") + } else { + # If cmake_version() couldn't determine version, it returns 0 + lg("Could not determine version; >= %s required", version_required, .indent = "*****") + } + } } } # If none found, return NULL @@ -890,11 +907,8 @@ if (not_cran || on_macos) { # and don't fall back to a full source build build_ok <- !env_is("LIBARROW_BUILD", "false") -# Check if we're authorized to download (not asked an offline build). -# (Note that cmake will still be downloaded if necessary -# https://arrow.apache.org/docs/developers/cpp/building.html#offline-builds) +# Check if we're authorized to download download_ok <- !test_mode && !env_is("TEST_OFFLINE_BUILD", "true") - download_libarrow_ok <- download_ok && !env_is("LIBARROW_DOWNLOAD", "false") # This "tools/thirdparty_dependencies" path, within the tar file, might exist if diff --git a/r/vignettes/developers/setup.Rmd b/r/vignettes/developers/setup.Rmd index 8e7cff7410473..119bc78419410 100644 --- a/r/vignettes/developers/setup.Rmd +++ b/r/vignettes/developers/setup.Rmd @@ -281,11 +281,10 @@ withr::with_makevars(list(CPPFLAGS = "", LDFLAGS = ""), remotes::install_github( environment variables that determine how the build works and what features get built. * `TEST_OFFLINE_BUILD`: When set to `true`, the build script will not download - prebuilt the C++ library binary. + prebuilt the C++ library binary or, if needed, `cmake`. It will turn off any features that require a download, unless they're available in `ARROW_THIRDPARTY_DEPENDENCY_DIR` or the `tools/thirdparty_download/` subfolder. `create_package_with_all_dependencies()` creates that subfolder. - Regardless of this flag's value, `cmake` will be downloaded if it's unavailable. # Troubleshooting From 1df2e4ac828fe1fc1034e47bc5233cb312337bd4 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 15 Jan 2024 21:35:58 -0600 Subject: [PATCH 26/92] GH-39584: [R] fallback to source gracefully (#39587) ### Rationale for this change Resolves #39584 ### What changes are included in this PR? We now only check the checksum after the download succeeded, and try to be quieter about it when we do. We also use bundled boost and lz4 source on macos by default (to avoid system versions of each on cran that seem to have issues) ### Are these changes tested? I submitted a download-malignant (and verbose) build to [CRAN's macbuilder](https://mac.r-project.org/macbuilder/results/1705088784-991a5beacf4ec26e/) and it succeeds. ### Are there any user-facing changes? In principle the macos source build is slightly altered + we have a cleaner path when file downloads fail. But both of these should be relatively non-impactful since most macos users are getting binaries from CRAN. Most importantly it helps us stay on CRAN. **This PR contains a "Critical Fix".** * Closes: #39584 Lead-authored-by: Jonathan Keane Co-authored-by: Jacob Wujciak-Jens Signed-off-by: Jacob Wujciak-Jens --- r/inst/build_arrow_static.sh | 2 + r/tools/nixlibs.R | 101 +++++++++++++++++++++++------------ 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index 9c9fadea4757b..d28cbcb08fbec 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -74,6 +74,8 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_DATASET=${ARROW_DATASET:-ON} \ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ + -DBoost_SOURCE=${Boost_SOURCE:-} \ + -Dlz4_SOURCE=${lz4_SOURCE:-} \ -DARROW_FILESYSTEM=ON \ -DARROW_GCS=${ARROW_GCS:-$ARROW_DEFAULT_PARAM} \ -DARROW_JEMALLOC=${ARROW_JEMALLOC:-$ARROW_DEFAULT_PARAM} \ diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index cb664388094b0..dfe379ebe20df 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -100,18 +100,7 @@ try_download <- function(from_url, to_file, hush = quietly) { !inherits(status, "try-error") && status == 0 } -download_binary <- function(lib) { - libfile <- paste0("arrow-", VERSION, ".zip") - binary_url <- paste0(arrow_repo, "bin/", lib, "/arrow-", VERSION, ".zip") - if (try_download(binary_url, libfile)) { - lg("Successfully retrieved libarrow (%s)", lib) - } else { - lg( - "Downloading libarrow failed for version %s (%s)\n at %s", - VERSION, lib, binary_url - ) - libfile <- NULL - } +validate_checksum <- function(binary_url, libfile, hush = quietly) { # Explicitly setting the env var to "false" will skip checksum validation # e.g. in case the included checksums are stale. skip_checksum <- env_is("ARROW_R_ENFORCE_CHECKSUM", "false") @@ -120,33 +109,66 @@ download_binary <- function(lib) { # validate binary checksum for CRAN release only if (!skip_checksum && dir.exists(checksum_path) && is_release || enforce_checksum) { + # Munge the path to the correct sha file which we include during the + # release process checksum_file <- sub(".+/bin/(.+\\.zip)", "\\1\\.sha512", binary_url) checksum_file <- file.path(checksum_path, checksum_file) - checksum_cmd <- "shasum" - checksum_args <- c("--status", "-a", "512", "-c", checksum_file) - - # shasum is not available on all linux versions - status_shasum <- try( - suppressWarnings( - system2("shasum", args = c("--help"), stdout = FALSE, stderr = FALSE) - ), - silent = TRUE - ) - if (inherits(status_shasum, "try-error") || is.integer(status_shasum) && status_shasum != 0) { - checksum_cmd <- "sha512sum" - checksum_args <- c("--status", "-c", checksum_file) + # Try `shasum`, and if that doesn't work, fall back to `sha512sum` if not found + # system2 doesn't generate an R error, so we can't use a tryCatch to + # move from shasum to sha512sum. + # The warnings from system2 if it fails pop up later in the log and thus are + # more confusing than they are helpful (so we suppress them) + checksum_ok <- suppressWarnings(system2( + "shasum", + args = c("--status", "-a", "512", "-c", checksum_file), + stdout = ifelse(quietly, FALSE, ""), + stderr = ifelse(quietly, FALSE, "") + )) == 0 + + if (!checksum_ok) { + checksum_ok <- suppressWarnings(system2( + "sha512sum", + args = c("--status", "-c", checksum_file), + stdout = ifelse(quietly, FALSE, ""), + stderr = ifelse(quietly, FALSE, "") + )) == 0 } - checksum_ok <- system2(checksum_cmd, args = checksum_args) - - if (checksum_ok != 0) { - lg("Checksum validation failed for libarrow: %s/%s", lib, libfile) - unlink(libfile) - libfile <- NULL + if (checksum_ok) { + lg("Checksum validated successfully for libarrow") } else { - lg("Checksum validated successfully for libarrow: %s/%s", lib, libfile) + lg("Checksum validation failed for libarrow") + unlink(libfile) } + } else { + checksum_ok <- TRUE + } + + # Return whether the checksum was successful + checksum_ok +} + +download_binary <- function(lib) { + libfile <- paste0("arrow-", VERSION, ".zip") + binary_url <- paste0(arrow_repo, "bin/", lib, "/arrow-", VERSION, ".zip") + if (try_download(binary_url, libfile) && validate_checksum(binary_url, libfile)) { + lg("Successfully retrieved libarrow (%s)", lib) + } else { + # If the download or checksum fail, we will set libfile to NULL this will + # normally result in a source build after this. + # TODO: should we condense these together and only call them when verbose? + lg( + "Unable to retrieve libarrow for version %s (%s)", + VERSION, lib + ) + if (!quietly) { + lg( + "Attempted to download the libarrow binary from: %s", + binary_url + ) + } + libfile <- NULL } libfile @@ -468,7 +490,7 @@ env_vars_as_string <- function(env_var_list) { stopifnot( length(env_var_list) == length(names(env_var_list)), all(grepl("^[^0-9]", names(env_var_list))), - all(grepl("^[A-Z0-9_]+$", names(env_var_list))), + all(grepl("^[a-zA-Z0-9_]+$", names(env_var_list))), !any(grepl("'", env_var_list, fixed = TRUE)) ) env_var_string <- paste0(names(env_var_list), "='", env_var_list, "'", collapse = " ") @@ -543,6 +565,19 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- c(env_var_list, ARROW_DEPENDENCY_SOURCE = "BUNDLED") } + # On macOS, if not otherwise set, let's override Boost_SOURCE to be bundled + # Necessary due to #39590 for CRAN + if (on_macos) { + # Using lowercase (e.g. Boost_SOURCE) to match the cmake args we use already. + deps_to_bundle <- c("Boost", "lz4") + for (dep_to_bundle in deps_to_bundle) { + env_var <- paste0(dep_to_bundle, "_SOURCE") + if (Sys.getenv(env_var) == "") { + env_var_list <- c(env_var_list, setNames("BUNDLED", env_var)) + } + } + } + env_var_list <- with_cloud_support(env_var_list) # turn_off_all_optional_features() needs to happen after From ac50918a63f13088a5ea9e6c27a9fadbbb19d53f Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Tue, 16 Jan 2024 04:38:55 +0100 Subject: [PATCH 27/92] GH-39624: [R][CI] Add CMake to docker file and update envvars (#39625) ### Rationale for this change CMake is now a sysreq and we don't want to default to using nightly builds in CI ### Are these changes tested? Crossbos * Closes: #39624 Authored-by: Jacob Wujciak-Jens Signed-off-by: Jacob Wujciak-Jens --- ci/scripts/r_docker_configure.sh | 5 +++-- docker-compose.yml | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 1cbd5f0b5ea96..52db2e6df6611 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -91,8 +91,9 @@ if [ -f "${ARROW_SOURCE_HOME}/ci/scripts/r_install_system_dependencies.sh" ]; th "${ARROW_SOURCE_HOME}/ci/scripts/r_install_system_dependencies.sh" fi -# Install rsync for bundling cpp source and curl to make sure it is installed on all images -$PACKAGE_MANAGER install -y rsync curl +# Install rsync for bundling cpp source and curl to make sure it is installed on all images, +# cmake is now a listed sys req. +$PACKAGE_MANAGER install -y rsync cmake curl # Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786 Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}' diff --git a/docker-compose.yml b/docker-compose.yml index 39cd473c2741b..14eff67f38971 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1414,6 +1414,7 @@ services: ARROW_DEPENDENCY_SOURCE: '' ARROW_SOURCE_HOME: '/arrow' FORCE_BUNDLED_BUILD: 'true' + LIBARROW_BINARY: 'false' LIBARROW_BUILD: 'true' extends: ubuntu-r command: > @@ -1443,7 +1444,7 @@ services: shm_size: *shm-size environment: <<: [*common, *sccache] - LIBARROW_DOWNLOAD: "false" + LIBARROW_BINARY: "false" ARROW_SOURCE_HOME: "/arrow" ARROW_R_DEV: ${ARROW_R_DEV} # To test for CRAN release, delete ^^ these two env vars so we download the Apache release From eec4942d9ceb7d6c5632584a53363c627f8fd599 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Tue, 16 Jan 2024 07:22:16 -0500 Subject: [PATCH 28/92] GH-39604: [JS] Do not use resizable buffers yet (#39607) --- js/src/builder/buffer.ts | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/js/src/builder/buffer.ts b/js/src/builder/buffer.ts index 18c6dcda738b9..ad1c06b0d9f0f 100644 --- a/js/src/builder/buffer.ts +++ b/js/src/builder/buffer.ts @@ -27,30 +27,17 @@ function roundLengthUpToNearest64Bytes(len: number, BPE: number) { /** @ignore */ function resizeArray(arr: T, len = 0): T { - // TODO: remove when https://github.com/microsoft/TypeScript/issues/54636 is fixed - const buffer = arr.buffer as ArrayBufferLike & { resizable: boolean; resize: (byteLength: number) => void; maxByteLength: number }; - const byteLength = len * arr.BYTES_PER_ELEMENT; - if (buffer.resizable && byteLength <= buffer.maxByteLength) { - buffer.resize(byteLength); - return arr; - } - - // Fallback for non-resizable buffers return arr.length >= len ? arr.subarray(0, len) as T : memcpy(new (arr.constructor as any)(len), arr, 0); } -/** @ignore */ -export const SAFE_ARRAY_SIZE = 2 ** 32 - 1; - /** @ignore */ export class BufferBuilder { constructor(bufferType: ArrayCtor, initialSize = 0, stride = 1) { this.length = Math.ceil(initialSize / stride); - // TODO: remove as any when https://github.com/microsoft/TypeScript/issues/54636 is fixed - this.buffer = new bufferType(new (ArrayBuffer as any)(this.length * bufferType.BYTES_PER_ELEMENT, { maxByteLength: SAFE_ARRAY_SIZE })) as T; + this.buffer = new bufferType(this.length) as T; this.stride = stride; this.BYTES_PER_ELEMENT = bufferType.BYTES_PER_ELEMENT; this.ArrayType = bufferType; @@ -94,8 +81,7 @@ export class BufferBuilder { } public clear() { this.length = 0; - // TODO: remove as any when https://github.com/microsoft/TypeScript/issues/54636 is fixed - this.buffer = new this.ArrayType(new (ArrayBuffer as any)(0, { maxByteLength: SAFE_ARRAY_SIZE })) as T; + this.buffer = new this.ArrayType() as T; return this; } protected _resize(newLength: number) { From d6a8305c3c52abd6c1ed65b6081dea323a4d0247 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Tue, 16 Jan 2024 04:36:35 -0800 Subject: [PATCH 29/92] GH-39598: [C#] Fix verification script (#39605) ### What changes are included in this PR? The verification script is modified to look for the versions of .NET now supported by the package. ### Are these changes tested? Manually tested the verification command. * Closes: #39598 Authored-by: Curt Hagenlocher Signed-off-by: Curt Hagenlocher --- dev/release/verify-release-candidate.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index ab5c476768ed5..c5e27d083013e 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -878,10 +878,10 @@ test_csharp() { fi if [ "${SOURCE_KIND}" = "local" ]; then - echo "Skipping sourelink verification on local build" + echo "Skipping sourcelink verification on local build" else - dotnet tool run sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb - dotnet tool run sourcelink test artifacts/Apache.Arrow/Release/netcoreapp3.1/Apache.Arrow.pdb + dotnet tool run sourcelink test artifacts/Apache.Arrow/Release/netstandard2.0/Apache.Arrow.pdb + dotnet tool run sourcelink test artifacts/Apache.Arrow/Release/net6.0/Apache.Arrow.pdb fi popd From b233b0195de1b532fb92b4dad039b17b9427f428 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 08:54:55 -0500 Subject: [PATCH 30/92] MINOR: [Java] Bump dep.slf4j.version from 1.7.5 to 2.0.11 in /java (#39613) Bumps `dep.slf4j.version` from 1.7.5 to 2.0.11. Updates `org.slf4j:jcl-over-slf4j` from 1.7.5 to 2.0.11 Updates `org.slf4j:slf4j-api` from 2.0.9 to 2.0.11 Updates `org.slf4j:jul-to-slf4j` from 2.0.9 to 2.0.11 Updates `org.slf4j:log4j-over-slf4j` from 2.0.9 to 2.0.11 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/maven/pom.xml | 2 +- java/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/maven/pom.xml b/java/maven/pom.xml index 56f3c4c434f64..6e8a4cb0102f6 100644 --- a/java/maven/pom.xml +++ b/java/maven/pom.xml @@ -240,7 +240,7 @@ org.slf4j jcl-over-slf4j - 1.7.5 + 2.0.11 diff --git a/java/pom.xml b/java/pom.xml index 8a8d576320300..147f08fe57030 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -31,7 +31,7 @@ ${project.build.directory}/generated-sources 1.9.0 5.10.1 - 2.0.9 + 2.0.11 33.0.0-jre 4.1.104.Final 1.60.0 From a2be302b4f499b39b3d281b677c75e8f640ad7c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 08:55:52 -0500 Subject: [PATCH 31/92] MINOR: [Java] Bump org.apache.orc:orc-core from 1.7.6 to 1.9.2 in /java (#39614) Bumps org.apache.orc:orc-core from 1.7.6 to 1.9.2. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.orc:orc-core&package-manager=maven&previous-version=1.7.6&new-version=1.9.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index 7d6092743bf4d..bb5636b745490 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -120,7 +120,7 @@ org.apache.orc orc-core - 1.7.6 + 1.9.2 test From 4b3de819708d4de29247676bacb78f3e7a749a6f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 16 Jan 2024 15:24:17 +0100 Subject: [PATCH 32/92] GH-39562: [C++][Parquet] Fix crash in test_parquet_dataset_lazy_filtering (#39632) ### Rationale for this change `ParquetFileFragment` stores a `SchemaManifest` that has a raw pointer to a `SchemaDescriptor`. The `SchemaDescriptor` is originally provided by a `FileMetadata` instance but, in some cases, the `FileMetadata` instance can be destroyed while the `ParquetFileFragment` is still in use. This can typically lead to bugs or crashes. ### What changes are included in this PR? Ensure that `ParquetFileFragment` keeps an owning pointer to the `FileMetadata` instance that provides its `SchemaManifest`'s schema descriptor. ### Are these changes tested? An assertion is added that would fail deterministically in the Python test suite. ### Are there any user-facing changes? No. * Closes: #39562 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/dataset/file_parquet.cc | 14 +++++++++++--- cpp/src/arrow/dataset/file_parquet.h | 5 ++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/dataset/file_parquet.cc b/cpp/src/arrow/dataset/file_parquet.cc index 0ce08502921f3..140917a2e6341 100644 --- a/cpp/src/arrow/dataset/file_parquet.cc +++ b/cpp/src/arrow/dataset/file_parquet.cc @@ -813,11 +813,17 @@ Status ParquetFileFragment::EnsureCompleteMetadata(parquet::arrow::FileReader* r Status ParquetFileFragment::SetMetadata( std::shared_ptr metadata, - std::shared_ptr manifest) { + std::shared_ptr manifest, + std::shared_ptr original_metadata) { DCHECK(row_groups_.has_value()); metadata_ = std::move(metadata); manifest_ = std::move(manifest); + original_metadata_ = original_metadata ? std::move(original_metadata) : metadata_; + // The SchemaDescriptor needs to be owned by a FileMetaData instance, + // because SchemaManifest only stores a raw pointer (GH-39562). + DCHECK_EQ(manifest_->descr, original_metadata_->schema()) + << "SchemaDescriptor should be owned by the original FileMetaData"; statistics_expressions_.resize(row_groups_->size(), compute::literal(true)); statistics_expressions_complete_.resize(manifest_->descr->num_columns(), false); @@ -846,7 +852,8 @@ Result ParquetFileFragment::SplitByRowGroup( parquet_format_.MakeFragment(source_, partition_expression(), physical_schema_, {row_group})); - RETURN_NOT_OK(fragment->SetMetadata(metadata_, manifest_)); + RETURN_NOT_OK(fragment->SetMetadata(metadata_, manifest_, + /*original_metadata=*/original_metadata_)); fragments[i++] = std::move(fragment); } @@ -1106,7 +1113,8 @@ ParquetDatasetFactory::CollectParquetFragments(const Partitioning& partitioning) format_->MakeFragment({path, filesystem_}, std::move(partition_expression), physical_schema_, std::move(row_groups))); - RETURN_NOT_OK(fragment->SetMetadata(metadata_subset, manifest_)); + RETURN_NOT_OK(fragment->SetMetadata(metadata_subset, manifest_, + /*original_metadata=*/metadata_)); fragments[i++] = std::move(fragment); } diff --git a/cpp/src/arrow/dataset/file_parquet.h b/cpp/src/arrow/dataset/file_parquet.h index 1e81a34fb3cf0..5141f36385e3f 100644 --- a/cpp/src/arrow/dataset/file_parquet.h +++ b/cpp/src/arrow/dataset/file_parquet.h @@ -188,7 +188,8 @@ class ARROW_DS_EXPORT ParquetFileFragment : public FileFragment { std::optional> row_groups); Status SetMetadata(std::shared_ptr metadata, - std::shared_ptr manifest); + std::shared_ptr manifest, + std::shared_ptr original_metadata = {}); // Overridden to opportunistically set metadata since a reader must be opened anyway. Result> ReadPhysicalSchemaImpl() override { @@ -219,6 +220,8 @@ class ARROW_DS_EXPORT ParquetFileFragment : public FileFragment { std::vector statistics_expressions_complete_; std::shared_ptr metadata_; std::shared_ptr manifest_; + // The FileMetaData that owns the SchemaDescriptor pointed by SchemaManifest. + std::shared_ptr original_metadata_; friend class ParquetFileFormat; friend class ParquetDatasetFactory; From 980e7d7ca2b4ea98b029d40cfe44588a07c85b4f Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Tue, 16 Jan 2024 17:04:15 +0100 Subject: [PATCH 33/92] GH-39628: [C++] Use -j1 for cmake >= 3.28 (#39629) ### Rationale for this change Prevent 'bad file descriptor' issue. ### What changes are included in this PR? Use -j1 for make on CMake >= 3.28 ### Are these changes tested? Crossbow * Closes: #39628 Authored-by: Jacob Wujciak-Jens Signed-off-by: Jacob Wujciak-Jens --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 6bb9c0f6af2ca..1f5cd3a2b4d5d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1005,8 +1005,13 @@ if("${MAKE}" STREQUAL "") endif() endif() -# Args for external projects using make. -set(MAKE_BUILD_ARGS "-j${NPROC}") +# Args for external projects using make +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.28") + # Prevent 'bad file descriptor' error see #39517 #39628 + set(MAKE_BUILD_ARGS "-j1") +else() + set(MAKE_BUILD_ARGS "-j${NPROC}") +endif() include(FetchContent) set(FC_DECLARE_COMMON_OPTIONS) @@ -2634,7 +2639,7 @@ macro(build_bzip2) BUILD_IN_SOURCE 1 BUILD_COMMAND ${MAKE} libbz2.a ${MAKE_BUILD_ARGS} ${BZIP2_EXTRA_ARGS} - INSTALL_COMMAND ${MAKE} install PREFIX=${BZIP2_PREFIX} + INSTALL_COMMAND ${MAKE} install -j1 PREFIX=${BZIP2_PREFIX} ${BZIP2_EXTRA_ARGS} INSTALL_DIR ${BZIP2_PREFIX} URL ${ARROW_BZIP2_SOURCE_URL} From cd3321b28b0c9703e5d7105d6146c1270bbadd7f Mon Sep 17 00:00:00 2001 From: "Rossi(Ruoxi) Sun" Date: Wed, 17 Jan 2024 01:14:03 +0800 Subject: [PATCH 34/92] GH-39577: [C++] Fix tail-word access cross buffer boundary in `CompareBinaryColumnToRow` (#39606) ### Rationale for this change Default buffer alignment (64b) doesn't guarantee the safety of tail-word access in `KeyCompare::CompareBinaryColumnToRow`. Comment https://github.com/apache/arrow/issues/39577#issuecomment-1889090279 is a concrete example. ### What changes are included in this PR? Make `KeyCompare::CompareBinaryColumnToRow` tail-word safe. ### Are these changes tested? UT included. ### Are there any user-facing changes? No. * Closes: #39577 Authored-by: zanmato1984 Signed-off-by: Antoine Pitrou --- cpp/src/arrow/compute/CMakeLists.txt | 3 +- cpp/src/arrow/compute/row/compare_internal.cc | 11 +- cpp/src/arrow/compute/row/compare_test.cc | 110 ++++++++++++++++++ 3 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 cpp/src/arrow/compute/row/compare_test.cc diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt index 1134e0a98ae45..e14d78ff6e5ca 100644 --- a/cpp/src/arrow/compute/CMakeLists.txt +++ b/cpp/src/arrow/compute/CMakeLists.txt @@ -89,7 +89,8 @@ add_arrow_test(internals_test kernel_test.cc light_array_test.cc registry_test.cc - key_hash_test.cc) + key_hash_test.cc + row/compare_test.cc) add_arrow_compute_test(expression_test SOURCES expression_test.cc) diff --git a/cpp/src/arrow/compute/row/compare_internal.cc b/cpp/src/arrow/compute/row/compare_internal.cc index 7c402e7a2384d..078a8287c71c0 100644 --- a/cpp/src/arrow/compute/row/compare_internal.cc +++ b/cpp/src/arrow/compute/row/compare_internal.cc @@ -208,8 +208,7 @@ void KeyCompare::CompareBinaryColumnToRow(uint32_t offset_within_row, // Non-zero length guarantees no underflow int32_t num_loops_less_one = static_cast(bit_util::CeilDiv(length, 8)) - 1; - - uint64_t tail_mask = ~0ULL >> (64 - 8 * (length - num_loops_less_one * 8)); + int32_t num_tail_bytes = length - num_loops_less_one * 8; const uint64_t* key_left_ptr = reinterpret_cast(left_base + irow_left * length); @@ -224,9 +223,11 @@ void KeyCompare::CompareBinaryColumnToRow(uint32_t offset_within_row, uint64_t key_right = key_right_ptr[i]; result_or |= key_left ^ key_right; } - uint64_t key_left = util::SafeLoad(key_left_ptr + i); - uint64_t key_right = key_right_ptr[i]; - result_or |= tail_mask & (key_left ^ key_right); + uint64_t key_left = 0; + memcpy(&key_left, key_left_ptr + i, num_tail_bytes); + uint64_t key_right = 0; + memcpy(&key_right, key_right_ptr + i, num_tail_bytes); + result_or |= key_left ^ key_right; return result_or == 0 ? 0xff : 0; }); } diff --git a/cpp/src/arrow/compute/row/compare_test.cc b/cpp/src/arrow/compute/row/compare_test.cc new file mode 100644 index 0000000000000..1d8562cd56d3c --- /dev/null +++ b/cpp/src/arrow/compute/row/compare_test.cc @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "arrow/compute/row/compare_internal.h" +#include "arrow/testing/gtest_util.h" + +namespace arrow { +namespace compute { + +using arrow::bit_util::BytesForBits; +using arrow::internal::CpuInfo; +using arrow::util::MiniBatch; +using arrow::util::TempVectorStack; + +// Specialized case for GH-39577. +TEST(KeyCompare, CompareColumnsToRowsCuriousFSB) { + int fsb_length = 9; + MemoryPool* pool = default_memory_pool(); + TempVectorStack stack; + ASSERT_OK(stack.Init(pool, 8 * MiniBatch::kMiniBatchLength * sizeof(uint64_t))); + + int num_rows = 7; + auto column_right = ArrayFromJSON(fixed_size_binary(fsb_length), R"([ + "000000000", + "111111111", + "222222222", + "333333333", + "444444444", + "555555555", + "666666666"])"); + ExecBatch batch_right({column_right}, num_rows); + + std::vector column_metadatas_right; + ASSERT_OK(ColumnMetadatasFromExecBatch(batch_right, &column_metadatas_right)); + + RowTableMetadata table_metadata_right; + table_metadata_right.FromColumnMetadataVector(column_metadatas_right, sizeof(uint64_t), + sizeof(uint64_t)); + + std::vector column_arrays_right; + ASSERT_OK(ColumnArraysFromExecBatch(batch_right, &column_arrays_right)); + + RowTableImpl row_table; + ASSERT_OK(row_table.Init(pool, table_metadata_right)); + + RowTableEncoder row_encoder; + row_encoder.Init(column_metadatas_right, sizeof(uint64_t), sizeof(uint64_t)); + row_encoder.PrepareEncodeSelected(0, num_rows, column_arrays_right); + + std::vector row_ids_right(num_rows); + std::iota(row_ids_right.begin(), row_ids_right.end(), 0); + ASSERT_OK(row_encoder.EncodeSelected(&row_table, num_rows, row_ids_right.data())); + + auto column_left = ArrayFromJSON(fixed_size_binary(fsb_length), R"([ + "000000000", + "111111111", + "222222222", + "333333333", + "444444444", + "555555555", + "777777777"])"); + ExecBatch batch_left({column_left}, num_rows); + std::vector column_arrays_left; + ASSERT_OK(ColumnArraysFromExecBatch(batch_left, &column_arrays_left)); + + std::vector row_ids_left(num_rows); + std::iota(row_ids_left.begin(), row_ids_left.end(), 0); + + LightContext ctx{CpuInfo::GetInstance()->hardware_flags(), &stack}; + + { + uint32_t num_rows_no_match; + std::vector row_ids_out(num_rows); + KeyCompare::CompareColumnsToRows(num_rows, NULLPTR, row_ids_left.data(), &ctx, + &num_rows_no_match, row_ids_out.data(), + column_arrays_left, row_table, true, NULLPTR); + ASSERT_EQ(num_rows_no_match, 1); + ASSERT_EQ(row_ids_out[0], 6); + } + + { + std::vector match_bitvector(BytesForBits(num_rows)); + KeyCompare::CompareColumnsToRows(num_rows, NULLPTR, row_ids_left.data(), &ctx, + NULLPTR, NULLPTR, column_arrays_left, row_table, + true, match_bitvector.data()); + for (int i = 0; i < num_rows; ++i) { + SCOPED_TRACE(i); + ASSERT_EQ(arrow::bit_util::GetBit(match_bitvector.data(), i), i != 6); + } + } +} + +} // namespace compute +} // namespace arrow From f55c0d75ce3ca2e52aa7c410b1a71642d40784d9 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 17 Jan 2024 05:09:53 +0100 Subject: [PATCH 35/92] GH-39626: [Docs][R] Update NEWS.md for 15.0.0 (#39627) ### What changes are included in this PR? Update NEWS.md with things not included in 14.0.2. * Closes: #39626 Lead-authored-by: Jacob Wujciak-Jens Co-authored-by: Bryce Mecum Signed-off-by: Jacob Wujciak-Jens --- r/NEWS.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/r/NEWS.md b/r/NEWS.md index 1744e6e96e936..9badf4700a36e 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -19,6 +19,35 @@ # arrow 14.0.2.9000 +## New features + +* Bindings for `base::prod` have been added so you can now use it in your dplyr + pipelines (i.e., `tbl |> summarize(prod(col))`) without having to pull the + data into R (@m-muecke, #38601). +* Calling `dimnames` or `colnames` on `Dataset` objects now returns a useful + result rather than just `NULL` (#38377). +* The `code()` method on Schema objects now takes an optional `namespace` + argument which, when `TRUE`, prefixes names with `arrow::` which makes + the output more portable (@orgadish, #38144). + +## Minor improvements and fixes + +* Don't download cmake when TEST_OFFLINE_BUILD=true and update `SystemRequirements` (#39602). +* Fallback to source build gracefully if binary download fails (#39587). +* An error is now thrown instead of warning and pulling the data into R when any + of `sub`, `gsub`, `stringr::str_replace`, `stringr::str_replace_all` are + passed a length > 1 vector of values in `pattern` (@abfleishman, #39219). +* Missing documentation was added to `?open_dataset` documenting how to use the + ND-JSON support added in arrow 13.0.0 (@Divyansh200102, #38258). +* To make debugging problems easier when using arrow with AWS S3 + (e.g., `s3_bucket`, `S3FileSystem`), the debug log level for S3 can be set + with the `AWS_S3_LOG_LEVEL` environment variable. + See `?S3FileSystem` for more information. (#38267) +* Using arrow with duckdb (i.e., `to_duckdb()`) no longer results in warnings + when quitting your R session. (#38495) +* A large number of minor spelling mistakes were fixed (@jsoref, #38929, #38257) +* The developer documentation has been updated to match changes made in recent releases (#38220) + # arrow 14.0.2 ## Minor improvements and fixes From 6eeee3b769f3ad6c724305b4182526307ab025d5 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 17 Jan 2024 11:12:41 +0100 Subject: [PATCH 36/92] GH-36412: [Python][CI] Fix extra deprecation warnings in the pandas nightly build (#39609) Fixes left deprecation warnings coming from the pandas development version, by updating our test code to avoid the deprecated patterns. * Closes: #36412 Lead-authored-by: AlenkaF Co-authored-by: Alenka Frim Co-authored-by: Joris Van den Bossche Signed-off-by: Joris Van den Bossche --- python/pyarrow/pandas_compat.py | 15 +------ python/pyarrow/tests/parquet/test_datetime.py | 4 +- python/pyarrow/tests/test_compute.py | 6 +-- python/pyarrow/tests/test_dataset.py | 6 ++- python/pyarrow/tests/test_pandas.py | 42 +++++++++++-------- 5 files changed, 35 insertions(+), 38 deletions(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 39dee85492400..61e6318e29c24 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -967,20 +967,9 @@ def _extract_index_level(table, result_table, field_name, # The serialized index column was removed by the user return result_table, None, None - pd = _pandas_api.pd - col = table.column(i) - values = col.to_pandas(types_mapper=types_mapper).values - - if hasattr(values, 'flags') and not values.flags.writeable: - # ARROW-1054: in pandas 0.19.2, factorize will reject - # non-writeable arrays when calling MultiIndex.from_arrays - values = values.copy() - - if isinstance(col.type, pa.lib.TimestampType) and col.type.tz is not None: - index_level = make_tz_aware(pd.Series(values, copy=False), col.type.tz) - else: - index_level = pd.Series(values, dtype=values.dtype, copy=False) + index_level = col.to_pandas(types_mapper=types_mapper) + index_level.name = None result_table = result_table.remove_column( result_table.schema.get_field_index(field_name) ) diff --git a/python/pyarrow/tests/parquet/test_datetime.py b/python/pyarrow/tests/parquet/test_datetime.py index 6a9cbd4f73d4f..0896eb37e6473 100644 --- a/python/pyarrow/tests/parquet/test_datetime.py +++ b/python/pyarrow/tests/parquet/test_datetime.py @@ -116,7 +116,7 @@ def test_coerce_timestamps(tempdir): df_expected = df.copy() for i, x in enumerate(df_expected['datetime64']): if isinstance(x, np.ndarray): - df_expected['datetime64'][i] = x.astype('M8[us]') + df_expected.loc[i, 'datetime64'] = x.astype('M8[us]') tm.assert_frame_equal(df_expected, df_read) @@ -429,7 +429,7 @@ def test_noncoerced_nanoseconds_written_without_exception(tempdir): # nanosecond timestamps by default n = 9 df = pd.DataFrame({'x': range(n)}, - index=pd.date_range('2017-01-01', freq='1n', periods=n)) + index=pd.date_range('2017-01-01', freq='ns', periods=n)) tb = pa.Table.from_pandas(df) filename = tempdir / 'written.parquet' diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index d1eb605c71881..34d4da580f526 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -2360,10 +2360,10 @@ def _check_temporal_rounding(ts, values, unit): unit_shorthand = { "nanosecond": "ns", "microsecond": "us", - "millisecond": "L", + "millisecond": "ms", "second": "s", "minute": "min", - "hour": "H", + "hour": "h", "day": "D" } greater_unit = { @@ -2371,7 +2371,7 @@ def _check_temporal_rounding(ts, values, unit): "microsecond": "ms", "millisecond": "s", "second": "min", - "minute": "H", + "minute": "h", "hour": "d", } ta = pa.array(ts) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index ae2146c0bdaee..d473299f20320 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -178,12 +178,14 @@ def multisourcefs(request): # simply split the dataframe into four chunks to construct a data source # from each chunk into its own directory - df_a, df_b, df_c, df_d = np.array_split(df, 4) + n = len(df) + df_a, df_b, df_c, df_d = [df.iloc[i:i+n//4] for i in range(0, n, n//4)] # create a directory containing a flat sequence of parquet files without # any partitioning involved mockfs.create_dir('plain') - for i, chunk in enumerate(np.array_split(df_a, 10)): + n = len(df_a) + for i, chunk in enumerate([df_a.iloc[i:i+n//10] for i in range(0, n, n//10)]): path = 'plain/chunk-{}.parquet'.format(i) with mockfs.open_output_stream(path) as out: pq.write_table(_table_from_pandas(chunk), out) diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index d15ee82d5dbf1..8106219057efe 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -113,6 +113,10 @@ def _check_pandas_roundtrip(df, expected=None, use_threads=False, if expected is None: expected = df + for col in expected.columns: + if expected[col].dtype == 'object': + expected[col] = expected[col].replace({np.nan: None}) + with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "elementwise comparison failed", DeprecationWarning) @@ -152,6 +156,9 @@ def _check_array_roundtrip(values, expected=None, mask=None, expected = pd.Series(values).copy() expected[mask.copy()] = None + if expected.dtype == 'object': + expected = expected.replace({np.nan: None}) + tm.assert_series_equal(pd.Series(result), expected, check_names=False) @@ -478,7 +485,7 @@ def test_mixed_column_names(self): preserve_index=True) def test_binary_column_name(self): - if Version("2.0.0") <= Version(pd.__version__) < Version("2.3.0"): + if Version("2.0.0") <= Version(pd.__version__) < Version("3.0.0"): # TODO: regression in pandas, hopefully fixed in next version # https://issues.apache.org/jira/browse/ARROW-18394 # https://github.com/pandas-dev/pandas/issues/50127 @@ -3108,7 +3115,7 @@ def _fully_loaded_dataframe_example(): @pytest.mark.parametrize('columns', ([b'foo'], ['foo'])) def test_roundtrip_with_bytes_unicode(columns): - if Version("2.0.0") <= Version(pd.__version__) < Version("2.3.0"): + if Version("2.0.0") <= Version(pd.__version__) < Version("3.0.0"): # TODO: regression in pandas, hopefully fixed in next version # https://issues.apache.org/jira/browse/ARROW-18394 # https://github.com/pandas-dev/pandas/issues/50127 @@ -3491,7 +3498,7 @@ def test_table_from_pandas_schema_field_order_metadata(): # ensure that a different field order in specified schema doesn't # mangle metadata df = pd.DataFrame({ - "datetime": pd.date_range("2020-01-01T00:00:00Z", freq="H", periods=2), + "datetime": pd.date_range("2020-01-01T00:00:00Z", freq="h", periods=2), "float": np.random.randn(2) }) @@ -4181,8 +4188,6 @@ def _Int64Dtype__from_arrow__(self, array): def test_convert_to_extension_array(monkeypatch): - import pandas.core.internals as _int - # table converted from dataframe with extension types (so pandas_metadata # has this information) df = pd.DataFrame( @@ -4193,16 +4198,15 @@ def test_convert_to_extension_array(monkeypatch): # Int64Dtype is recognized -> convert to extension block by default # for a proper roundtrip result = table.to_pandas() - assert not isinstance(_get_mgr(result).blocks[0], _int.ExtensionBlock) assert _get_mgr(result).blocks[0].values.dtype == np.dtype("int64") - assert isinstance(_get_mgr(result).blocks[1], _int.ExtensionBlock) + assert _get_mgr(result).blocks[1].values.dtype == pd.Int64Dtype() tm.assert_frame_equal(result, df) # test with missing values df2 = pd.DataFrame({'a': pd.array([1, 2, None], dtype='Int64')}) table2 = pa.table(df2) result = table2.to_pandas() - assert isinstance(_get_mgr(result).blocks[0], _int.ExtensionBlock) + assert _get_mgr(result).blocks[0].values.dtype == pd.Int64Dtype() tm.assert_frame_equal(result, df2) # monkeypatch pandas Int64Dtype to *not* have the protocol method @@ -4215,7 +4219,7 @@ def test_convert_to_extension_array(monkeypatch): # Int64Dtype has no __from_arrow__ -> use normal conversion result = table.to_pandas() assert len(_get_mgr(result).blocks) == 1 - assert not isinstance(_get_mgr(result).blocks[0], _int.ExtensionBlock) + assert _get_mgr(result).blocks[0].values.dtype == np.dtype("int64") class MyCustomIntegerType(pa.ExtensionType): @@ -4233,8 +4237,6 @@ def to_pandas_dtype(self): def test_conversion_extensiontype_to_extensionarray(monkeypatch): # converting extension type to linked pandas ExtensionDtype/Array - import pandas.core.internals as _int - storage = pa.array([1, 2, 3, 4], pa.int64()) arr = pa.ExtensionArray.from_storage(MyCustomIntegerType(), storage) table = pa.table({'a': arr}) @@ -4242,12 +4244,12 @@ def test_conversion_extensiontype_to_extensionarray(monkeypatch): # extension type points to Int64Dtype, which knows how to create a # pandas ExtensionArray result = arr.to_pandas() - assert isinstance(_get_mgr(result).blocks[0], _int.ExtensionBlock) + assert _get_mgr(result).blocks[0].values.dtype == pd.Int64Dtype() expected = pd.Series([1, 2, 3, 4], dtype='Int64') tm.assert_series_equal(result, expected) result = table.to_pandas() - assert isinstance(_get_mgr(result).blocks[0], _int.ExtensionBlock) + assert _get_mgr(result).blocks[0].values.dtype == pd.Int64Dtype() expected = pd.DataFrame({'a': pd.array([1, 2, 3, 4], dtype='Int64')}) tm.assert_frame_equal(result, expected) @@ -4261,7 +4263,7 @@ def test_conversion_extensiontype_to_extensionarray(monkeypatch): pd.core.arrays.integer.NumericDtype, "__from_arrow__") result = arr.to_pandas() - assert not isinstance(_get_mgr(result).blocks[0], _int.ExtensionBlock) + assert _get_mgr(result).blocks[0].values.dtype == np.dtype("int64") expected = pd.Series([1, 2, 3, 4]) tm.assert_series_equal(result, expected) @@ -4312,10 +4314,14 @@ def test_array_to_pandas(): def test_roundtrip_empty_table_with_extension_dtype_index(): df = pd.DataFrame(index=pd.interval_range(start=0, end=3)) table = pa.table(df) - table.to_pandas().index == pd.Index([{'left': 0, 'right': 1}, - {'left': 1, 'right': 2}, - {'left': 2, 'right': 3}], - dtype='object') + if Version(pd.__version__) > Version("1.0"): + tm.assert_index_equal(table.to_pandas().index, df.index) + else: + tm.assert_index_equal(table.to_pandas().index, + pd.Index([{'left': 0, 'right': 1}, + {'left': 1, 'right': 2}, + {'left': 2, 'right': 3}], + dtype='object')) @pytest.mark.parametrize("index", ["a", ["a", "b"]]) From 96645ebc5037b6b4eab127c274f4871bbef99d77 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 17 Jan 2024 11:26:37 +0100 Subject: [PATCH 37/92] GH-39599: [Python] Avoid leaking references to Numpy dtypes (#39636) ### Rationale for this change `PyArray_DescrFromScalar` returns a new reference, so we should be careful to decref it when we don't use it anymore. ### Are these changes tested? No. ### Are there any user-facing changes? No. * Closes: #39599 Authored-by: Antoine Pitrou Signed-off-by: Joris Van den Bossche --- python/pyarrow/array.pxi | 3 +- python/pyarrow/includes/libarrow_python.pxd | 2 +- python/pyarrow/src/arrow/python/inference.cc | 5 +- .../pyarrow/src/arrow/python/numpy_convert.cc | 77 ++++++++----------- .../pyarrow/src/arrow/python/numpy_convert.h | 6 +- .../src/arrow/python/numpy_to_arrow.cc | 11 ++- .../src/arrow/python/python_to_arrow.cc | 6 +- python/pyarrow/types.pxi | 6 +- 8 files changed, 48 insertions(+), 68 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 5c2d22aef1895..1416f5f4346d9 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -66,8 +66,7 @@ cdef shared_ptr[CDataType] _ndarray_to_type(object values, dtype = values.dtype if type is None and dtype != object: - with nogil: - check_status(NumPyDtypeToArrow(dtype, &c_type)) + c_type = GetResultValue(NumPyDtypeToArrow(dtype)) if type is not None: c_type = type.sp_type diff --git a/python/pyarrow/includes/libarrow_python.pxd b/python/pyarrow/includes/libarrow_python.pxd index e3179062a1e52..906f0b7d28e59 100644 --- a/python/pyarrow/includes/libarrow_python.pxd +++ b/python/pyarrow/includes/libarrow_python.pxd @@ -73,7 +73,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: object obj, object mask, const PyConversionOptions& options, CMemoryPool* pool) - CStatus NumPyDtypeToArrow(object dtype, shared_ptr[CDataType]* type) + CResult[shared_ptr[CDataType]] NumPyDtypeToArrow(object dtype) CStatus NdarrayToArrow(CMemoryPool* pool, object ao, object mo, c_bool from_pandas, diff --git a/python/pyarrow/src/arrow/python/inference.cc b/python/pyarrow/src/arrow/python/inference.cc index 9537aec574470..10116f9afad69 100644 --- a/python/pyarrow/src/arrow/python/inference.cc +++ b/python/pyarrow/src/arrow/python/inference.cc @@ -468,10 +468,7 @@ class TypeInferrer { if (numpy_dtype_count_ > 0) { // All NumPy scalars and Nones/nulls if (numpy_dtype_count_ + none_count_ == total_count_) { - std::shared_ptr type; - RETURN_NOT_OK(NumPyDtypeToArrow(numpy_unifier_.current_dtype(), &type)); - *out = type; - return Status::OK(); + return NumPyDtypeToArrow(numpy_unifier_.current_dtype()).Value(out); } // The "bad path": data contains a mix of NumPy scalars and diff --git a/python/pyarrow/src/arrow/python/numpy_convert.cc b/python/pyarrow/src/arrow/python/numpy_convert.cc index 49706807644d2..dfee88c092e65 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.cc +++ b/python/pyarrow/src/arrow/python/numpy_convert.cc @@ -59,12 +59,11 @@ NumPyBuffer::~NumPyBuffer() { #define TO_ARROW_TYPE_CASE(NPY_NAME, FACTORY) \ case NPY_##NPY_NAME: \ - *out = FACTORY(); \ - break; + return FACTORY(); namespace { -Status GetTensorType(PyObject* dtype, std::shared_ptr* out) { +Result> GetTensorType(PyObject* dtype) { if (!PyObject_TypeCheck(dtype, &PyArrayDescr_Type)) { return Status::TypeError("Did not pass numpy.dtype object"); } @@ -84,11 +83,8 @@ Status GetTensorType(PyObject* dtype, std::shared_ptr* out) { TO_ARROW_TYPE_CASE(FLOAT16, float16); TO_ARROW_TYPE_CASE(FLOAT32, float32); TO_ARROW_TYPE_CASE(FLOAT64, float64); - default: { - return Status::NotImplemented("Unsupported numpy type ", descr->type_num); - } } - return Status::OK(); + return Status::NotImplemented("Unsupported numpy type ", descr->type_num); } Status GetNumPyType(const DataType& type, int* type_num) { @@ -120,15 +116,21 @@ Status GetNumPyType(const DataType& type, int* type_num) { } // namespace -Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr* out) { +Result> NumPyScalarToArrowDataType(PyObject* scalar) { + PyArray_Descr* descr = PyArray_DescrFromScalar(scalar); + OwnedRef descr_ref(reinterpret_cast(descr)); + return NumPyDtypeToArrow(descr); +} + +Result> NumPyDtypeToArrow(PyObject* dtype) { if (!PyObject_TypeCheck(dtype, &PyArrayDescr_Type)) { return Status::TypeError("Did not pass numpy.dtype object"); } PyArray_Descr* descr = reinterpret_cast(dtype); - return NumPyDtypeToArrow(descr, out); + return NumPyDtypeToArrow(descr); } -Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr* out) { +Result> NumPyDtypeToArrow(PyArray_Descr* descr) { int type_num = fix_numpy_type_num(descr->type_num); switch (type_num) { @@ -151,20 +153,15 @@ Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr* out) { reinterpret_cast(descr->c_metadata); switch (date_dtype->meta.base) { case NPY_FR_s: - *out = timestamp(TimeUnit::SECOND); - break; + return timestamp(TimeUnit::SECOND); case NPY_FR_ms: - *out = timestamp(TimeUnit::MILLI); - break; + return timestamp(TimeUnit::MILLI); case NPY_FR_us: - *out = timestamp(TimeUnit::MICRO); - break; + return timestamp(TimeUnit::MICRO); case NPY_FR_ns: - *out = timestamp(TimeUnit::NANO); - break; + return timestamp(TimeUnit::NANO); case NPY_FR_D: - *out = date32(); - break; + return date32(); case NPY_FR_GENERIC: return Status::NotImplemented("Unbound or generic datetime64 time unit"); default: @@ -176,29 +173,22 @@ Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr* out) { reinterpret_cast(descr->c_metadata); switch (timedelta_dtype->meta.base) { case NPY_FR_s: - *out = duration(TimeUnit::SECOND); - break; + return duration(TimeUnit::SECOND); case NPY_FR_ms: - *out = duration(TimeUnit::MILLI); - break; + return duration(TimeUnit::MILLI); case NPY_FR_us: - *out = duration(TimeUnit::MICRO); - break; + return duration(TimeUnit::MICRO); case NPY_FR_ns: - *out = duration(TimeUnit::NANO); - break; + return duration(TimeUnit::NANO); case NPY_FR_GENERIC: return Status::NotImplemented("Unbound or generic timedelta64 time unit"); default: return Status::NotImplemented("Unsupported timedelta64 time unit"); } } break; - default: { - return Status::NotImplemented("Unsupported numpy type ", descr->type_num); - } } - return Status::OK(); + return Status::NotImplemented("Unsupported numpy type ", descr->type_num); } #undef TO_ARROW_TYPE_CASE @@ -230,9 +220,8 @@ Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, strides[i] = array_strides[i]; } - std::shared_ptr type; - RETURN_NOT_OK( - GetTensorType(reinterpret_cast(PyArray_DESCR(ndarray)), &type)); + ARROW_ASSIGN_OR_RAISE( + auto type, GetTensorType(reinterpret_cast(PyArray_DESCR(ndarray)))); *out = std::make_shared(type, data, shape, strides, dim_names); return Status::OK(); } @@ -435,9 +424,9 @@ Status NdarraysToSparseCOOTensor(MemoryPool* pool, PyObject* data_ao, PyObject* PyArrayObject* ndarray_data = reinterpret_cast(data_ao); std::shared_ptr data = std::make_shared(data_ao); - std::shared_ptr type_data; - RETURN_NOT_OK(GetTensorType(reinterpret_cast(PyArray_DESCR(ndarray_data)), - &type_data)); + ARROW_ASSIGN_OR_RAISE( + auto type_data, + GetTensorType(reinterpret_cast(PyArray_DESCR(ndarray_data)))); std::shared_ptr coords; RETURN_NOT_OK(NdarrayToTensor(pool, coords_ao, {}, &coords)); @@ -462,9 +451,9 @@ Status NdarraysToSparseCSXMatrix(MemoryPool* pool, PyObject* data_ao, PyObject* PyArrayObject* ndarray_data = reinterpret_cast(data_ao); std::shared_ptr data = std::make_shared(data_ao); - std::shared_ptr type_data; - RETURN_NOT_OK(GetTensorType(reinterpret_cast(PyArray_DESCR(ndarray_data)), - &type_data)); + ARROW_ASSIGN_OR_RAISE( + auto type_data, + GetTensorType(reinterpret_cast(PyArray_DESCR(ndarray_data)))); std::shared_ptr indptr, indices; RETURN_NOT_OK(NdarrayToTensor(pool, indptr_ao, {}, &indptr)); @@ -491,9 +480,9 @@ Status NdarraysToSparseCSFTensor(MemoryPool* pool, PyObject* data_ao, PyObject* const int ndim = static_cast(shape.size()); PyArrayObject* ndarray_data = reinterpret_cast(data_ao); std::shared_ptr data = std::make_shared(data_ao); - std::shared_ptr type_data; - RETURN_NOT_OK(GetTensorType(reinterpret_cast(PyArray_DESCR(ndarray_data)), - &type_data)); + ARROW_ASSIGN_OR_RAISE( + auto type_data, + GetTensorType(reinterpret_cast(PyArray_DESCR(ndarray_data)))); std::vector> indptr(ndim - 1); std::vector> indices(ndim); diff --git a/python/pyarrow/src/arrow/python/numpy_convert.h b/python/pyarrow/src/arrow/python/numpy_convert.h index 10451077a221d..2d1086e135528 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.h +++ b/python/pyarrow/src/arrow/python/numpy_convert.h @@ -49,9 +49,11 @@ class ARROW_PYTHON_EXPORT NumPyBuffer : public Buffer { }; ARROW_PYTHON_EXPORT -Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr* out); +Result> NumPyDtypeToArrow(PyObject* dtype); ARROW_PYTHON_EXPORT -Status NumPyDtypeToArrow(PyArray_Descr* descr, std::shared_ptr* out); +Result> NumPyDtypeToArrow(PyArray_Descr* descr); +ARROW_PYTHON_EXPORT +Result> NumPyScalarToArrowDataType(PyObject* scalar); ARROW_PYTHON_EXPORT Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, const std::vector& dim_names, diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 2727ce32f4494..8903df31be826 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -462,8 +462,7 @@ template inline Status NumPyConverter::ConvertData(std::shared_ptr* data) { RETURN_NOT_OK(PrepareInputData(data)); - std::shared_ptr input_type; - RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); + ARROW_ASSIGN_OR_RAISE(auto input_type, NumPyDtypeToArrow(dtype_)); if (!input_type->Equals(*type_)) { RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count_, type_, @@ -490,7 +489,7 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d Status s = StaticCastBuffer(**data, length_, pool_, data); RETURN_NOT_OK(s); } else { - RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); + ARROW_ASSIGN_OR_RAISE(input_type, NumPyDtypeToArrow(dtype_)); if (!input_type->Equals(*type_)) { // The null bitmap was already computed in VisitNative() RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count_, @@ -498,7 +497,7 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d } } } else { - RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); + ARROW_ASSIGN_OR_RAISE(input_type, NumPyDtypeToArrow(dtype_)); if (!input_type->Equals(*type_)) { RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count_, type_, cast_options_, pool_, data)); @@ -531,7 +530,7 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d } *data = std::move(result); } else { - RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); + ARROW_ASSIGN_OR_RAISE(input_type, NumPyDtypeToArrow(dtype_)); if (!input_type->Equals(*type_)) { // The null bitmap was already computed in VisitNative() RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count_, @@ -539,7 +538,7 @@ inline Status NumPyConverter::ConvertData(std::shared_ptr* d } } } else { - RETURN_NOT_OK(NumPyDtypeToArrow(reinterpret_cast(dtype_), &input_type)); + ARROW_ASSIGN_OR_RAISE(input_type, NumPyDtypeToArrow(dtype_)); if (!input_type->Equals(*type_)) { RETURN_NOT_OK(CastBuffer(input_type, *data, length_, null_bitmap_, null_count_, type_, cast_options_, pool_, data)); diff --git a/python/pyarrow/src/arrow/python/python_to_arrow.cc b/python/pyarrow/src/arrow/python/python_to_arrow.cc index 23b92598e321e..d1d94ac17a13e 100644 --- a/python/pyarrow/src/arrow/python/python_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/python_to_arrow.cc @@ -386,8 +386,7 @@ class PyValue { } } else if (PyArray_CheckAnyScalarExact(obj)) { // validate that the numpy scalar has np.datetime64 dtype - std::shared_ptr numpy_type; - RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &numpy_type)); + ARROW_ASSIGN_OR_RAISE(auto numpy_type, NumPyScalarToArrowDataType(obj)); if (!numpy_type->Equals(*type)) { return Status::NotImplemented("Expected np.datetime64 but got: ", numpy_type->ToString()); @@ -466,8 +465,7 @@ class PyValue { } } else if (PyArray_CheckAnyScalarExact(obj)) { // validate that the numpy scalar has np.datetime64 dtype - std::shared_ptr numpy_type; - RETURN_NOT_OK(NumPyDtypeToArrow(PyArray_DescrFromScalar(obj), &numpy_type)); + ARROW_ASSIGN_OR_RAISE(auto numpy_type, NumPyScalarToArrowDataType(obj)); if (!numpy_type->Equals(*type)) { return Status::NotImplemented("Expected np.timedelta64 but got: ", numpy_type->ToString()); diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 912ee39f7d712..b6dc53d633543 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -5140,12 +5140,8 @@ def from_numpy_dtype(object dtype): >>> pa.from_numpy_dtype(np.str_) DataType(string) """ - cdef shared_ptr[CDataType] c_type dtype = np.dtype(dtype) - with nogil: - check_status(NumPyDtypeToArrow(dtype, &c_type)) - - return pyarrow_wrap_data_type(c_type) + return pyarrow_wrap_data_type(GetResultValue(NumPyDtypeToArrow(dtype))) def is_boolean_value(object obj): From f15a7002794865be9ffc6fe338c2fce4f259c476 Mon Sep 17 00:00:00 2001 From: mwish Date: Wed, 17 Jan 2024 20:37:03 +0800 Subject: [PATCH 38/92] GH-39336: [C++][Parquet] Minor: Style enhancement for parquet::FileMetaData (#39337) ### Rationale for this change Enhance the style of `parquet::FileMetaData::Subset`. ### Are these changes tested? Already tested ### Are there any user-facing changes? no * Closes: #39336 Lead-authored-by: mwish Co-authored-by: mwish <1506118561@qq.com> Co-authored-by: Antoine Pitrou Signed-off-by: mwish --- cpp/src/arrow/util/bit_stream_utils.h | 2 +- cpp/src/parquet/metadata.cc | 18 ++++++++++-------- cpp/src/parquet/metadata.h | 8 +++++++- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/util/bit_stream_utils.h b/cpp/src/arrow/util/bit_stream_utils.h index 2afb2e5193697..811694e43b76c 100644 --- a/cpp/src/arrow/util/bit_stream_utils.h +++ b/cpp/src/arrow/util/bit_stream_utils.h @@ -183,7 +183,7 @@ class BitReader { /// Returns the number of bytes left in the stream, not including the current /// byte (i.e., there may be an additional fraction of a byte). - int bytes_left() { + int bytes_left() const { return max_bytes_ - (byte_offset_ + static_cast(bit_util::BytesForBits(bit_offset_))); } diff --git a/cpp/src/parquet/metadata.cc b/cpp/src/parquet/metadata.cc index d651ea5db0f18..3f101b5ae3ac6 100644 --- a/cpp/src/parquet/metadata.cc +++ b/cpp/src/parquet/metadata.cc @@ -761,7 +761,7 @@ class FileMetaData::FileMetaDataImpl { return metadata_->row_groups[i]; } - void AppendRowGroups(const std::unique_ptr& other) { + void AppendRowGroups(FileMetaDataImpl* other) { std::ostringstream diff_output; if (!schema()->Equals(*other->schema(), &diff_output)) { auto msg = "AppendRowGroups requires equal schemas.\n" + diff_output.str(); @@ -800,6 +800,7 @@ class FileMetaData::FileMetaDataImpl { metadata->schema = metadata_->schema; metadata->row_groups.resize(row_groups.size()); + int i = 0; for (int selected_index : row_groups) { metadata->num_rows += row_group(selected_index).num_rows; @@ -822,7 +823,7 @@ class FileMetaData::FileMetaDataImpl { } void set_file_decryptor(std::shared_ptr file_decryptor) { - file_decryptor_ = file_decryptor; + file_decryptor_ = std::move(file_decryptor); } private: @@ -886,13 +887,14 @@ std::shared_ptr FileMetaData::Make( const void* metadata, uint32_t* metadata_len, std::shared_ptr file_decryptor) { return std::shared_ptr(new FileMetaData( - metadata, metadata_len, default_reader_properties(), file_decryptor)); + metadata, metadata_len, default_reader_properties(), std::move(file_decryptor))); } FileMetaData::FileMetaData(const void* metadata, uint32_t* metadata_len, const ReaderProperties& properties, std::shared_ptr file_decryptor) - : impl_(new FileMetaDataImpl(metadata, metadata_len, properties, file_decryptor)) {} + : impl_(new FileMetaDataImpl(metadata, metadata_len, properties, + std::move(file_decryptor))) {} FileMetaData::FileMetaData() : impl_(new FileMetaDataImpl()) {} @@ -942,7 +944,7 @@ const std::string& FileMetaData::footer_signing_key_metadata() const { void FileMetaData::set_file_decryptor( std::shared_ptr file_decryptor) { - impl_->set_file_decryptor(file_decryptor); + impl_->set_file_decryptor(std::move(file_decryptor)); } ParquetVersion::type FileMetaData::version() const { @@ -975,7 +977,7 @@ const std::shared_ptr& FileMetaData::key_value_metadata( void FileMetaData::set_file_path(const std::string& path) { impl_->set_file_path(path); } void FileMetaData::AppendRowGroups(const FileMetaData& other) { - impl_->AppendRowGroups(other.impl_); + impl_->AppendRowGroups(other.impl_.get()); } std::shared_ptr FileMetaData::Subset( @@ -1839,7 +1841,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl { std::unique_ptr Finish( const std::shared_ptr& key_value_metadata) { int64_t total_rows = 0; - for (auto row_group : row_groups_) { + for (const auto& row_group : row_groups_) { total_rows += row_group.num_rows; } metadata_->__set_num_rows(total_rows); @@ -1858,7 +1860,7 @@ class FileMetaDataBuilder::FileMetaDataBuilderImpl { format::KeyValue kv_pair; kv_pair.__set_key(key_value_metadata_->key(i)); kv_pair.__set_value(key_value_metadata_->value(i)); - metadata_->key_value_metadata.push_back(kv_pair); + metadata_->key_value_metadata.push_back(std::move(kv_pair)); } metadata_->__isset.key_value_metadata = true; } diff --git a/cpp/src/parquet/metadata.h b/cpp/src/parquet/metadata.h index e47c45ff0492a..640b898024346 100644 --- a/cpp/src/parquet/metadata.h +++ b/cpp/src/parquet/metadata.h @@ -306,9 +306,15 @@ class PARQUET_EXPORT FileMetaData { int num_schema_elements() const; /// \brief The total number of rows. + /// + /// If the FileMetaData was obtained by calling `SubSet()`, this is the total + /// number of rows in the selected row groups. int64_t num_rows() const; /// \brief The number of row groups in the file. + /// + /// If the FileMetaData was obtained by calling `SubSet()`, this is the number + /// of selected row groups. int num_row_groups() const; /// \brief Return the RowGroupMetaData of the corresponding row group ordinal. @@ -338,7 +344,7 @@ class PARQUET_EXPORT FileMetaData { /// \brief Size of the original thrift encoded metadata footer. uint32_t size() const; - /// \brief Indicate if all of the FileMetadata's RowGroups can be decompressed. + /// \brief Indicate if all of the FileMetaData's RowGroups can be decompressed. /// /// This will return false if any of the RowGroup's page is compressed with a /// compression format which is not compiled in the current parquet library. From 1076229dbc35c9b29cc170ff3193620277c280a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Wed, 17 Jan 2024 14:35:04 +0100 Subject: [PATCH 39/92] GH-39654: [Java] Upgrade to Netty 4.1.105.Final (#39655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Netty 4.1.105.Final brings few fixes that would be good to have in Arrow, especially: - DNS error improvements - Fix exception on HTTP chunk size overflow - Save HTTP2 pseudo-header lower-case validation The release news is there: https://netty.io/news/2024/01/16/4-1-105-Final.html and the full Release Notes here: https://github.com/netty/netty/issues?q=milestone%3A4.1.105.Final+is%3Aclosed ### What changes are included in this PR? Netty 4.1.105.Final update. ### Are these changes tested? Yes tested locally. ### Are there any user-facing changes? Not directly. * Closes: #39654 Authored-by: JB Onofré Signed-off-by: David Li --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 147f08fe57030..b2513d586268b 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -33,7 +33,7 @@ 5.10.1 2.0.11 33.0.0-jre - 4.1.104.Final + 4.1.105.Final 1.60.0 3.23.1 2.16.0 From 3acc2eab739bfcc89bd39c3242dad154a980acb7 Mon Sep 17 00:00:00 2001 From: mwish Date: Wed, 17 Jan 2024 21:36:53 +0800 Subject: [PATCH 40/92] GH-39390: [C++] Thirdparty: bump default zlib to 1.3 (#39391) ### Rationale for this change This patch bump default zlib to 1.3 (currently we're default using 1.2.13 as default). 1.3 includes some bugfixes. ### What changes are included in this PR? Update vendor in arrow ### Are these changes tested? Maybe we already has tests? ### Are there any user-facing changes? no * Closes: #39390 Authored-by: mwish Signed-off-by: mwish --- cpp/thirdparty/versions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index e9df0c8d7566b..2664775c0fbf4 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -115,8 +115,8 @@ ARROW_UTF8PROC_BUILD_VERSION=v2.7.0 ARROW_UTF8PROC_BUILD_SHA256_CHECKSUM=4bb121e297293c0fd55f08f83afab6d35d48f0af4ecc07523ad8ec99aa2b12a1 ARROW_XSIMD_BUILD_VERSION=9.0.1 ARROW_XSIMD_BUILD_SHA256_CHECKSUM=b1bb5f92167fd3a4f25749db0be7e61ed37e0a5d943490f3accdcd2cd2918cc0 -ARROW_ZLIB_BUILD_VERSION=1.2.13 -ARROW_ZLIB_BUILD_SHA256_CHECKSUM=b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30 +ARROW_ZLIB_BUILD_VERSION=1.3 +ARROW_ZLIB_BUILD_SHA256_CHECKSUM=ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e ARROW_ZSTD_BUILD_VERSION=1.5.5 ARROW_ZSTD_BUILD_SHA256_CHECKSUM=9c4396cc829cfae319a6e2615202e82aad41372073482fce286fac78646d3ee4 From 4a33d2f8f8895da46e1d19e7662c344bc452bda2 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 17 Jan 2024 16:58:48 +0100 Subject: [PATCH 41/92] GH-39640: [Docs] Pin pydata-sphinx-theme to 0.14.1 (#39658) The version warning banner in the documentation has the wrong version: it currently uses `pydata-sphinx-version` instead of Arrow dev version. Testing if the update in upstream fixes this error in version `14.0.1`. * Closes: #39640 Authored-by: AlenkaF Signed-off-by: AlenkaF --- ci/conda_env_sphinx.txt | 2 +- docs/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt index 0e50875fc1ef8..d0f494d2e085d 100644 --- a/ci/conda_env_sphinx.txt +++ b/ci/conda_env_sphinx.txt @@ -20,7 +20,7 @@ breathe doxygen ipython numpydoc -pydata-sphinx-theme=0.14 +pydata-sphinx-theme=0.14.1 sphinx-autobuild sphinx-design sphinx-copybutton diff --git a/docs/requirements.txt b/docs/requirements.txt index da2327a6df5fc..aee2eb662c06b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,7 +5,7 @@ breathe ipython numpydoc -pydata-sphinx-theme==0.14 +pydata-sphinx-theme==0.14.1 sphinx-autobuild sphinx-design sphinx-copybutton From 1dc3b81875d243f5d135caa6b2db7a669888ecb4 Mon Sep 17 00:00:00 2001 From: "Rossi(Ruoxi) Sun" Date: Thu, 18 Jan 2024 00:26:48 +0800 Subject: [PATCH 42/92] GH-39583: [C++] Fix the issue of ExecBatchBuilder when appending consecutive tail rows with the same id may exceed buffer boundary (for fixed size types) (#39585) ### Rationale for this change #39583 is a subsequent issue of #32570 (fixed by #39234). The last issue and fixed only resolved var length types. It turns out fixed size types have the same issue. ### What changes are included in this PR? Do the same fix of #39234 for fixed size types. ### Are these changes tested? UT included. ### Are there any user-facing changes? * Closes: #39583 Authored-by: zanmato1984 Signed-off-by: Antoine Pitrou --- cpp/src/arrow/compute/light_array.cc | 21 +++---- cpp/src/arrow/compute/light_array_test.cc | 75 +++++++++++++++++++++-- 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/cpp/src/arrow/compute/light_array.cc b/cpp/src/arrow/compute/light_array.cc index 73ea01a03a8fa..66d8477b029a0 100644 --- a/cpp/src/arrow/compute/light_array.cc +++ b/cpp/src/arrow/compute/light_array.cc @@ -383,27 +383,22 @@ int ExecBatchBuilder::NumRowsToSkip(const std::shared_ptr& column, KeyColumnMetadata column_metadata = ColumnMetadataFromDataType(column->type).ValueOrDie(); + ARROW_DCHECK(!column_metadata.is_fixed_length || column_metadata.fixed_length > 0); int num_rows_left = num_rows; int num_bytes_skipped = 0; while (num_rows_left > 0 && num_bytes_skipped < num_tail_bytes_to_skip) { + --num_rows_left; + int row_id_removed = row_ids[num_rows_left]; if (column_metadata.is_fixed_length) { - if (column_metadata.fixed_length == 0) { - num_rows_left = std::max(num_rows_left, 8) - 8; - ++num_bytes_skipped; - } else { - --num_rows_left; - num_bytes_skipped += column_metadata.fixed_length; - } + num_bytes_skipped += column_metadata.fixed_length; } else { - --num_rows_left; - int row_id_removed = row_ids[num_rows_left]; const int32_t* offsets = column->GetValues(1); num_bytes_skipped += offsets[row_id_removed + 1] - offsets[row_id_removed]; - // Skip consecutive rows with the same id - while (num_rows_left > 0 && row_id_removed == row_ids[num_rows_left - 1]) { - --num_rows_left; - } + } + // Skip consecutive rows with the same id + while (num_rows_left > 0 && row_id_removed == row_ids[num_rows_left - 1]) { + --num_rows_left; } } diff --git a/cpp/src/arrow/compute/light_array_test.cc b/cpp/src/arrow/compute/light_array_test.cc index 3ceba43604b28..d50e9675517c3 100644 --- a/cpp/src/arrow/compute/light_array_test.cc +++ b/cpp/src/arrow/compute/light_array_test.cc @@ -474,15 +474,18 @@ TEST(ExecBatchBuilder, AppendBatchesSomeRows) { TEST(ExecBatchBuilder, AppendBatchDupRows) { std::unique_ptr owned_pool = MemoryPool::CreateDefault(); MemoryPool* pool = owned_pool.get(); + // Case of cross-word copying for the last row, which may exceed the buffer boundary. - // This is a simplified case of GH-32570 + // { + // This is a simplified case of GH-32570 // 64-byte data fully occupying one minimal 64-byte aligned memory region. - ExecBatch batch_string = JSONToExecBatch({binary()}, R"([["123456789ABCDEF0"], - ["123456789ABCDEF0"], - ["123456789ABCDEF0"], - ["ABCDEF0"], - ["123456789"]])"); // 9-byte tail row, larger than a word. + ExecBatch batch_string = JSONToExecBatch({binary()}, R"([ + ["123456789ABCDEF0"], + ["123456789ABCDEF0"], + ["123456789ABCDEF0"], + ["ABCDEF0"], + ["123456789"]])"); // 9-byte tail row, larger than a word. ASSERT_EQ(batch_string[0].array()->buffers[1]->capacity(), 64); ASSERT_EQ(batch_string[0].array()->buffers[2]->capacity(), 64); ExecBatchBuilder builder; @@ -494,6 +497,66 @@ TEST(ExecBatchBuilder, AppendBatchDupRows) { ASSERT_EQ(batch_string_appended, built); ASSERT_NE(0, pool->bytes_allocated()); } + + { + // This is a simplified case of GH-39583, using fsb(3) type. + // 63-byte data occupying almost one minimal 64-byte aligned memory region. + ExecBatch batch_fsb = JSONToExecBatch({fixed_size_binary(3)}, R"([ + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["000"], + ["123"]])"); // 3-byte tail row, not aligned to a word. + ASSERT_EQ(batch_fsb[0].array()->buffers[1]->capacity(), 64); + ExecBatchBuilder builder; + uint16_t row_ids[4] = {20, 20, 20, + 20}; // Get the last row 4 times, 3 to skip a word. + ASSERT_OK(builder.AppendSelected(pool, batch_fsb, 4, row_ids, /*num_cols=*/1)); + ExecBatch built = builder.Flush(); + ExecBatch batch_fsb_appended = JSONToExecBatch( + {fixed_size_binary(3)}, R"([["123"], ["123"], ["123"], ["123"]])"); + ASSERT_EQ(batch_fsb_appended, built); + ASSERT_NE(0, pool->bytes_allocated()); + } + + { + // This is a simplified case of GH-39583, using fsb(9) type. + // 63-byte data occupying almost one minimal 64-byte aligned memory region. + ExecBatch batch_fsb = JSONToExecBatch({fixed_size_binary(9)}, R"([ + ["000000000"], + ["000000000"], + ["000000000"], + ["000000000"], + ["000000000"], + ["000000000"], + ["123456789"]])"); // 9-byte tail row, not aligned to a word. + ASSERT_EQ(batch_fsb[0].array()->buffers[1]->capacity(), 64); + ExecBatchBuilder builder; + uint16_t row_ids[2] = {6, 6}; // Get the last row 2 times, 1 to skip a word. + ASSERT_OK(builder.AppendSelected(pool, batch_fsb, 2, row_ids, /*num_cols=*/1)); + ExecBatch built = builder.Flush(); + ExecBatch batch_fsb_appended = + JSONToExecBatch({fixed_size_binary(9)}, R"([["123456789"], ["123456789"]])"); + ASSERT_EQ(batch_fsb_appended, built); + ASSERT_NE(0, pool->bytes_allocated()); + } + ASSERT_EQ(0, pool->bytes_allocated()); } From ed726fa7e1786719af1f0f3a731f615fe46ae7b1 Mon Sep 17 00:00:00 2001 From: Dane Pitkin <48041712+danepitkin@users.noreply.github.com> Date: Wed, 17 Jan 2024 11:52:57 -0500 Subject: [PATCH 43/92] MINOR: [CI] Add new collaborator (vibhatha) (#39661) * Add new collaborator @ vibhatha (60 commits to Arrow repo) * Remove @ felipecrv, who has become a committer * Alphabetize list Authored-by: Dane Pitkin Signed-off-by: Jacob Wujciak-Jens --- .asf.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index d2522ecae0b43..ae2709a8b0acd 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -19,13 +19,13 @@ github: description: "Apache Arrow is a multi-language toolbox for accelerated data interchange and in-memory processing" homepage: https://arrow.apache.org/ collaborators: + - amoeba - anjakefala - benibus - danepitkin - davisusanibar - - felipecrv - js8544 - - amoeba + - vibhatha notifications: commits: commits@arrow.apache.org From c170af41ba0c30b80aa4172da0b3637206368cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A2nio?= Date: Wed, 17 Jan 2024 14:00:39 -0300 Subject: [PATCH 44/92] GH-39552: [Go] inclusion of option to use replacer when creating csv strings with go library (#39576) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rationale for this change Make it possible to remove unwanted characters from strings What changes are included in this PR? Add new function to optionally setup a replacer in csv Writer Write method Are these changes tested? Yes Are there any user-facing changes? Adds an optional methods. * Closes: #39552 Lead-authored-by: Jânio Co-authored-by: janiodev Signed-off-by: Matt Topol --- go/arrow/csv/common.go | 14 ++++++++++++++ go/arrow/csv/transformer.go | 12 ++++++------ go/arrow/csv/writer.go | 24 +++++++++++++----------- go/arrow/csv/writer_test.go | 6 ++++-- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/go/arrow/csv/common.go b/go/arrow/csv/common.go index 99dac29f4d728..31ca61f323d36 100644 --- a/go/arrow/csv/common.go +++ b/go/arrow/csv/common.go @@ -21,6 +21,7 @@ package csv import ( "errors" "fmt" + "strings" "github.com/apache/arrow/go/v15/arrow" "github.com/apache/arrow/go/v15/arrow/memory" @@ -223,6 +224,19 @@ func WithIncludeColumns(cols []string) Option { } } +// WithStringsReplacer receives a replacer to be applied in the string fields +// of the CSV. This is useful to remove unwanted characters from the string. +func WithStringsReplacer(replacer *strings.Replacer) Option { + return func(cfg config) { + switch cfg := cfg.(type) { + case *Writer: + cfg.stringReplacer = replacer.Replace + default: + panic(fmt.Errorf("arrow/csv: unknown config type %T", cfg)) + } + } +} + func validate(schema *arrow.Schema) { for i, f := range schema.Fields() { switch ft := f.Type.(type) { diff --git a/go/arrow/csv/transformer.go b/go/arrow/csv/transformer.go index 0f0181520b847..78b16446d4def 100644 --- a/go/arrow/csv/transformer.go +++ b/go/arrow/csv/transformer.go @@ -29,7 +29,7 @@ import ( "github.com/apache/arrow/go/v15/arrow/array" ) -func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array) []string { +func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array, stringsReplacer func(string)string) []string { res := make([]string, col.Len()) switch typ.(type) { case *arrow.BooleanType: @@ -144,7 +144,7 @@ func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array) [] arr := col.(*array.String) for i := 0; i < arr.Len(); i++ { if arr.IsValid(i) { - res[i] = arr.Value(i) + res[i] = stringsReplacer(arr.Value(i)) } else { res[i] = w.nullValue } @@ -153,7 +153,7 @@ func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array) [] arr := col.(*array.LargeString) for i := 0; i < arr.Len(); i++ { if arr.IsValid(i) { - res[i] = arr.Value(i) + res[i] = stringsReplacer(arr.Value(i)) } else { res[i] = w.nullValue } @@ -224,7 +224,7 @@ func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array) [] var b bytes.Buffer b.Write([]byte{'{'}) writer := csv.NewWriter(&b) - writer.Write(w.transformColToStringArr(list.DataType(), list)) + writer.Write(w.transformColToStringArr(list.DataType(), list, stringsReplacer)) writer.Flush() b.Truncate(b.Len() - 1) b.Write([]byte{'}'}) @@ -243,7 +243,7 @@ func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array) [] var b bytes.Buffer b.Write([]byte{'{'}) writer := csv.NewWriter(&b) - writer.Write(w.transformColToStringArr(list.DataType(), list)) + writer.Write(w.transformColToStringArr(list.DataType(), list, stringsReplacer)) writer.Flush() b.Truncate(b.Len() - 1) b.Write([]byte{'}'}) @@ -262,7 +262,7 @@ func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array) [] var b bytes.Buffer b.Write([]byte{'{'}) writer := csv.NewWriter(&b) - writer.Write(w.transformColToStringArr(list.DataType(), list)) + writer.Write(w.transformColToStringArr(list.DataType(), list, stringsReplacer)) writer.Flush() b.Truncate(b.Len() - 1) b.Write([]byte{'}'}) diff --git a/go/arrow/csv/writer.go b/go/arrow/csv/writer.go index a672008b58a07..b939b72984b0f 100644 --- a/go/arrow/csv/writer.go +++ b/go/arrow/csv/writer.go @@ -27,12 +27,13 @@ import ( // Writer wraps encoding/csv.Writer and writes arrow.Record based on a schema. type Writer struct { - boolFormatter func(bool) string - header bool - nullValue string - once sync.Once - schema *arrow.Schema - w *csv.Writer + boolFormatter func(bool) string + header bool + nullValue string + stringReplacer func(string) string + once sync.Once + schema *arrow.Schema + w *csv.Writer } // NewWriter returns a writer that writes arrow.Records to the CSV file @@ -45,10 +46,11 @@ func NewWriter(w io.Writer, schema *arrow.Schema, opts ...Option) *Writer { validate(schema) ww := &Writer{ - boolFormatter: strconv.FormatBool, // override by passing WithBoolWriter() as an option - nullValue: "NULL", // override by passing WithNullWriter() as an option - schema: schema, - w: csv.NewWriter(w), + boolFormatter: strconv.FormatBool, // override by passing WithBoolWriter() as an option + nullValue: "NULL", // override by passing WithNullWriter() as an option + stringReplacer: func(x string) string { return x }, // override by passing WithStringsReplacer() as an option + schema: schema, + w: csv.NewWriter(w), } for _, opt := range opts { opt(ww) @@ -81,7 +83,7 @@ func (w *Writer) Write(record arrow.Record) error { } for j, col := range record.Columns() { - rows := w.transformColToStringArr(w.schema.Field(j).Type, col) + rows := w.transformColToStringArr(w.schema.Field(j).Type, col, w.stringReplacer) for i, row := range rows { recs[i][j] = row } diff --git a/go/arrow/csv/writer_test.go b/go/arrow/csv/writer_test.go index 644cae0933f4c..b1bd3251c5622 100644 --- a/go/arrow/csv/writer_test.go +++ b/go/arrow/csv/writer_test.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "log" + "strings" "testing" "github.com/apache/arrow/go/v15/arrow" @@ -250,8 +251,8 @@ func testCSVWriter(t *testing.T, data [][]string, writeHeader bool, fmtr func(bo b.Field(9).(*array.Float16Builder).AppendValues([]float16.Num{float16.New(0.0), float16.New(0.1), float16.New(0.2)}, nil) b.Field(10).(*array.Float32Builder).AppendValues([]float32{0.0, 0.1, 0.2}, nil) b.Field(11).(*array.Float64Builder).AppendValues([]float64{0.0, 0.1, 0.2}, nil) - b.Field(12).(*array.StringBuilder).AppendValues([]string{"str-0", "str-1", "str-2"}, nil) - b.Field(13).(*array.LargeStringBuilder).AppendValues([]string{"str-0", "str-1", "str-2"}, nil) + b.Field(12).(*array.StringBuilder).AppendValues([]string{"str_0", "str-1", "str-2"}, nil) + b.Field(13).(*array.LargeStringBuilder).AppendValues([]string{"str_0", "str-1", "str-2"}, nil) b.Field(14).(*array.TimestampBuilder).AppendValues(genTimestamps(arrow.Second), nil) b.Field(15).(*array.Date32Builder).AppendValues([]arrow.Date32{17304, 19304, 20304}, nil) b.Field(16).(*array.Date64Builder).AppendValues([]arrow.Date64{1840400000000, 1940400000000, 2040400000000}, nil) @@ -300,6 +301,7 @@ func testCSVWriter(t *testing.T, data [][]string, writeHeader bool, fmtr func(bo csv.WithHeader(writeHeader), csv.WithNullWriter(nullVal), csv.WithBoolWriter(fmtr), + csv.WithStringsReplacer(strings.NewReplacer("_", "-")), ) err := w.Write(rec) if err != nil { From 1c49b9f0cf378c6a6455b17b5d0b9bc04c77ab5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 18 Jan 2024 04:22:57 +0100 Subject: [PATCH 45/92] GH-39656: [Release] Update platform tags for macOS wheels to macosx_10_15 (#39657) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Currently the binary verification for releases fails due to wrong macOS platform version. ### What changes are included in this PR? Update to the current generated platform tag. ### Are these changes tested? No, but I've validated this is the corrected generated platform tag for the wheels on the Release Candidate: https://apache.jfrog.io/ui/native/arrow/python-rc/15.0.0-rc1/ ### Are there any user-facing changes? Not because of this change but there was a minimum version of macOS as part of the PR that caused this issue. * Closes: #39656 Authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index c5e27d083013e..90f071c5b4865 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1136,7 +1136,7 @@ test_macos_wheels() { local check_flight=OFF else local python_versions="3.8 3.9 3.10 3.11 3.12" - local platform_tags="macosx_10_14_x86_64" + local platform_tags="macosx_10_15_x86_64" fi # verify arch-native wheels inside an arch-native conda environment From 1f5dece9681d804bd3b03b2106b7426217b213e9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 18 Jan 2024 18:46:17 +0900 Subject: [PATCH 46/92] GH-39683: [Release] Use temporary direction with TEST_BINARY=1 (#39684) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change We should use temporary directory to verify in clean environment. ### What changes are included in this PR? Use `ARROW_TMPDIR` for prefix of download directory. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #39683 Authored-by: Sutou Kouhei Signed-off-by: Raúl Cumplido --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 90f071c5b4865..04fc7fd563f65 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -173,7 +173,7 @@ test_binary() { show_header "Testing binary artifacts" maybe_setup_conda - local download_dir=binaries + local download_dir=${ARROW_TMPDIR}/binaries mkdir -p ${download_dir} ${PYTHON:-python3} $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \ From a2aa1c4122fca88f87b297711fa49b8ad355416d Mon Sep 17 00:00:00 2001 From: "Rossi(Ruoxi) Sun" Date: Thu, 18 Jan 2024 19:44:26 +0800 Subject: [PATCH 47/92] GH-39332: [C++] Explicit error in ExecBatchBuilder when appending var length data exceeds offset limit (int32 max) (#39383) ### Rationale for this change When appending var length data in `ExecBatchBuilder`, the offset is potentially to overflow if the batch contains 4GB data or more. This may further result in segmentation fault during the subsequent data content copying. For details, please refer to this comment: https://github.com/apache/arrow/issues/39332#issuecomment-1870690063. The solution is let user to use the "large" counterpart data type to avoid the overflow, but we may need explicit error information when such overflow happens. ### What changes are included in this PR? 1. Detect the offset overflow in appending data in `ExecBatchBuilder` and explicitly throw. 2. Change the offset type from `uint32_t` to `int32_t` in `ExecBatchBuilder` and respects the `BinaryBuilder::memory_limit()` which is `2GB - 2B` as the rest part of the codebase. ### Are these changes tested? UT included. ### Are there any user-facing changes? No. * Closes: #39332 Lead-authored-by: zanmato Co-authored-by: Antoine Pitrou Co-authored-by: Rossi(Ruoxi) Sun Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/compute/light_array.cc | 47 ++++++++++------- cpp/src/arrow/compute/light_array.h | 2 +- cpp/src/arrow/compute/light_array_test.cc | 64 +++++++++++++++++++++++ cpp/src/arrow/testing/generator.cc | 9 +++- 4 files changed, 100 insertions(+), 22 deletions(-) diff --git a/cpp/src/arrow/compute/light_array.cc b/cpp/src/arrow/compute/light_array.cc index 66d8477b029a0..b225e04b05cea 100644 --- a/cpp/src/arrow/compute/light_array.cc +++ b/cpp/src/arrow/compute/light_array.cc @@ -20,6 +20,8 @@ #include #include "arrow/util/bitmap_ops.h" +#include "arrow/util/int_util_overflow.h" +#include "arrow/util/macros.h" namespace arrow { namespace compute { @@ -325,11 +327,10 @@ Status ResizableArrayData::ResizeVaryingLengthBuffer() { column_metadata = ColumnMetadataFromDataType(data_type_).ValueOrDie(); if (!column_metadata.is_fixed_length) { - int min_new_size = static_cast(reinterpret_cast( - buffers_[kFixedLengthBuffer]->data())[num_rows_]); + int64_t min_new_size = buffers_[kFixedLengthBuffer]->data_as()[num_rows_]; ARROW_DCHECK(var_len_buf_size_ > 0); if (var_len_buf_size_ < min_new_size) { - int new_size = var_len_buf_size_; + int64_t new_size = var_len_buf_size_; while (new_size < min_new_size) { new_size *= 2; } @@ -465,12 +466,11 @@ void ExecBatchBuilder::Visit(const std::shared_ptr& column, int num_r if (!metadata.is_fixed_length) { const uint8_t* ptr_base = column->buffers[2]->data(); - const uint32_t* offsets = - reinterpret_cast(column->buffers[1]->data()) + column->offset; + const int32_t* offsets = column->GetValues(1); for (int i = 0; i < num_rows; ++i) { uint16_t row_id = row_ids[i]; const uint8_t* field_ptr = ptr_base + offsets[row_id]; - uint32_t field_length = offsets[row_id + 1] - offsets[row_id]; + int32_t field_length = offsets[row_id + 1] - offsets[row_id]; process_value_fn(i, field_ptr, field_length); } } else { @@ -480,7 +480,7 @@ void ExecBatchBuilder::Visit(const std::shared_ptr& column, int num_r const uint8_t* field_ptr = column->buffers[1]->data() + (column->offset + row_id) * static_cast(metadata.fixed_length); - process_value_fn(i, field_ptr, metadata.fixed_length); + process_value_fn(i, field_ptr, static_cast(metadata.fixed_length)); } } } @@ -511,14 +511,14 @@ Status ExecBatchBuilder::AppendSelected(const std::shared_ptr& source break; case 1: Visit(source, num_rows_to_append, row_ids, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { target->mutable_data(1)[num_rows_before + i] = *ptr; }); break; case 2: Visit( source, num_rows_to_append, row_ids, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { reinterpret_cast(target->mutable_data(1))[num_rows_before + i] = *reinterpret_cast(ptr); }); @@ -526,7 +526,7 @@ Status ExecBatchBuilder::AppendSelected(const std::shared_ptr& source case 4: Visit( source, num_rows_to_append, row_ids, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { reinterpret_cast(target->mutable_data(1))[num_rows_before + i] = *reinterpret_cast(ptr); }); @@ -534,7 +534,7 @@ Status ExecBatchBuilder::AppendSelected(const std::shared_ptr& source case 8: Visit( source, num_rows_to_append, row_ids, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { reinterpret_cast(target->mutable_data(1))[num_rows_before + i] = *reinterpret_cast(ptr); }); @@ -544,7 +544,7 @@ Status ExecBatchBuilder::AppendSelected(const std::shared_ptr& source num_rows_to_append - NumRowsToSkip(source, num_rows_to_append, row_ids, sizeof(uint64_t)); Visit(source, num_rows_to_process, row_ids, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { uint64_t* dst = reinterpret_cast( target->mutable_data(1) + static_cast(num_bytes) * (num_rows_before + i)); @@ -558,7 +558,7 @@ Status ExecBatchBuilder::AppendSelected(const std::shared_ptr& source if (num_rows_to_append > num_rows_to_process) { Visit(source, num_rows_to_append - num_rows_to_process, row_ids + num_rows_to_process, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { uint64_t* dst = reinterpret_cast( target->mutable_data(1) + static_cast(num_bytes) * @@ -575,16 +575,23 @@ Status ExecBatchBuilder::AppendSelected(const std::shared_ptr& source // Step 1: calculate target offsets // - uint32_t* offsets = reinterpret_cast(target->mutable_data(1)); - uint32_t sum = num_rows_before == 0 ? 0 : offsets[num_rows_before]; + int32_t* offsets = reinterpret_cast(target->mutable_data(1)); + int32_t sum = num_rows_before == 0 ? 0 : offsets[num_rows_before]; Visit(source, num_rows_to_append, row_ids, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { offsets[num_rows_before + i] = num_bytes; }); for (int i = 0; i < num_rows_to_append; ++i) { - uint32_t length = offsets[num_rows_before + i]; + int32_t length = offsets[num_rows_before + i]; offsets[num_rows_before + i] = sum; - sum += length; + int32_t new_sum_maybe_overflow = 0; + if (ARROW_PREDICT_FALSE( + arrow::internal::AddWithOverflow(sum, length, &new_sum_maybe_overflow))) { + return Status::Invalid("Overflow detected in ExecBatchBuilder when appending ", + num_rows_before + i + 1, "-th element of length ", length, + " bytes to current length ", sum, " bytes"); + } + sum = new_sum_maybe_overflow; } offsets[num_rows_before + num_rows_to_append] = sum; @@ -598,7 +605,7 @@ Status ExecBatchBuilder::AppendSelected(const std::shared_ptr& source num_rows_to_append - NumRowsToSkip(source, num_rows_to_append, row_ids, sizeof(uint64_t)); Visit(source, num_rows_to_process, row_ids, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { uint64_t* dst = reinterpret_cast(target->mutable_data(2) + offsets[num_rows_before + i]); const uint64_t* src = reinterpret_cast(ptr); @@ -608,7 +615,7 @@ Status ExecBatchBuilder::AppendSelected(const std::shared_ptr& source } }); Visit(source, num_rows_to_append - num_rows_to_process, row_ids + num_rows_to_process, - [&](int i, const uint8_t* ptr, uint32_t num_bytes) { + [&](int i, const uint8_t* ptr, int32_t num_bytes) { uint64_t* dst = reinterpret_cast( target->mutable_data(2) + offsets[num_rows_before + num_rows_to_process + i]); diff --git a/cpp/src/arrow/compute/light_array.h b/cpp/src/arrow/compute/light_array.h index 84aa86d64bb62..67de71bf56c92 100644 --- a/cpp/src/arrow/compute/light_array.h +++ b/cpp/src/arrow/compute/light_array.h @@ -353,7 +353,7 @@ class ARROW_EXPORT ResizableArrayData { MemoryPool* pool_; int num_rows_; int num_rows_allocated_; - int var_len_buf_size_; + int64_t var_len_buf_size_; static constexpr int kMaxBuffers = 3; std::shared_ptr buffers_[kMaxBuffers]; }; diff --git a/cpp/src/arrow/compute/light_array_test.cc b/cpp/src/arrow/compute/light_array_test.cc index d50e9675517c3..ecc5f3ad37931 100644 --- a/cpp/src/arrow/compute/light_array_test.cc +++ b/cpp/src/arrow/compute/light_array_test.cc @@ -407,6 +407,70 @@ TEST(ExecBatchBuilder, AppendValuesBeyondLimit) { ASSERT_EQ(0, pool->bytes_allocated()); } +TEST(ExecBatchBuilder, AppendVarLengthBeyondLimit) { + // GH-39332: check appending variable-length data past 2GB. + if constexpr (sizeof(void*) == 4) { + GTEST_SKIP() << "Test only works on 64-bit platforms"; + } + + std::unique_ptr owned_pool = MemoryPool::CreateDefault(); + MemoryPool* pool = owned_pool.get(); + constexpr auto eight_mb = 8 * 1024 * 1024; + constexpr auto eight_mb_minus_one = eight_mb - 1; + // String of size 8mb to repetitively fill the heading multiple of 8mbs of an array + // of int32_max bytes. + std::string str_8mb(eight_mb, 'a'); + // String of size (8mb - 1) to be the last element of an array of int32_max bytes. + std::string str_8mb_minus_1(eight_mb_minus_one, 'b'); + std::shared_ptr values_8mb = ConstantArrayGenerator::String(1, str_8mb); + std::shared_ptr values_8mb_minus_1 = + ConstantArrayGenerator::String(1, str_8mb_minus_1); + + ExecBatch batch_8mb({values_8mb}, 1); + ExecBatch batch_8mb_minus_1({values_8mb_minus_1}, 1); + + auto num_rows = std::numeric_limits::max() / eight_mb; + std::vector body_row_ids(num_rows, 0); + std::vector tail_row_id(1, 0); + + { + // Building an array of (int32_max + 1) = (8mb * num_rows + 8mb) bytes should raise an + // error of overflow. + ExecBatchBuilder builder; + ASSERT_OK(builder.AppendSelected(pool, batch_8mb, num_rows, body_row_ids.data(), + /*num_cols=*/1)); + std::stringstream ss; + ss << "Invalid: Overflow detected in ExecBatchBuilder when appending " << num_rows + 1 + << "-th element of length " << eight_mb << " bytes to current length " + << eight_mb * num_rows << " bytes"; + ASSERT_RAISES_WITH_MESSAGE( + Invalid, ss.str(), + builder.AppendSelected(pool, batch_8mb, 1, tail_row_id.data(), + /*num_cols=*/1)); + } + + { + // Building an array of int32_max = (8mb * num_rows + 8mb - 1) bytes should succeed. + ExecBatchBuilder builder; + ASSERT_OK(builder.AppendSelected(pool, batch_8mb, num_rows, body_row_ids.data(), + /*num_cols=*/1)); + ASSERT_OK(builder.AppendSelected(pool, batch_8mb_minus_1, 1, tail_row_id.data(), + /*num_cols=*/1)); + ExecBatch built = builder.Flush(); + auto datum = built[0]; + ASSERT_TRUE(datum.is_array()); + auto array = datum.array_as(); + ASSERT_EQ(array->length(), num_rows + 1); + for (int i = 0; i < num_rows; ++i) { + ASSERT_EQ(array->GetString(i), str_8mb); + } + ASSERT_EQ(array->GetString(num_rows), str_8mb_minus_1); + ASSERT_NE(0, pool->bytes_allocated()); + } + + ASSERT_EQ(0, pool->bytes_allocated()); +} + TEST(KeyColumnArray, FromExecBatch) { ExecBatch batch = JSONToExecBatch({int64(), boolean()}, "[[1, true], [2, false], [null, null]]"); diff --git a/cpp/src/arrow/testing/generator.cc b/cpp/src/arrow/testing/generator.cc index 36c88c20efe6e..5ea6a541e8922 100644 --- a/cpp/src/arrow/testing/generator.cc +++ b/cpp/src/arrow/testing/generator.cc @@ -38,6 +38,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/checked_cast.h" +#include "arrow/util/logging.h" #include "arrow/util/macros.h" #include "arrow/util/string.h" @@ -103,7 +104,13 @@ std::shared_ptr ConstantArrayGenerator::Float64(int64_t size, std::shared_ptr ConstantArrayGenerator::String(int64_t size, std::string value) { - return ConstantArray(size, value); + using BuilderType = typename TypeTraits::BuilderType; + auto type = TypeTraits::type_singleton(); + auto builder_fn = [&](BuilderType* builder) { + DCHECK_OK(builder->Append(std::string_view(value.data()))); + }; + return ArrayFromBuilderVisitor(type, value.size() * size, size, builder_fn) + .ValueOrDie(); } std::shared_ptr ConstantArrayGenerator::Zeroes( From 858574d0bd1f3ef4157d0446cfb05cef05aac96b Mon Sep 17 00:00:00 2001 From: Joel Lubinitsky <33523178+joellubi@users.noreply.github.com> Date: Thu, 18 Jan 2024 11:09:50 -0500 Subject: [PATCH 48/92] GH-39466: [Go][Parquet] Align Arrow and Parquet Timestamp Instant/Local Semantics (#39467) ### Rationale for this change Closes: #39466 ### What changes are included in this PR? - Update logic for determining whether an Arrow Timestamp should have `isAdjustedToUTC=true` on conversion to Parquet. - Update conversion from Parquet Timestamp to Arrow Timestamp to align with Parquet Format [backward-compatibilty](https://github.com/apache/parquet-format/blob/eb4b31c1d64a01088d02a2f9aefc6c17c54cc6fc/LogicalTypes.md?plain=1#L480-L485) rules. - Refactor Timestamp serialization methods to reduce duplicated code ### Are these changes tested? Yes, - Logical type mapping in existing test updated. - New tests for roundtrip behavior of timestamps with various timezone settings, with/without store_schema enabled. - New test to clarify equality behavior of timestamps with instant semantics, as well as Go-related quirks with timezone-unaware timestamps. ### Are there any user-facing changes? Yes, users of `pqarrow.FileWriter` will produce Parquet files in which the `TIMESTAMP` type is normalized to UTC IFF the Arrow type provided has a timezone specified. This is different from the current Go behavior but aligned that of other implementations. The conversion from Parquet to Arrow has been updated as well to reflect the Parquet format [document](https://github.com/apache/parquet-format/blob/eb4b31c1d64a01088d02a2f9aefc6c17c54cc6fc/LogicalTypes.md?plain=1#L480-L485). Rust already [implements](https://github.com/apache/arrow-rs/blob/a61e824abdd7b38ea214828480430ff2a13f2ead/parquet/src/arrow/schema/primitive.rs#L211-L239) the spec as described and #39489 has been reported due to a mismatch in the handling of convertedTypes in C++. * Closes: #39466 Authored-by: Joel Lubinitsky Signed-off-by: Matt Topol --- go/arrow/array/timestamp.go | 11 ++-- go/arrow/array/timestamp_test.go | 49 ++++++++++++++++- go/arrow/datatype_fixedwidth.go | 19 +++---- go/parquet/pqarrow/encode_arrow_test.go | 70 +++++++++++++++++++++++++ go/parquet/pqarrow/schema.go | 13 +++-- go/parquet/pqarrow/schema_test.go | 6 +-- 6 files changed, 140 insertions(+), 28 deletions(-) diff --git a/go/arrow/array/timestamp.go b/go/arrow/array/timestamp.go index 6ffb43e067af0..0cc46a127fc51 100644 --- a/go/arrow/array/timestamp.go +++ b/go/arrow/array/timestamp.go @@ -91,16 +91,15 @@ func (a *Timestamp) ValueStr(i int) string { return NullValueStr } - dt := a.DataType().(*arrow.TimestampType) - z, _ := dt.GetZone() - return a.values[i].ToTime(dt.Unit).In(z).Format("2006-01-02 15:04:05.999999999Z0700") + toTime, _ := a.DataType().(*arrow.TimestampType).GetToTimeFunc() + return toTime(a.values[i]).Format("2006-01-02 15:04:05.999999999Z0700") } func (a *Timestamp) GetOneForMarshal(i int) interface{} { - if a.IsNull(i) { - return nil + if val := a.ValueStr(i); val != NullValueStr { + return val } - return a.values[i].ToTime(a.DataType().(*arrow.TimestampType).Unit).Format("2006-01-02 15:04:05.999999999") + return nil } func (a *Timestamp) MarshalJSON() ([]byte, error) { diff --git a/go/arrow/array/timestamp_test.go b/go/arrow/array/timestamp_test.go index acbad8b586dd4..c172ad811dc37 100644 --- a/go/arrow/array/timestamp_test.go +++ b/go/arrow/array/timestamp_test.go @@ -234,7 +234,7 @@ func TestTimestampBuilder_Resize(t *testing.T) { assert.Equal(t, 5, ab.Len()) } -func TestTimestampValueStr(t *testing.T) { +func TestTimestampValueStr(t *testing.T) { mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) @@ -251,3 +251,50 @@ func TestTimestampValueStr(t *testing.T) { assert.Equal(t, "1968-11-30 13:30:45-0700", arr.ValueStr(0)) assert.Equal(t, "2016-02-29 10:42:23-0700", arr.ValueStr(1)) } + +func TestTimestampEquality(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + + tsDatatypes := []*arrow.TimestampType{ + {Unit: arrow.Second}, + {Unit: arrow.Second, TimeZone: "UTC"}, + {Unit: arrow.Second, TimeZone: "America/Phoenix"}, + } + + arrs := make([]*array.Timestamp, 0, len(tsDatatypes)) + for _, dt := range tsDatatypes { + bldr := array.NewTimestampBuilder(mem, dt) + defer bldr.Release() + + bldr.Append(-34226955) + bldr.Append(1456767743) + + arr := bldr.NewTimestampArray() + defer arr.Release() + + arrs = append(arrs, arr) + } + + // No timezone, "wall clock" semantics + // These timestamps have no actual timezone, but we still represent as UTC per Go conventions + assert.Equal(t, "1968-11-30 20:30:45Z", arrs[0].ValueStr(0)) + assert.Equal(t, "2016-02-29 17:42:23Z", arrs[0].ValueStr(1)) + + // UTC timezone, "instant" semantics + assert.Equal(t, "1968-11-30 20:30:45Z", arrs[1].ValueStr(0)) + assert.Equal(t, "2016-02-29 17:42:23Z", arrs[1].ValueStr(1)) + + // America/Phoenix timezone, "instant" semantics + assert.Equal(t, "1968-11-30 13:30:45-0700", arrs[2].ValueStr(0)) + assert.Equal(t, "2016-02-29 10:42:23-0700", arrs[2].ValueStr(1)) + + // Despite timezone and semantics, the physical values are equivalent + assert.Equal(t, arrs[0].Value(0), arrs[1].Value(0)) + assert.Equal(t, arrs[0].Value(0), arrs[2].Value(0)) + assert.Equal(t, arrs[1].Value(0), arrs[2].Value(0)) + + assert.Equal(t, arrs[0].Value(1), arrs[1].Value(1)) + assert.Equal(t, arrs[0].Value(1), arrs[2].Value(1)) + assert.Equal(t, arrs[1].Value(1), arrs[2].Value(1)) +} diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go index 1a3074e59e75f..158dbd67b1b5e 100644 --- a/go/arrow/datatype_fixedwidth.go +++ b/go/arrow/datatype_fixedwidth.go @@ -348,8 +348,11 @@ type TemporalWithUnit interface { } // TimestampType is encoded as a 64-bit signed integer since the UNIX epoch (2017-01-01T00:00:00Z). -// The zero-value is a second and time zone neutral. Time zone neutral can be -// considered UTC without having "UTC" as a time zone. +// The zero-value is a second and time zone neutral. In Arrow semantics, time zone neutral does not +// represent a physical point in time, but rather a "wall clock" time that only has meaning within +// the context that produced it. In Go, time.Time can only represent instants; there is no notion +// of "wall clock" time. Therefore, time zone neutral timestamps are represented as UTC per Go +// conventions even though the Arrow type itself has no time zone. type TimestampType struct { Unit TimeUnit TimeZone string @@ -454,17 +457,7 @@ func (t *TimestampType) GetToTimeFunc() (func(Timestamp) time.Time, error) { return nil, err } - switch t.Unit { - case Second: - return func(v Timestamp) time.Time { return time.Unix(int64(v), 0).In(tz) }, nil - case Millisecond: - return func(v Timestamp) time.Time { return time.UnixMilli(int64(v)).In(tz) }, nil - case Microsecond: - return func(v Timestamp) time.Time { return time.UnixMicro(int64(v)).In(tz) }, nil - case Nanosecond: - return func(v Timestamp) time.Time { return time.Unix(0, int64(v)).In(tz) }, nil - } - return nil, fmt.Errorf("invalid timestamp unit: %s", t.Unit) + return func(v Timestamp) time.Time { return v.ToTime(t.Unit).In(tz) }, nil } // Time32Type is encoded as a 32-bit signed integer, representing either seconds or milliseconds since midnight. diff --git a/go/parquet/pqarrow/encode_arrow_test.go b/go/parquet/pqarrow/encode_arrow_test.go index 75eb965d033b5..25d31b54e1b31 100644 --- a/go/parquet/pqarrow/encode_arrow_test.go +++ b/go/parquet/pqarrow/encode_arrow_test.go @@ -171,6 +171,41 @@ func makeDateTypeTable(mem memory.Allocator, expected bool, partialDays bool) ar return array.NewTableFromRecords(arrsc, []arrow.Record{rec}) } +func makeTimestampTypeTable(mem memory.Allocator, expected bool) arrow.Table { + isValid := []bool{true, true, true, false, true, true} + + // Timestamp with relative (i.e. local) semantics. Make sure it roundtrips without being incorrectly converted to an absolute point in time. + f0 := arrow.Field{Name: "f0", Type: &arrow.TimestampType{Unit: arrow.Millisecond}, Nullable: true, Metadata: arrow.NewMetadata([]string{"PARQUET:field_id"}, []string{"1"})} + + // Timestamp with absolute (i.e. instant) semantics. The physical representation is always from Unix epoch in UTC timezone. + // TimeZone is used for display purposes and can be stripped on roundtrip without changing the actual instant referred to. + // WithStoreSchema will preserve the original timezone, but the instant in will be equivalent even if it's not used. + f1 := arrow.Field{Name: "f1", Type: &arrow.TimestampType{Unit: arrow.Millisecond, TimeZone: "EST"}, Nullable: true, Metadata: arrow.NewMetadata([]string{"PARQUET:field_id"}, []string{"2"})} + f1X := arrow.Field{Name: "f1", Type: &arrow.TimestampType{Unit: arrow.Millisecond, TimeZone: "UTC"}, Nullable: true, Metadata: arrow.NewMetadata([]string{"PARQUET:field_id"}, []string{"2"})} + + fieldList := []arrow.Field{f0} + if expected { + fieldList = append(fieldList, f1X) + } else { + fieldList = append(fieldList, f1) + } + + arrsc := arrow.NewSchema(fieldList, nil) + + ts64msValues := []arrow.Timestamp{1489269, 1489270, 1489271, 1489272, 1489272, 1489273} + + bldr := array.NewRecordBuilder(mem, arrsc) + defer bldr.Release() + + bldr.Field(0).(*array.TimestampBuilder).AppendValues(ts64msValues, isValid) + bldr.Field(1).(*array.TimestampBuilder).AppendValues(ts64msValues, isValid) + + rec := bldr.NewRecord() + defer rec.Release() + + return array.NewTableFromRecords(arrsc, []arrow.Record{rec}) +} + func TestWriteArrowCols(t *testing.T) { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(t, 0) @@ -954,6 +989,25 @@ func (ps *ParquetIOTestSuite) TestDate64ReadWriteTable() { ps.Truef(array.TableEqual(date32ExpectedOutputTable, roundTripOutputTable), "expected table: %s\ngot table: %s", date32ExpectedOutputTable, roundTripOutputTable) } +func (ps *ParquetIOTestSuite) TestTimestampTZReadWriteTable() { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(ps.T(), 0) + + inputTable := makeTimestampTypeTable(mem, false) + defer inputTable.Release() + buf := writeTableToBuffer(ps.T(), mem, inputTable, inputTable.NumRows(), pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem))) + defer buf.Release() + + reader := ps.createReader(mem, buf.Bytes()) + roundTripOutputTable := ps.readTable(reader) + defer roundTripOutputTable.Release() + + expectedOutputTable := makeTimestampTypeTable(mem, true) + defer expectedOutputTable.Release() + + ps.Truef(array.TableEqual(expectedOutputTable, roundTripOutputTable), "expected table: %s\ngot table: %s", expectedOutputTable, roundTripOutputTable) +} + func (ps *ParquetIOTestSuite) TestDate64ReadWriteTableWithPartialDays() { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(ps.T(), 0) @@ -973,6 +1027,22 @@ func (ps *ParquetIOTestSuite) TestDate64ReadWriteTableWithPartialDays() { ps.Truef(array.TableEqual(date32ExpectedOutputTable, roundTripOutputTable), "expected table: %s\ngot table: %s", date32ExpectedOutputTable, roundTripOutputTable) } +func (ps *ParquetIOTestSuite) TestTimestampTZStoreSchemaReadWriteTable() { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(ps.T(), 0) + + inputTable := makeTimestampTypeTable(mem, false) + defer inputTable.Release() + buf := writeTableToBuffer(ps.T(), mem, inputTable, inputTable.NumRows(), pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem), pqarrow.WithStoreSchema())) + defer buf.Release() + + reader := ps.createReader(mem, buf.Bytes()) + roundTripOutputTable := ps.readTable(reader) + defer roundTripOutputTable.Release() + + ps.Truef(array.TableEqual(inputTable, roundTripOutputTable), "expected table: %s\ngot table: %s", inputTable, roundTripOutputTable) +} + func (ps *ParquetIOTestSuite) TestLargeBinaryReadWriteTable() { mem := memory.NewCheckedAllocator(memory.DefaultAllocator) defer mem.AssertSize(ps.T(), 0) diff --git a/go/parquet/pqarrow/schema.go b/go/parquet/pqarrow/schema.go index 383d47fbaabed..f2aa4cdfe05ad 100644 --- a/go/parquet/pqarrow/schema.go +++ b/go/parquet/pqarrow/schema.go @@ -125,7 +125,7 @@ func isDictionaryReadSupported(dt arrow.DataType) bool { } func arrowTimestampToLogical(typ *arrow.TimestampType, unit arrow.TimeUnit) schema.LogicalType { - utc := typ.TimeZone == "" || typ.TimeZone == "UTC" + isAdjustedToUTC := typ.TimeZone != "" // for forward compatibility reasons, and because there's no other way // to signal to old readers that values are timestamps, we force @@ -146,7 +146,7 @@ func arrowTimestampToLogical(typ *arrow.TimestampType, unit arrow.TimeUnit) sche return schema.NoLogicalType{} } - return schema.NewTimestampLogicalTypeForce(utc, scunit) + return schema.NewTimestampLogicalTypeForce(isAdjustedToUTC, scunit) } func getTimestampMeta(typ *arrow.TimestampType, props *parquet.WriterProperties, arrprops ArrowWriterProperties) (parquet.Type, schema.LogicalType, error) { @@ -519,9 +519,12 @@ func arrowTime64(logical *schema.TimeLogicalType) (arrow.DataType, error) { } func arrowTimestamp(logical *schema.TimestampLogicalType) (arrow.DataType, error) { - tz := "UTC" - if logical.IsFromConvertedType() { - tz = "" + tz := "" + + // ConvertedTypes are adjusted to UTC per backward compatibility guidelines + // https://github.com/apache/parquet-format/blob/eb4b31c1d64a01088d02a2f9aefc6c17c54cc6fc/LogicalTypes.md?plain=1#L480-L485 + if logical.IsAdjustedToUTC() || logical.IsFromConvertedType() { + tz = "UTC" } switch logical.TimeUnit() { diff --git a/go/parquet/pqarrow/schema_test.go b/go/parquet/pqarrow/schema_test.go index a3c2c7a4ff60c..f320b903033db 100644 --- a/go/parquet/pqarrow/schema_test.go +++ b/go/parquet/pqarrow/schema_test.go @@ -304,7 +304,7 @@ func TestCoerceTImestampV1(t *testing.T) { arrowFields := make([]arrow.Field, 0) parquetFields = append(parquetFields, schema.Must(schema.NewPrimitiveNodeLogical("timestamp", parquet.Repetitions.Required, - schema.NewTimestampLogicalTypeForce(false, schema.TimeUnitMicros), parquet.Types.Int64, 0, -1))) + schema.NewTimestampLogicalTypeForce(true, schema.TimeUnitMicros), parquet.Types.Int64, 0, -1))) arrowFields = append(arrowFields, arrow.Field{Name: "timestamp", Type: &arrow.TimestampType{Unit: arrow.Millisecond, TimeZone: "EST"}}) arrowSchema := arrow.NewSchema(arrowFields, nil) @@ -323,11 +323,11 @@ func TestAutoCoerceTImestampV1(t *testing.T) { arrowFields := make([]arrow.Field, 0) parquetFields = append(parquetFields, schema.Must(schema.NewPrimitiveNodeLogical("timestamp", parquet.Repetitions.Required, - schema.NewTimestampLogicalTypeForce(false, schema.TimeUnitMicros), parquet.Types.Int64, 0, -1))) + schema.NewTimestampLogicalTypeForce(true, schema.TimeUnitMicros), parquet.Types.Int64, 0, -1))) arrowFields = append(arrowFields, arrow.Field{Name: "timestamp", Type: &arrow.TimestampType{Unit: arrow.Nanosecond, TimeZone: "EST"}}) parquetFields = append(parquetFields, schema.Must(schema.NewPrimitiveNodeLogical("timestamp[ms]", parquet.Repetitions.Required, - schema.NewTimestampLogicalTypeForce(true, schema.TimeUnitMillis), parquet.Types.Int64, 0, -1))) + schema.NewTimestampLogicalTypeForce(false, schema.TimeUnitMillis), parquet.Types.Int64, 0, -1))) arrowFields = append(arrowFields, arrow.Field{Name: "timestamp[ms]", Type: &arrow.TimestampType{Unit: arrow.Second}}) arrowSchema := arrow.NewSchema(arrowFields, nil) From 55afcf0450aa2b611e78335bdbfd77e55ae3bc9f Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Thu, 18 Jan 2024 15:30:38 -0500 Subject: [PATCH 49/92] GH-39672: [Go] Time to Date32/Date64 conversion issues for non-UTC timezones (#39674) A failing unit test in release verification led to discovering an issue with timestamp to date conversions for non-utc timezones. Upon some investigation I was able to determine that it was the conflation of casting conversion behavior (normalize to cast a Timestamp to a Date) vs flat conversion. I've fixed this conflation of concerns and the version of the methods which are exported properly converts non-UTC timezones to dates without affecting Casting behavior. ### Are these changes tested? yes ### Are there any user-facing changes? The methods `Date32FromTime` and `Date64FromTime` will properly handle timezones now. * Closes: #39672 Authored-by: Matt Topol Signed-off-by: Matt Topol --- go/arrow/compute/internal/kernels/cast_temporal.go | 8 ++++++++ go/arrow/datatype_fixedwidth.go | 10 ---------- go/arrow/datatype_fixedwidth_test.go | 10 ++++++++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/go/arrow/compute/internal/kernels/cast_temporal.go b/go/arrow/compute/internal/kernels/cast_temporal.go index 542a8a4590b28..48e2bfb6cada1 100644 --- a/go/arrow/compute/internal/kernels/cast_temporal.go +++ b/go/arrow/compute/internal/kernels/cast_temporal.go @@ -112,6 +112,10 @@ func TimestampToDate32(ctx *exec.KernelCtx, batch *exec.ExecSpan, out *exec.Exec return ScalarUnaryNotNull(func(_ *exec.KernelCtx, arg0 arrow.Timestamp, _ *error) arrow.Date32 { tm := fnToTime(arg0) + if _, offset := tm.Zone(); offset != 0 { + // normalize the tm + tm = tm.Add(time.Duration(offset) * time.Second).UTC() + } return arrow.Date32FromTime(tm) })(ctx, batch, out) } @@ -125,6 +129,10 @@ func TimestampToDate64(ctx *exec.KernelCtx, batch *exec.ExecSpan, out *exec.Exec return ScalarUnaryNotNull(func(_ *exec.KernelCtx, arg0 arrow.Timestamp, _ *error) arrow.Date64 { tm := fnToTime(arg0) + if _, offset := tm.Zone(); offset != 0 { + // normalize the tm + tm = tm.Add(time.Duration(offset) * time.Second).UTC() + } return arrow.Date64FromTime(tm) })(ctx, batch, out) } diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go index 158dbd67b1b5e..9248dcb8c9a35 100644 --- a/go/arrow/datatype_fixedwidth.go +++ b/go/arrow/datatype_fixedwidth.go @@ -70,11 +70,6 @@ type ( // Date32FromTime returns a Date32 value from a time object func Date32FromTime(t time.Time) Date32 { - if _, offset := t.Zone(); offset != 0 { - // properly account for timezone adjustments before we calculate - // the number of days by adjusting the time and converting to UTC - t = t.Add(time.Duration(offset) * time.Second).UTC() - } return Date32(t.Truncate(24*time.Hour).Unix() / int64((time.Hour * 24).Seconds())) } @@ -88,11 +83,6 @@ func (d Date32) FormattedString() string { // Date64FromTime returns a Date64 value from a time object func Date64FromTime(t time.Time) Date64 { - if _, offset := t.Zone(); offset != 0 { - // properly account for timezone adjustments before we calculate - // the actual value by adjusting the time and converting to UTC - t = t.Add(time.Duration(offset) * time.Second).UTC() - } // truncate to the start of the day to get the correct value t = t.Truncate(24 * time.Hour) return Date64(t.Unix()*1e3 + int64(t.Nanosecond())/1e6) diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go index b3cbb465f3db6..d6caa21e1a255 100644 --- a/go/arrow/datatype_fixedwidth_test.go +++ b/go/arrow/datatype_fixedwidth_test.go @@ -428,3 +428,13 @@ func TestMonthIntervalType(t *testing.T) { t.Fatalf("invalid type stringer: got=%q, want=%q", got, want) } } + +func TestDateFromTime(t *testing.T) { + loc, _ := time.LoadLocation("Asia/Hong_Kong") + tm := time.Date(2024, time.January, 18, 3, 0, 0, 0, loc) + + wantD32 := time.Date(2024, time.January, 17, 0, 0, 0, 0, time.UTC).Truncate(24*time.Hour).Unix() / int64((time.Hour * 24).Seconds()) + wantD64 := time.Date(2024, time.January, 17, 0, 0, 0, 0, time.UTC).UnixMilli() + assert.EqualValues(t, wantD64, arrow.Date64FromTime(tm)) + assert.EqualValues(t, wantD32, arrow.Date32FromTime(tm)) +} From 143a7da1038c3b9b9ad9587f668ca7abcc3520f8 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 19 Jan 2024 11:25:30 -0500 Subject: [PATCH 50/92] GH-39574: [Go] Enable PollFlightInfo in Flight RPC (#39575) ### Rationale for this change It's impossible to use the current bindings with PollFlightInfo. Required for apache/arrow-adbc#1457. ### What changes are included in this PR? Add new methods that expose PollFlightInfo. ### Are these changes tested? Yes ### Are there any user-facing changes? Adds new methods. * Closes: #39574 Authored-by: David Li Signed-off-by: David Li --- go/arrow/flight/flightsql/client.go | 92 ++++++++++++++++++++++++ go/arrow/flight/flightsql/server.go | 54 ++++++++++++++ go/arrow/flight/flightsql/server_test.go | 60 ++++++++++++++++ 3 files changed, 206 insertions(+) diff --git a/go/arrow/flight/flightsql/client.go b/go/arrow/flight/flightsql/client.go index c0c7e2cf20a28..89784b483b01b 100644 --- a/go/arrow/flight/flightsql/client.go +++ b/go/arrow/flight/flightsql/client.go @@ -82,6 +82,17 @@ func flightInfoForCommand(ctx context.Context, cl *Client, cmd proto.Message, op return cl.getFlightInfo(ctx, desc, opts...) } +func pollInfoForCommand(ctx context.Context, cl *Client, cmd proto.Message, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + if retryDescriptor != nil { + return cl.Client.PollFlightInfo(ctx, retryDescriptor, opts...) + } + desc, err := descForCommand(cmd) + if err != nil { + return nil, err + } + return cl.Client.PollFlightInfo(ctx, desc, opts...) +} + func schemaForCommand(ctx context.Context, cl *Client, cmd proto.Message, opts ...grpc.CallOption) (*flight.SchemaResult, error) { desc, err := descForCommand(cmd) if err != nil { @@ -123,6 +134,14 @@ func (c *Client) Execute(ctx context.Context, query string, opts ...grpc.CallOpt return flightInfoForCommand(ctx, c, &cmd, opts...) } +// ExecutePoll idempotently starts execution of a query/checks for completion. +// To check for completion, pass the FlightDescriptor from the previous call +// to ExecutePoll as the retryDescriptor. +func (c *Client) ExecutePoll(ctx context.Context, query string, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + cmd := pb.CommandStatementQuery{Query: query} + return pollInfoForCommand(ctx, c, &cmd, retryDescriptor, opts...) +} + // GetExecuteSchema gets the schema of the result set of a query without // executing the query itself. func (c *Client) GetExecuteSchema(ctx context.Context, query string, opts ...grpc.CallOption) (*flight.SchemaResult, error) { @@ -136,6 +155,12 @@ func (c *Client) ExecuteSubstrait(ctx context.Context, plan SubstraitPlan, opts return flightInfoForCommand(ctx, c, &cmd, opts...) } +func (c *Client) ExecuteSubstraitPoll(ctx context.Context, plan SubstraitPlan, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + cmd := pb.CommandStatementSubstraitPlan{ + Plan: &pb.SubstraitPlan{Plan: plan.Plan, Version: plan.Version}} + return pollInfoForCommand(ctx, c, &cmd, retryDescriptor, opts...) +} + func (c *Client) GetExecuteSubstraitSchema(ctx context.Context, plan SubstraitPlan, opts ...grpc.CallOption) (*flight.SchemaResult, error) { cmd := pb.CommandStatementSubstraitPlan{ Plan: &pb.SubstraitPlan{Plan: plan.Plan, Version: plan.Version}} @@ -606,6 +631,15 @@ func (tx *Txn) Execute(ctx context.Context, query string, opts ...grpc.CallOptio return flightInfoForCommand(ctx, tx.c, cmd, opts...) } +func (tx *Txn) ExecutePoll(ctx context.Context, query string, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + if !tx.txn.IsValid() { + return nil, ErrInvalidTxn + } + // The server should encode the transaction into the retry descriptor + cmd := &pb.CommandStatementQuery{Query: query, TransactionId: tx.txn} + return pollInfoForCommand(ctx, tx.c, cmd, retryDescriptor, opts...) +} + func (tx *Txn) ExecuteSubstrait(ctx context.Context, plan SubstraitPlan, opts ...grpc.CallOption) (*flight.FlightInfo, error) { if !tx.txn.IsValid() { return nil, ErrInvalidTxn @@ -616,6 +650,18 @@ func (tx *Txn) ExecuteSubstrait(ctx context.Context, plan SubstraitPlan, opts .. return flightInfoForCommand(ctx, tx.c, cmd, opts...) } +func (tx *Txn) ExecuteSubstraitPoll(ctx context.Context, plan SubstraitPlan, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + if !tx.txn.IsValid() { + return nil, ErrInvalidTxn + } + // The server should encode the transaction into the retry descriptor + cmd := &pb.CommandStatementSubstraitPlan{ + Plan: &pb.SubstraitPlan{Plan: plan.Plan, Version: plan.Version}, + TransactionId: tx.txn, + } + return pollInfoForCommand(ctx, tx.c, cmd, retryDescriptor, opts...) +} + func (tx *Txn) GetExecuteSchema(ctx context.Context, query string, opts ...grpc.CallOption) (*flight.SchemaResult, error) { if !tx.txn.IsValid() { return nil, ErrInvalidTxn @@ -981,6 +1027,52 @@ func (p *PreparedStatement) Execute(ctx context.Context, opts ...grpc.CallOption return p.client.getFlightInfo(ctx, desc, opts...) } +// ExecutePoll executes the prepared statement on the server and returns a PollInfo +// indicating the progress of execution. +// +// Will error if already closed. +func (p *PreparedStatement) ExecutePoll(ctx context.Context, retryDescriptor *flight.FlightDescriptor, opts ...grpc.CallOption) (*flight.PollInfo, error) { + if p.closed { + return nil, errors.New("arrow/flightsql: prepared statement already closed") + } + + cmd := &pb.CommandPreparedStatementQuery{PreparedStatementHandle: p.handle} + + desc := retryDescriptor + var err error + + if desc == nil { + desc, err = descForCommand(cmd) + if err != nil { + return nil, err + } + } + + if retryDescriptor == nil { + if p.hasBindParameters() { + pstream, err := p.client.Client.DoPut(ctx, opts...) + if err != nil { + return nil, err + } + + wr, err := p.writeBindParameters(pstream, desc) + if err != nil { + return nil, err + } + if err = wr.Close(); err != nil { + return nil, err + } + pstream.CloseSend() + + // wait for the server to ack the result + if _, err = pstream.Recv(); err != nil && err != io.EOF { + return nil, err + } + } + } + return p.client.Client.PollFlightInfo(ctx, desc, opts...) +} + // ExecuteUpdate executes the prepared statement update query on the server // and returns the number of rows affected. If SetParameters was called, // the parameter bindings will be sent with the request to execute. diff --git a/go/arrow/flight/flightsql/server.go b/go/arrow/flight/flightsql/server.go index 5b1764707c298..2ec02e2829962 100644 --- a/go/arrow/flight/flightsql/server.go +++ b/go/arrow/flight/flightsql/server.go @@ -524,6 +524,22 @@ func (BaseServer) RenewFlightEndpoint(context.Context, *flight.RenewFlightEndpoi return nil, status.Error(codes.Unimplemented, "RenewFlightEndpoint not implemented") } +func (BaseServer) PollFlightInfo(context.Context, *flight.FlightDescriptor) (*flight.PollInfo, error) { + return nil, status.Error(codes.Unimplemented, "PollFlightInfo not implemented") +} + +func (BaseServer) PollFlightInfoStatement(context.Context, StatementQuery, *flight.FlightDescriptor) (*flight.PollInfo, error) { + return nil, status.Error(codes.Unimplemented, "PollFlightInfoStatement not implemented") +} + +func (BaseServer) PollFlightInfoSubstraitPlan(context.Context, StatementSubstraitPlan, *flight.FlightDescriptor) (*flight.PollInfo, error) { + return nil, status.Error(codes.Unimplemented, "PollFlightInfoSubstraitPlan not implemented") +} + +func (BaseServer) PollFlightInfoPreparedStatement(context.Context, PreparedStatementQuery, *flight.FlightDescriptor) (*flight.PollInfo, error) { + return nil, status.Error(codes.Unimplemented, "PollFlightInfoPreparedStatement not implemented") +} + func (BaseServer) EndTransaction(context.Context, ActionEndTransactionRequest) error { return status.Error(codes.Unimplemented, "EndTransaction not implemented") } @@ -652,6 +668,14 @@ type Server interface { CancelFlightInfo(context.Context, *flight.CancelFlightInfoRequest) (flight.CancelFlightInfoResult, error) // RenewFlightEndpoint attempts to extend the expiration of a FlightEndpoint RenewFlightEndpoint(context.Context, *flight.RenewFlightEndpointRequest) (*flight.FlightEndpoint, error) + // PollFlightInfo is a generic handler for PollFlightInfo requests. + PollFlightInfo(context.Context, *flight.FlightDescriptor) (*flight.PollInfo, error) + // PollFlightInfoStatement handles polling for query execution. + PollFlightInfoStatement(context.Context, StatementQuery, *flight.FlightDescriptor) (*flight.PollInfo, error) + // PollFlightInfoSubstraitPlan handles polling for query execution. + PollFlightInfoSubstraitPlan(context.Context, StatementSubstraitPlan, *flight.FlightDescriptor) (*flight.PollInfo, error) + // PollFlightInfoPreparedStatement handles polling for query execution. + PollFlightInfoPreparedStatement(context.Context, PreparedStatementQuery, *flight.FlightDescriptor) (*flight.PollInfo, error) mustEmbedBaseServer() } @@ -729,6 +753,36 @@ func (f *flightSqlServer) GetFlightInfo(ctx context.Context, request *flight.Fli return nil, status.Error(codes.InvalidArgument, "requested command is invalid") } +func (f *flightSqlServer) PollFlightInfo(ctx context.Context, request *flight.FlightDescriptor) (*flight.PollInfo, error) { + var ( + anycmd anypb.Any + cmd proto.Message + err error + ) + // If we can't parse things, be friendly and defer to the server + // implementation. This is especially important for this method since + // the server returns a custom FlightDescriptor for future requests. + if err = proto.Unmarshal(request.Cmd, &anycmd); err != nil { + return f.srv.PollFlightInfo(ctx, request) + } + + if cmd, err = anycmd.UnmarshalNew(); err != nil { + return f.srv.PollFlightInfo(ctx, request) + } + + switch cmd := cmd.(type) { + case *pb.CommandStatementQuery: + return f.srv.PollFlightInfoStatement(ctx, cmd, request) + case *pb.CommandStatementSubstraitPlan: + return f.srv.PollFlightInfoSubstraitPlan(ctx, &statementSubstraitPlan{cmd}, request) + case *pb.CommandPreparedStatementQuery: + return f.srv.PollFlightInfoPreparedStatement(ctx, cmd, request) + } + // XXX: for now we won't support the other methods + + return f.srv.PollFlightInfo(ctx, request) +} + func (f *flightSqlServer) GetSchema(ctx context.Context, request *flight.FlightDescriptor) (*flight.SchemaResult, error) { var ( anycmd anypb.Any diff --git a/go/arrow/flight/flightsql/server_test.go b/go/arrow/flight/flightsql/server_test.go index e444da4aaf4a2..956a1714c671c 100644 --- a/go/arrow/flight/flightsql/server_test.go +++ b/go/arrow/flight/flightsql/server_test.go @@ -56,6 +56,36 @@ func (*testServer) GetFlightInfoStatement(ctx context.Context, q flightsql.State }, nil } +func (*testServer) PollFlightInfo(ctx context.Context, fd *flight.FlightDescriptor) (*flight.PollInfo, error) { + return &flight.PollInfo{ + Info: &flight.FlightInfo{ + FlightDescriptor: fd, + Endpoint: []*flight.FlightEndpoint{{ + Ticket: &flight.Ticket{Ticket: []byte{}}, + }, { + Ticket: &flight.Ticket{Ticket: []byte{}}, + }}, + }, + FlightDescriptor: nil, + }, nil +} + +func (*testServer) PollFlightInfoStatement(ctx context.Context, q flightsql.StatementQuery, fd *flight.FlightDescriptor) (*flight.PollInfo, error) { + ticket, err := flightsql.CreateStatementQueryTicket([]byte(q.GetQuery())) + if err != nil { + return nil, err + } + return &flight.PollInfo{ + Info: &flight.FlightInfo{ + FlightDescriptor: fd, + Endpoint: []*flight.FlightEndpoint{{ + Ticket: &flight.Ticket{Ticket: ticket}, + }}, + }, + FlightDescriptor: &flight.FlightDescriptor{Cmd: []byte{}}, + }, nil +} + func (*testServer) DoGetStatement(ctx context.Context, ticket flightsql.StatementQueryTicket) (sc *arrow.Schema, cc <-chan flight.StreamChunk, err error) { handle := string(ticket.GetStatementHandle()) switch handle { @@ -189,6 +219,20 @@ func (s *FlightSqlServerSuite) TestExecuteChunkError() { } } +func (s *FlightSqlServerSuite) TestExecutePoll() { + poll, err := s.cl.ExecutePoll(context.TODO(), "1", nil) + s.NoError(err) + s.NotNil(poll) + s.NotNil(poll.GetFlightDescriptor()) + s.Len(poll.GetInfo().Endpoint, 1) + + poll, err = s.cl.ExecutePoll(context.TODO(), "1", poll.GetFlightDescriptor()) + s.NoError(err) + s.NotNil(poll) + s.Nil(poll.GetFlightDescriptor()) + s.Len(poll.GetInfo().Endpoint, 2) +} + type UnimplementedFlightSqlServerSuite struct { suite.Suite @@ -314,6 +358,22 @@ func (s *UnimplementedFlightSqlServerSuite) TestGetTypeInfo() { s.Nil(info) } +func (s *UnimplementedFlightSqlServerSuite) TestPoll() { + poll, err := s.cl.ExecutePoll(context.TODO(), "", nil) + st, ok := status.FromError(err) + s.True(ok) + s.Equal(codes.Unimplemented, st.Code()) + s.Equal("PollFlightInfoStatement not implemented", st.Message()) + s.Nil(poll) + + poll, err = s.cl.ExecuteSubstraitPoll(context.TODO(), flightsql.SubstraitPlan{}, nil) + st, ok = status.FromError(err) + s.True(ok) + s.Equal(codes.Unimplemented, st.Code()) + s.Equal("PollFlightInfoSubstraitPlan not implemented", st.Message()) + s.Nil(poll) +} + func getTicket(cmd proto.Message) *flight.Ticket { var anycmd anypb.Any anycmd.MarshalFrom(cmd) From 92682f0f6064224acd6dd746ac45d6df4b1963c4 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 19 Jan 2024 14:56:16 -0800 Subject: [PATCH 51/92] GH-39001: [Java] Modularize remaining modules (#39221) ### Rationale for this change Modularize remaining modules outside of memory modules, vector, and format. ### What changes are included in this PR? ### Are these changes tested? Yes, existing unit tests now run with modules when using JDK9+. ### Are there any user-facing changes? Yes. There are new command-line options that may be necessary. The way of specifying the output directory for JNI native library builds differs. The flight-grpc module has been eliminated since it is now built into flight-core. Documentation has been updated for these changes. **This PR includes breaking changes to public APIs.** There are a number of package structure changes and some modules now need additional command-line arguments. * Closes: #39001 Authored-by: James Duong Signed-off-by: David Li --- ci/scripts/integration_arrow_build.sh | 2 +- ci/scripts/java_jni_build.sh | 2 - ci/scripts/java_jni_macos_build.sh | 11 +- ci/scripts/java_jni_manylinux_build.sh | 11 +- ci/scripts/java_jni_windows_build.sh | 3 +- .../archery/integration/tester_java.py | 2 + dev/tasks/java-jars/github.yml | 38 +++--- dev/tasks/tasks.yml | 7 - docker-compose.yml | 2 +- docs/source/developers/java/building.rst | 54 ++++---- docs/source/java/install.rst | 35 ++++- docs/source/java/overview.rst | 3 - java/CMakeLists.txt | 17 +++ java/adapter/avro/pom.xml | 5 + .../avro/src/main/java/module-info.java | 26 ++++ .../arrow/{ => adapter/avro}/AvroToArrow.java | 2 +- .../{ => adapter/avro}/AvroToArrowConfig.java | 2 +- .../avro}/AvroToArrowConfigBuilder.java | 2 +- .../{ => adapter/avro}/AvroToArrowUtils.java | 50 +++---- .../avro}/AvroToArrowVectorIterator.java | 4 +- .../avro}/consumers/AvroArraysConsumer.java | 2 +- .../avro}/consumers/AvroBooleanConsumer.java | 2 +- .../avro}/consumers/AvroBytesConsumer.java | 2 +- .../avro}/consumers/AvroDoubleConsumer.java | 2 +- .../avro}/consumers/AvroEnumConsumer.java | 2 +- .../avro}/consumers/AvroFixedConsumer.java | 2 +- .../avro}/consumers/AvroFloatConsumer.java | 2 +- .../avro}/consumers/AvroIntConsumer.java | 2 +- .../avro}/consumers/AvroLongConsumer.java | 2 +- .../avro}/consumers/AvroMapConsumer.java | 2 +- .../avro}/consumers/AvroNullConsumer.java | 2 +- .../avro}/consumers/AvroStringConsumer.java | 2 +- .../avro}/consumers/AvroStructConsumer.java | 2 +- .../avro}/consumers/AvroUnionsConsumer.java | 2 +- .../avro}/consumers/BaseAvroConsumer.java | 2 +- .../consumers/CompositeAvroConsumer.java | 2 +- .../avro}/consumers/Consumer.java | 2 +- .../avro}/consumers/SkipConsumer.java | 2 +- .../avro}/consumers/SkipFunction.java | 2 +- .../consumers/logical/AvroDateConsumer.java | 4 +- .../logical/AvroDecimalConsumer.java | 4 +- .../logical/AvroTimeMicroConsumer.java | 4 +- .../logical/AvroTimeMillisConsumer.java | 4 +- .../logical/AvroTimestampMicrosConsumer.java | 4 +- .../logical/AvroTimestampMillisConsumer.java | 4 +- .../avro}/AvroLogicalTypesTest.java | 2 +- .../{ => adapter/avro}/AvroSkipFieldTest.java | 2 +- .../{ => adapter/avro}/AvroTestBase.java | 28 ++-- .../avro}/AvroToArrowIteratorTest.java | 2 +- .../{ => adapter/avro}/AvroToArrowTest.java | 2 +- .../avro}/TestWriteReadAvroRecord.java | 9 +- java/adapter/jdbc/pom.xml | 28 ++++ .../jdbc/src/main/java/module-info.java | 28 ++++ java/adapter/orc/CMakeLists.txt | 9 +- java/adapter/orc/pom.xml | 4 + .../orc/src/main/java/module-info.java | 24 ++++ .../apache/arrow/adapter/orc/OrcJniUtils.java | 2 +- java/algorithm/pom.xml | 4 + java/algorithm/src/main/java/module-info.java | 29 +++++ java/bom/pom.xml | 5 - java/c/CMakeLists.txt | 9 +- java/c/pom.xml | 4 + java/c/src/main/java/module-info.java | 27 ++++ .../main/java/org/apache/arrow/c/Data.java | 2 - .../{vector => c}/StructVectorLoader.java | 4 +- .../{vector => c}/StructVectorUnloader.java | 4 +- .../org/apache/arrow/c/jni/JniLoader.java | 2 +- java/compression/pom.xml | 4 + .../src/main/java/module-info.java | 25 ++++ java/dataset/CMakeLists.txt | 10 +- java/dataset/pom.xml | 39 ++++++ java/dataset/src/main/java/module-info.java | 29 +++++ .../jni/DirectReservationListener.java | 7 +- .../apache/arrow/dataset/jni/JniLoader.java | 2 +- .../arrow/dataset/ParquetWriteSupport.java | 27 +++- .../apache/arrow/dataset/TestAllTypes.java | 2 +- java/flight/flight-core/pom.xml | 37 ++++++ .../src/main/java/module-info.java | 42 ++++++ .../apache/arrow/flight/FlightGrpcUtils.java | 0 .../apache/arrow/flight/FlightTestUtil.java | 2 +- .../arrow/flight/TestFlightGrpcUtils.java | 0 .../src/test/protobuf/test.proto | 0 java/flight/flight-grpc/pom.xml | 123 ------------------ .../src/test/resources/logback.xml | 28 ---- java/flight/flight-sql/pom.xml | 27 ++++ .../flight-sql/src/main/java/module-info.java | 29 +++++ .../flight/{ => sql/test}/TestFlightSql.java | 13 +- .../{ => sql/test}/TestFlightSqlStreams.java | 11 +- java/flight/pom.xml | 1 - java/gandiva/CMakeLists.txt | 9 +- java/gandiva/pom.xml | 4 + java/gandiva/src/main/java/module-info.java | 29 +++++ .../arrow/gandiva/evaluator/JniLoader.java | 2 +- .../arrow/adapter/AvroAdapterBenchmarks.java | 8 +- java/pom.xml | 5 - java/tools/pom.xml | 4 + java/tools/src/main/java/module-info.java | 27 ++++ .../{ => test}/util/ArrowTestDataUtil.java | 2 +- 98 files changed, 763 insertions(+), 355 deletions(-) create mode 100644 java/adapter/avro/src/main/java/module-info.java rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrow.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowConfig.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowConfigBuilder.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowUtils.java (95%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowVectorIterator.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroArraysConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroBooleanConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroBytesConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroDoubleConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroEnumConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroFixedConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroFloatConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroIntConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroLongConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroMapConsumer.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroNullConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroStringConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroStructConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroUnionsConsumer.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/BaseAvroConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/CompositeAvroConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/Consumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/SkipConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/SkipFunction.java (95%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroDateConsumer.java (91%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroDecimalConsumer.java (95%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroTimeMicroConsumer.java (91%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroTimeMillisConsumer.java (91%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroTimestampMicrosConsumer.java (92%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroTimestampMillisConsumer.java (92%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroLogicalTypesTest.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroSkipFieldTest.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroTestBase.java (88%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowIteratorTest.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowTest.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/TestWriteReadAvroRecord.java (91%) create mode 100644 java/adapter/jdbc/src/main/java/module-info.java create mode 100644 java/adapter/orc/src/main/java/module-info.java create mode 100644 java/algorithm/src/main/java/module-info.java create mode 100644 java/c/src/main/java/module-info.java rename java/c/src/main/java/org/apache/arrow/{vector => c}/StructVectorLoader.java (98%) rename java/c/src/main/java/org/apache/arrow/{vector => c}/StructVectorUnloader.java (97%) create mode 100644 java/compression/src/main/java/module-info.java create mode 100644 java/dataset/src/main/java/module-info.java create mode 100644 java/flight/flight-core/src/main/java/module-info.java rename java/flight/{flight-grpc => flight-core}/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java (100%) rename java/flight/{flight-grpc => flight-core}/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java (100%) rename java/flight/{flight-grpc => flight-core}/src/test/protobuf/test.proto (100%) delete mode 100644 java/flight/flight-grpc/pom.xml delete mode 100644 java/flight/flight-grpc/src/test/resources/logback.xml create mode 100644 java/flight/flight-sql/src/main/java/module-info.java rename java/flight/flight-sql/src/test/java/org/apache/arrow/flight/{ => sql/test}/TestFlightSql.java (98%) rename java/flight/flight-sql/src/test/java/org/apache/arrow/flight/{ => sql/test}/TestFlightSqlStreams.java (96%) create mode 100644 java/gandiva/src/main/java/module-info.java create mode 100644 java/tools/src/main/java/module-info.java rename java/vector/src/test/java/org/apache/arrow/vector/{ => test}/util/ArrowTestDataUtil.java (97%) diff --git a/ci/scripts/integration_arrow_build.sh b/ci/scripts/integration_arrow_build.sh index 02f593bf77b23..e5c31527aedff 100755 --- a/ci/scripts/integration_arrow_build.sh +++ b/ci/scripts/integration_arrow_build.sh @@ -46,7 +46,7 @@ if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then export ARROW_JAVA_CDATA="ON" export JAVA_JNI_CMAKE_ARGS="-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DARROW_JAVA_JNI_ENABLE_C=ON" - ${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java/$(arch) + ${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java ${arrow_dir}/ci/scripts/java_build.sh ${arrow_dir} ${build_dir} /tmp/dist/java fi diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh index 320c98c04df1e..d989351ab7e4d 100755 --- a/ci/scripts/java_jni_build.sh +++ b/ci/scripts/java_jni_build.sh @@ -24,7 +24,6 @@ arrow_install_dir=${2} build_dir=${3}/java_jni # The directory where the final binaries will be stored when scripts finish dist_dir=${4} - prefix_dir="${build_dir}/java-jni" echo "=== Clear output directories and leftovers ===" @@ -56,7 +55,6 @@ cmake \ -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ - -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ -DProtobuf_USE_STATIC_LIBS=ON \ diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index d66c39a37c5bd..4ecc029bdd3c2 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -31,7 +31,7 @@ case ${normalized_arch} in ;; esac # The directory where the final binaries will be stored when scripts finish -dist_dir=${3}/${normalized_arch} +dist_dir=${3} echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers @@ -82,7 +82,6 @@ cmake \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DGTest_SOURCE=BUNDLED \ @@ -138,8 +137,8 @@ archery linking check-dependencies \ --allow libncurses \ --allow libobjc \ --allow libz \ - libarrow_cdata_jni.dylib \ - libarrow_dataset_jni.dylib \ - libarrow_orc_jni.dylib \ - libgandiva_jni.dylib + arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.dylib \ + arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.dylib \ + arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib \ + gandiva_jni/${normalized_arch}/libgandiva_jni.dylib popd diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 03939715e390f..da4987d307ce4 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -28,7 +28,7 @@ case ${normalized_arch} in ;; esac # The directory where the final binaries will be stored when scripts finish -dist_dir=${3}/${normalized_arch} +dist_dir=${3} echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers @@ -91,7 +91,6 @@ cmake \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DGTest_SOURCE=BUNDLED \ @@ -164,8 +163,8 @@ archery linking check-dependencies \ --allow libstdc++ \ --allow libz \ --allow linux-vdso \ - libarrow_cdata_jni.so \ - libarrow_dataset_jni.so \ - libarrow_orc_jni.so \ - libgandiva_jni.so + arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.so \ + arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.so \ + arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.so \ + gandiva_jni/${normalized_arch}/libgandiva_jni.so popd diff --git a/ci/scripts/java_jni_windows_build.sh b/ci/scripts/java_jni_windows_build.sh index 778ee9696790e..39288f4a9d0ce 100755 --- a/ci/scripts/java_jni_windows_build.sh +++ b/ci/scripts/java_jni_windows_build.sh @@ -22,7 +22,7 @@ set -ex arrow_dir=${1} build_dir=${2} # The directory where the final binaries will be stored when scripts finish -dist_dir=${3}/x86_64 +dist_dir=${3} echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers @@ -72,7 +72,6 @@ cmake \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZSTD=ON \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -GNinja \ diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index 857fe0c50af06..8e7a0bb99f9de 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -259,6 +259,8 @@ def __init__(self, *args, **kwargs): self._java_opts.append( '--add-opens=java.base/java.nio=' 'org.apache.arrow.memory.core,ALL-UNNAMED') + self._java_opts.append( + '--add-reads=org.apache.arrow.flight.core=ALL-UNNAMED') def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): cmd = ( diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 7ee68e77ee637..086d1fdbe811f 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -208,29 +208,29 @@ jobs: run: | set -x - test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so - test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so - test -f arrow/java-dist/x86_64/libarrow_orc_jni.so - test -f arrow/java-dist/x86_64/libgandiva_jni.so + test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so + test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so + test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so + test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so - test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.so - test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.so - test -f arrow/java-dist/aarch_64/libarrow_orc_jni.so - test -f arrow/java-dist/aarch_64/libgandiva_jni.so + test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so + test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so + test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so + test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so - test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib - test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib - test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib - test -f arrow/java-dist/x86_64/libgandiva_jni.dylib + test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib + test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib + test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib + test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib - test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.dylib - test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.dylib - test -f arrow/java-dist/aarch_64/libarrow_orc_jni.dylib - test -f arrow/java-dist/aarch_64/libgandiva_jni.dylib + test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib + test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib + test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib + test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib - test -f arrow/java-dist/x86_64/arrow_cdata_jni.dll - test -f arrow/java-dist/x86_64/arrow_dataset_jni.dll - test -f arrow/java-dist/x86_64/arrow_orc_jni.dll + test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll + test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll + test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Build bundled jar run: | set -e diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 04faef427e281..ca45d48bcd470 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -810,13 +810,6 @@ tasks: - flight-core-{no_rc_snapshot_version}-tests.jar - flight-core-{no_rc_snapshot_version}.jar - flight-core-{no_rc_snapshot_version}.pom - - flight-grpc-{no_rc_snapshot_version}-cyclonedx.json - - flight-grpc-{no_rc_snapshot_version}-cyclonedx.xml - - flight-grpc-{no_rc_snapshot_version}-javadoc.jar - - flight-grpc-{no_rc_snapshot_version}-sources.jar - - flight-grpc-{no_rc_snapshot_version}-tests.jar - - flight-grpc-{no_rc_snapshot_version}.jar - - flight-grpc-{no_rc_snapshot_version}.pom - flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.json - flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.xml - flight-integration-tests-{no_rc_snapshot_version}-jar-with-dependencies.jar diff --git a/docker-compose.yml b/docker-compose.yml index 14eff67f38971..a08345c198fa0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1343,7 +1343,7 @@ services: command: [ "/arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) && + /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/ && /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java && /arrow/ci/scripts/java_cdata_integration.sh /arrow /tmp/dist/java" ] diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 0e831915e09b9..27e2de97328c3 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -115,9 +115,8 @@ Maven $ export JAVA_HOME= $ java --version $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N - $ ls -latr ../java-dist/lib/ - |__ libarrow_cdata_jni.dylib - |__ libarrow_cdata_jni.so + $ ls -latr ../java-dist/lib + |__ arrow_cdata_jni/ - To build only the JNI C Data Interface library (Windows): @@ -125,8 +124,8 @@ Maven $ cd arrow/java $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N - $ dir "../java-dist/bin/x86_64" - |__ arrow_cdata_jni.dll + $ dir "../java-dist/bin" + |__ arrow_cdata_jni/ - To build all JNI libraries (macOS / Linux) except the JNI C Data Interface library: @@ -136,10 +135,10 @@ Maven $ export JAVA_HOME= $ java --version $ mvn generate-resources -Pgenerate-libs-jni-macos-linux -N - $ ls -latr java-dist/lib//*_{jni,java}.* - |__ libarrow_dataset_jni.dylib - |__ libarrow_orc_jni.dylib - |__ libgandiva_jni.dylib + $ ls -latr java-dist/lib + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ - To build all JNI libraries (Windows) except the JNI C Data Interface library: @@ -147,8 +146,8 @@ Maven $ cd arrow/java $ mvn generate-resources -Pgenerate-libs-jni-windows -N - $ dir "../java-dist/bin/x86_64" - |__ arrow_dataset_jni.dll + $ dir "../java-dist/bin" + |__ arrow_dataset_jni/ CMake ~~~~~ @@ -166,12 +165,10 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR=lib/ \ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ ls -latr java-dist/lib - |__ libarrow_cdata_jni.dylib - |__ libarrow_cdata_jni.so + |__ arrow_cdata_jni/ - To build only the JNI C Data Interface library (Windows): @@ -186,11 +183,10 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^ -DBUILD_TESTING=OFF ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ dir "java-dist/bin" - |__ arrow_cdata_jni.dll + |__ arrow_cdata_jni/ - To build all JNI libraries (macOS / Linux) except the JNI C Data Interface library: @@ -222,7 +218,6 @@ CMake -DARROW_SUBSTRAIT=ON \ -DARROW_USE_CCACHE=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR=lib/ \ -DCMAKE_INSTALL_PREFIX=java-dist \ -DCMAKE_UNITY_BUILD=ON $ cmake --build cpp-jni --target install --config Release @@ -233,16 +228,15 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR=lib/ \ -DCMAKE_INSTALL_PREFIX=java-dist \ -DCMAKE_PREFIX_PATH=$PWD/java-dist \ -DProtobuf_ROOT=$PWD/../cpp-jni/protobuf_ep-install \ -DProtobuf_USE_STATIC_LIBS=ON $ cmake --build java-jni --target install --config Release - $ ls -latr java-dist/lib//*_{jni,java}.* - |__ libarrow_dataset_jni.dylib - |__ libarrow_orc_jni.dylib - |__ libgandiva_jni.dylib + $ ls -latr java-dist/lib/ + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ - To build all JNI libraries (Windows) except the JNI C Data Interface library: @@ -271,7 +265,6 @@ CMake -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist ^ -DCMAKE_UNITY_BUILD=ON ^ -GNinja @@ -288,13 +281,12 @@ CMake -DARROW_JAVA_JNI_ENABLE_ORC=ON ^ -DBUILD_TESTING=OFF ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist ^ -DCMAKE_PREFIX_PATH=$PWD/java-dist $ cmake --build java-jni --target install --config Release $ dir "java-dist/bin" - |__ arrow_orc_jni.dll - |__ arrow_dataset_jni.dll + |__ arrow_orc_jni/ + |__ arrow_dataset_jni/ Archery ~~~~~~~ @@ -303,11 +295,11 @@ Archery $ cd arrow $ archery docker run java-jni-manylinux-2014 - $ ls -latr java-dist// - |__ libarrow_cdata_jni.so - |__ libarrow_dataset_jni.so - |__ libarrow_orc_jni.so - |__ libgandiva_jni.so + $ ls -latr java-dist + |__ arrow_cdata_jni/ + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ Building Java JNI Modules ------------------------- diff --git a/docs/source/java/install.rst b/docs/source/java/install.rst index b7484536f2367..783687fb1f773 100644 --- a/docs/source/java/install.rst +++ b/docs/source/java/install.rst @@ -43,7 +43,40 @@ adding ``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED $ env _JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... Otherwise, you may see errors like ``module java.base does not "opens -java.nio" to unnamed module``. +java.nio" to unnamed module`` or ``module java.base does not "opens +java.nio" to org.apache.arrow.memory.core`` + +Note that the command has changed from Arrow 15 and earlier. If you are still using the flags from that version +(``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED``) you will see the +``module java.base does not "opens java.nio" to org.apache.arrow.memory.core`` error. + +If you are using flight-core or dependent modules, you will need to mark that flight-core can read unnamed modules. +Modifying the command above for Flight: + +.. code-block:: shell + + # Directly on the command line + $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ... + # Indirectly via environment variables + $ env _JAVA_OPTIONS="--add-reads=org.apache.arrow.flight.core=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... + +Otherwise, you may see errors like ``java.lang.IllegalAccessError: superclass access check failed: class +org.apache.arrow.flight.ArrowMessage$ArrowBufRetainingCompositeByteBuf (in module org.apache.arrow.flight.core) +cannot access class io.netty.buffer.CompositeByteBuf (in unnamed module ...) because module +org.apache.arrow.flight.core does not read unnamed module ... + +Finally, if you are using arrow-dataset, you'll also need to report that JDK internals need to be exposed. +Modifying the command above for arrow-memory: +.. code-block:: shell + + # Directly on the command line + $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ... + # Indirectly via environment variables + $ env _JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.dataset,org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... + +Otherwise you may see errors such as ``java.lang.RuntimeException: java.lang.reflect.InaccessibleObjectException: +Unable to make static void java.nio.Bits.reserveMemory(long,long) accessible: module +java.base does not "opens java.nio" to module org.apache.arrow.dataset`` If using Maven and Surefire for unit testing, :ref:`this argument must be added to Surefire as well `. diff --git a/docs/source/java/overview.rst b/docs/source/java/overview.rst index 4b30b8e000f12..9d9cbad8a26c1 100644 --- a/docs/source/java/overview.rst +++ b/docs/source/java/overview.rst @@ -56,9 +56,6 @@ but some modules are JNI bindings to the C++ library. * - flight-core - (Experimental) An RPC mechanism for transferring ValueVectors. - Native - * - flight-grpc - - (Experimental) Contains utility class to expose Flight gRPC service and client. - - Native * - flight-sql - (Experimental) Contains utility classes to expose Flight SQL semantics for clients and servers over Arrow Flight. - Native diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 04fa51ff98ca0..8b29f37d80a1b 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -74,6 +74,23 @@ if(BUILD_TESTING) GTest::gtest_main) endif() +# The ARROW_JAVA_JNI_ARCH_DIR will automatically be derived the normalized +# operating system from system processor. The user can override this variable +# if auto-detection fails. +if("${ARROW_JAVA_JNI_ARCH_DIR}" STREQUAL "") + if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(ARROW_JAVA_JNI_ARCH_DIR "aarch_64") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "i386") + set(ARROW_JAVA_JNI_ARCH_DIR "x86_64") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm64") + set(ARROW_JAVA_JNI_ARCH_DIR "aarch_64") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "AMD64") + set(ARROW_JAVA_JNI_ARCH_DIR "x86_64") + else() + set(ARROW_JAVA_JNI_ARCH_DIR "${CMAKE_SYSTEM_PROCESSOR}") + endif() +endif() + if(ARROW_JAVA_JNI_ENABLE_C) add_subdirectory(c) endif() diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index c0410ea4c2314..90864eab006a2 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -46,6 +46,11 @@ arrow-vector + + org.immutables + value + + org.apache.avro avro diff --git a/java/adapter/avro/src/main/java/module-info.java b/java/adapter/avro/src/main/java/module-info.java new file mode 100644 index 0000000000000..5c6204be60e9c --- /dev/null +++ b/java/adapter/avro/src/main/java/module-info.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.adapter.avro { + exports org.apache.arrow.adapter.avro.consumers; + exports org.apache.arrow.adapter.avro.consumers.logical; + exports org.apache.arrow.adapter.avro; + + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires org.apache.avro; +} diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrow.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrow.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java index 33f180393780e..8baa60a72ddc3 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrow.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfig.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfig.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java index 4f59ef3843dda..f9210fb012523 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfig.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import java.util.Set; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfigBuilder.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfigBuilder.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfigBuilder.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfigBuilder.java index 474c1eb5ca7c2..41e486d0a1ce0 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfigBuilder.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfigBuilder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import java.util.HashSet; import java.util.Set; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java similarity index 95% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java index 80293c8b85c8b..1f5ad9e768950 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -31,30 +31,30 @@ import java.util.Set; import java.util.stream.Collectors; -import org.apache.arrow.consumers.AvroArraysConsumer; -import org.apache.arrow.consumers.AvroBooleanConsumer; -import org.apache.arrow.consumers.AvroBytesConsumer; -import org.apache.arrow.consumers.AvroDoubleConsumer; -import org.apache.arrow.consumers.AvroEnumConsumer; -import org.apache.arrow.consumers.AvroFixedConsumer; -import org.apache.arrow.consumers.AvroFloatConsumer; -import org.apache.arrow.consumers.AvroIntConsumer; -import org.apache.arrow.consumers.AvroLongConsumer; -import org.apache.arrow.consumers.AvroMapConsumer; -import org.apache.arrow.consumers.AvroNullConsumer; -import org.apache.arrow.consumers.AvroStringConsumer; -import org.apache.arrow.consumers.AvroStructConsumer; -import org.apache.arrow.consumers.AvroUnionsConsumer; -import org.apache.arrow.consumers.CompositeAvroConsumer; -import org.apache.arrow.consumers.Consumer; -import org.apache.arrow.consumers.SkipConsumer; -import org.apache.arrow.consumers.SkipFunction; -import org.apache.arrow.consumers.logical.AvroDateConsumer; -import org.apache.arrow.consumers.logical.AvroDecimalConsumer; -import org.apache.arrow.consumers.logical.AvroTimeMicroConsumer; -import org.apache.arrow.consumers.logical.AvroTimeMillisConsumer; -import org.apache.arrow.consumers.logical.AvroTimestampMicrosConsumer; -import org.apache.arrow.consumers.logical.AvroTimestampMillisConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroArraysConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroBooleanConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroBytesConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroDoubleConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroEnumConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroFixedConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroFloatConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroIntConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroLongConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroMapConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroNullConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroStringConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroStructConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroUnionsConsumer; +import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.Consumer; +import org.apache.arrow.adapter.avro.consumers.SkipConsumer; +import org.apache.arrow.adapter.avro.consumers.SkipFunction; +import org.apache.arrow.adapter.avro.consumers.logical.AvroDateConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroDecimalConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimeMicroConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimeMillisConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMicrosConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMillisConsumer; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseIntVector; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowVectorIterator.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowVectorIterator.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java index 1faa7595c68b0..4a439ade81181 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowVectorIterator.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import java.io.EOFException; import java.util.ArrayList; @@ -23,7 +23,7 @@ import java.util.List; import java.util.stream.Collectors; -import org.apache.arrow.consumers.CompositeAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroArraysConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroArraysConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java index b9d0f84cfde4e..fd25986c32b95 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroArraysConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBooleanConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBooleanConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java index 4ca5f24451f90..bf41828d19f7a 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBooleanConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBytesConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBytesConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java index eede68ebd39dc..c8370e480608d 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBytesConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroDoubleConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroDoubleConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java index 356707a140b0a..7cc7dd33b15a9 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroDoubleConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroEnumConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroEnumConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java index 2f4443b741785..32a2c85f6fc50 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroEnumConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFixedConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFixedConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java index a065466e395cf..16b70898fd36a 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFixedConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFloatConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFloatConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java index c8de4a21af448..b09d2881875b6 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFloatConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroIntConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroIntConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java index bc8d4de78abc3..ae5a2719c5642 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroIntConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroLongConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroLongConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java index b9016c58f05c8..4db836acc4586 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroLongConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroMapConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroMapConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java index b8e8bd585eef3..1ea97e63b61e5 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroMapConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroNullConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroNullConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java index 64768008a9571..4c7bb8c03bad3 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroNullConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStringConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStringConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java index 10fe234ac6cb8..072270aa6c081 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStringConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStructConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStructConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java index 792d01ee502c3..a02b1577f9fa8 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStructConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroUnionsConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroUnionsConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java index c0bb0200fcd71..76287543b0646 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroUnionsConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/BaseAvroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/BaseAvroConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java index 303be8e504f54..66a6cda68401e 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/BaseAvroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import org.apache.arrow.vector.FieldVector; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/CompositeAvroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/CompositeAvroConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java index af476d27cb8f7..97812226180ac 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/CompositeAvroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.util.List; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/Consumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/Consumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java index 8c4ee9a96e004..c2ae1ce77b282 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/Consumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java index 94c5b339d87f3..1ac0a6d71557b 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipFunction.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java similarity index 95% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipFunction.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java index 61938916a5eca..93fc4a7fede3f 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipFunction.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDateConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java similarity index 91% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDateConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java index 3aa8970d9a97f..a5c36d88fb76a 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDateConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.DateDayVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDecimalConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java similarity index 95% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDecimalConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java index 24d73cf829ffc..ebe5ca3884e5e 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDecimalConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.DecimalVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMicroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java similarity index 91% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMicroConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java index e68ba158ffaf3..89216d4ad1436 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMicroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeMicroVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMillisConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java similarity index 91% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMillisConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java index f76186fc3785a..ab5df8d4bc8ac 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMillisConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeMilliVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMicrosConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java similarity index 92% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMicrosConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java index 82da0e8054b50..93b39d479ff0e 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMicrosConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeStampMicroVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMillisConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java similarity index 92% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMillisConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java index 159f49e1480a6..9e651c3959f81 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMillisConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeStampMilliVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java index 050a50ddaeae4..6ee04e33a5ce1 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static junit.framework.TestCase.assertNull; import static junit.framework.TestCase.assertTrue; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java index b946dbd8653fd..a37eca6514e04 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java similarity index 88% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java index 16d8e52722c44..60a3a285db3aa 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -24,9 +24,9 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.InputStream; +import java.lang.reflect.Method; import java.nio.ByteBuffer; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -62,11 +62,23 @@ public void init() { config = new AvroToArrowConfigBuilder(allocator).build(); } - protected Schema getSchema(String schemaName) throws Exception { - Path schemaPath = Paths.get(Paths.get(TestWriteReadAvroRecord.class.getResource("/").toURI()).toString(), - "schema", schemaName); - - return new Schema.Parser().parse(schemaPath.toFile()); + public static Schema getSchema(String schemaName) throws Exception { + try { + // Attempt to use JDK 9 behavior of getting the module then the resource stream from the module. + // Note that this code is caller-sensitive. + Method getModuleMethod = Class.class.getMethod("getModule"); + Object module = getModuleMethod.invoke(TestWriteReadAvroRecord.class); + Method getResourceAsStreamFromModule = module.getClass().getMethod("getResourceAsStream", String.class); + try (InputStream is = (InputStream) getResourceAsStreamFromModule.invoke(module, "/schema/" + schemaName)) { + return new Schema.Parser() + .parse(is); + } + } catch (NoSuchMethodException ex) { + // Use JDK8 behavior. + try (InputStream is = TestWriteReadAvroRecord.class.getResourceAsStream("/schema/" + schemaName)) { + return new Schema.Parser().parse(is); + } + } } protected VectorSchemaRoot writeAndRead(Schema schema, List data) throws Exception { diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java index 2b05a19f38067..02f7a3733734c 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java index c007e1ac7ebd2..1c64204191762 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java similarity index 91% rename from java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java index 0a153a28cbc2e..afbddaa6ed87a 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java @@ -15,13 +15,11 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -48,10 +46,7 @@ public class TestWriteReadAvroRecord { public void testWriteAndRead() throws Exception { File dataFile = TMP.newFile(); - Path schemaPath = Paths.get( - Paths.get(TestWriteReadAvroRecord.class.getResource("/").toURI()).toString(), - "schema", "test.avsc"); - Schema schema = new Schema.Parser().parse(schemaPath.toFile()); + Schema schema = AvroTestBase.getSchema("test.avsc"); //write data to disk GenericRecord user1 = new GenericData.Record(schema); diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index f95956d1f61d5..e964aa1871a0e 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -47,6 +47,11 @@ ${arrow.vector.classifier} + + org.immutables + value + + com.h2database @@ -85,4 +90,27 @@ + + + jdk11+ + + [11,] + + !m2e.version + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-reads=org.apache.arrow.adapter.jdbc=com.fasterxml.jackson.dataformat.yaml --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + + + diff --git a/java/adapter/jdbc/src/main/java/module-info.java b/java/adapter/jdbc/src/main/java/module-info.java new file mode 100644 index 0000000000000..5b59ce768472a --- /dev/null +++ b/java/adapter/jdbc/src/main/java/module-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.adapter.jdbc { + exports org.apache.arrow.adapter.jdbc.consumer; + exports org.apache.arrow.adapter.jdbc; + exports org.apache.arrow.adapter.jdbc.binder; + + requires com.fasterxml.jackson.databind; + requires java.sql; + requires jdk.unsupported; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; +} diff --git a/java/adapter/orc/CMakeLists.txt b/java/adapter/orc/CMakeLists.txt index a9b3a48027937..d29856ff8cd5e 100644 --- a/java/adapter/orc/CMakeLists.txt +++ b/java/adapter/orc/CMakeLists.txt @@ -37,6 +37,11 @@ set_property(TARGET arrow_java_jni_orc PROPERTY OUTPUT_NAME "arrow_orc_jni") target_link_libraries(arrow_java_jni_orc arrow_java_jni_orc_headers jni Arrow::arrow_static) +set(ARROW_JAVA_JNI_ORC_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${ARROW_JAVA_JNI_ARCH_DIR}") +set(ARROW_JAVA_JNI_ORC_BINDIR + "${CMAKE_INSTALL_PREFIX}/bin/arrow_orc_jni/${ARROW_JAVA_JNI_ARCH_DIR}") + install(TARGETS arrow_java_jni_orc - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_ORC_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_ORC_BINDIR}) diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index a42a458e2072a..605b9871639ea 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -31,6 +31,10 @@ compile ${arrow.vector.classifier} + + org.immutables + value + org.apache.orc orc-core diff --git a/java/adapter/orc/src/main/java/module-info.java b/java/adapter/orc/src/main/java/module-info.java new file mode 100644 index 0000000000000..d18a978e93fa8 --- /dev/null +++ b/java/adapter/orc/src/main/java/module-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +open module org.apache.arrow.adapter.orc { + exports org.apache.arrow.adapter.orc; + + requires hadoop.client.api; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; +} diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java index 2701c228709c2..9b599234bdf51 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java @@ -39,7 +39,7 @@ static void loadOrcAdapterLibraryFromJar() synchronized (OrcJniUtils.class) { if (!isLoaded) { final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); + LIBRARY_NAME + "/" + getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); final File libraryFile = moveFileFromJarToTemp(System.getProperty("java.io.tmpdir"), libraryToLoad, LIBRARY_NAME); System.load(libraryFile.getAbsolutePath()); diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index 3e32d955ec417..99740f2002847 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -42,6 +42,10 @@ arrow-memory-netty test + + org.immutables + value + diff --git a/java/algorithm/src/main/java/module-info.java b/java/algorithm/src/main/java/module-info.java new file mode 100644 index 0000000000000..b347f55aa4d00 --- /dev/null +++ b/java/algorithm/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.algorithm { + exports org.apache.arrow.algorithm.search; + exports org.apache.arrow.algorithm.deduplicate; + exports org.apache.arrow.algorithm.dictionary; + exports org.apache.arrow.algorithm.rank; + exports org.apache.arrow.algorithm.misc; + exports org.apache.arrow.algorithm.sort; + + requires jdk.unsupported; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; +} diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 5c2ed33dadddf..7ffb833e7f6d7 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -82,11 +82,6 @@ flight-core ${project.version} - - org.apache.arrow - flight-grpc - ${project.version} - org.apache.arrow flight-integration-tests diff --git a/java/c/CMakeLists.txt b/java/c/CMakeLists.txt index 8ff208aaeb010..83909c5e13e1b 100644 --- a/java/c/CMakeLists.txt +++ b/java/c/CMakeLists.txt @@ -30,6 +30,11 @@ add_library(arrow_java_jni_cdata SHARED src/main/cpp/jni_wrapper.cc) set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni") target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni) +set(ARROW_JAVA_JNI_C_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${ARROW_JAVA_JNI_ARCH_DIR}") +set(ARROW_JAVA_JNI_C_BINDIR + "${CMAKE_INSTALL_PREFIX}/bin/arrow_cdata_jni/${ARROW_JAVA_JNI_ARCH_DIR}") + install(TARGETS arrow_java_jni_cdata - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_C_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_C_BINDIR}) diff --git a/java/c/pom.xml b/java/c/pom.xml index 8fc3f36994d8a..a999292979d56 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -48,6 +48,10 @@ org.slf4j slf4j-api + + org.immutables + value + org.apache.arrow arrow-memory-unsafe diff --git a/java/c/src/main/java/module-info.java b/java/c/src/main/java/module-info.java new file mode 100644 index 0000000000000..0a62c9b9875b4 --- /dev/null +++ b/java/c/src/main/java/module-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +open module org.apache.arrow.c { + exports org.apache.arrow.c; + exports org.apache.arrow.c.jni; + + requires flatbuffers.java; + requires jdk.unsupported; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires org.slf4j; +} diff --git a/java/c/src/main/java/org/apache/arrow/c/Data.java b/java/c/src/main/java/org/apache/arrow/c/Data.java index a92853b3504f0..c90ce7604d6e7 100644 --- a/java/c/src/main/java/org/apache/arrow/c/Data.java +++ b/java/c/src/main/java/org/apache/arrow/c/Data.java @@ -19,8 +19,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.StructVectorLoader; -import org.apache.arrow.vector.StructVectorUnloader; import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.VectorUnloader; diff --git a/java/c/src/main/java/org/apache/arrow/vector/StructVectorLoader.java b/java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java similarity index 98% rename from java/c/src/main/java/org/apache/arrow/vector/StructVectorLoader.java rename to java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java index 4a62be7851ac7..d9afd0189d807 100644 --- a/java/c/src/main/java/org/apache/arrow/vector/StructVectorLoader.java +++ b/java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.vector; +package org.apache.arrow.c; import static org.apache.arrow.util.Preconditions.checkArgument; @@ -27,6 +27,8 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Collections2; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.compression.CompressionCodec; import org.apache.arrow.vector.compression.CompressionUtil; diff --git a/java/c/src/main/java/org/apache/arrow/vector/StructVectorUnloader.java b/java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java similarity index 97% rename from java/c/src/main/java/org/apache/arrow/vector/StructVectorUnloader.java rename to java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java index e75156cf237bb..aa6d9b4d0f6a7 100644 --- a/java/c/src/main/java/org/apache/arrow/vector/StructVectorUnloader.java +++ b/java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.arrow.vector; +package org.apache.arrow.c; import java.util.ArrayList; import java.util.List; import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.compression.CompressionCodec; import org.apache.arrow.vector.compression.CompressionUtil; diff --git a/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java b/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java index e435461349257..ef9f432cf0036 100644 --- a/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java +++ b/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java @@ -80,7 +80,7 @@ private synchronized void loadRemaining() { private void load(String name) { final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(name); + name + "/" + getNormalizedArch() + "/" + System.mapLibraryName(name); try { File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir"))); temp.deleteOnExit(); diff --git a/java/compression/pom.xml b/java/compression/pom.xml index 9a9f029fee137..e8008c9754374 100644 --- a/java/compression/pom.xml +++ b/java/compression/pom.xml @@ -35,6 +35,10 @@ arrow-memory-unsafe test + + org.immutables + value + org.apache.commons commons-compress diff --git a/java/compression/src/main/java/module-info.java b/java/compression/src/main/java/module-info.java new file mode 100644 index 0000000000000..6bf989e4c142e --- /dev/null +++ b/java/compression/src/main/java/module-info.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.compression { + exports org.apache.arrow.compression; + + requires com.github.luben.zstd_jni; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires org.apache.commons.compress; +} diff --git a/java/dataset/CMakeLists.txt b/java/dataset/CMakeLists.txt index ede3ee7330d21..348850c3be5da 100644 --- a/java/dataset/CMakeLists.txt +++ b/java/dataset/CMakeLists.txt @@ -47,6 +47,12 @@ if(BUILD_TESTING) add_test(NAME arrow-java-jni-dataset-test COMMAND arrow-java-jni-dataset-test) endif() +set(ARROW_JAVA_JNI_DATASET_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${ARROW_JAVA_JNI_ARCH_DIR}") + +set(ARROW_JAVA_JNI_DATASET_BINDIR + "${CMAKE_INSTALL_PREFIX}/bin/arrow_dataset_jni/${ARROW_JAVA_JNI_ARCH_DIR}") + install(TARGETS arrow_java_jni_dataset - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_DATASET_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_DATASET_BINDIR}) diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index bb5636b745490..a18f443b7e15a 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -47,6 +47,10 @@ arrow-c-data compile + + org.immutables + value + org.apache.arrow arrow-memory-netty @@ -161,6 +165,15 @@ + + maven-surefire-plugin + + false + + ${project.basedir}/../../testing/data + + + org.xolstice.maven.plugins protobuf-maven-plugin @@ -182,4 +195,30 @@ + + + jdk11+ + + [11,] + + !m2e.version + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + ${project.basedir}/../../testing/data + + --add-reads=org.apache.arrow.dataset=com.fasterxml.jackson.databind --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + + diff --git a/java/dataset/src/main/java/module-info.java b/java/dataset/src/main/java/module-info.java new file mode 100644 index 0000000000000..1672d12ffec69 --- /dev/null +++ b/java/dataset/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +open module org.apache.arrow.dataset { + exports org.apache.arrow.dataset.file; + exports org.apache.arrow.dataset.source; + exports org.apache.arrow.dataset.jni; + exports org.apache.arrow.dataset.substrait; + exports org.apache.arrow.dataset.scanner; + + requires org.apache.arrow.c; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; +} diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java index eb26400cbf882..3922e90335da4 100644 --- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java +++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java @@ -40,7 +40,12 @@ private DirectReservationListener() { methodUnreserve = this.getDeclaredMethodBaseOnJDKVersion(classBits, "unreserveMemory"); methodUnreserve.setAccessible(true); } catch (Exception e) { - throw new RuntimeException(e); + final RuntimeException failure = new RuntimeException( + "Failed to initialize DirectReservationListener. When starting Java you must include " + + "`--add-opens=java.base/java.nio=org.apache.arrow.dataset,org.apache.arrow.memory.core,ALL-UNNAMED` " + + "(See https://arrow.apache.org/docs/java/install.html)", e); + failure.printStackTrace(); + throw failure; } } diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java index a3b31c73e8540..cf2f8fe29e8ba 100644 --- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java +++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java @@ -79,7 +79,7 @@ private synchronized void loadRemaining() { private void load(String name) { final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(name); + name + "/" + getNormalizedArch() + "/" + System.mapLibraryName(name); try { File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir"))); temp.deleteOnExit(); diff --git a/java/dataset/src/test/java/org/apache/arrow/dataset/ParquetWriteSupport.java b/java/dataset/src/test/java/org/apache/arrow/dataset/ParquetWriteSupport.java index 75f905877cd1f..2352a65e8fb62 100644 --- a/java/dataset/src/test/java/org/apache/arrow/dataset/ParquetWriteSupport.java +++ b/java/dataset/src/test/java/org/apache/arrow/dataset/ParquetWriteSupport.java @@ -18,8 +18,8 @@ package org.apache.arrow.dataset; import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.InputStream; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -47,7 +47,7 @@ public class ParquetWriteSupport implements AutoCloseable { public ParquetWriteSupport(String schemaName, File outputFolder) throws Exception { - avroSchema = readSchemaFromFile(schemaName); + avroSchema = getSchema(schemaName); path = outputFolder.getPath() + "/" + "generated-" + random.nextLong() + ".parquet"; uri = "file://" + path; writer = AvroParquetWriter @@ -56,10 +56,23 @@ public ParquetWriteSupport(String schemaName, File outputFolder) throws Exceptio .build(); } - private static Schema readSchemaFromFile(String schemaName) throws Exception { - Path schemaPath = Paths.get(ParquetWriteSupport.class.getResource("/").getPath(), - "avroschema", schemaName); - return new org.apache.avro.Schema.Parser().parse(schemaPath.toFile()); + public static Schema getSchema(String schemaName) throws Exception { + try { + // Attempt to use JDK 9 behavior of getting the module then the resource stream from the module. + // Note that this code is caller-sensitive. + Method getModuleMethod = Class.class.getMethod("getModule"); + Object module = getModuleMethod.invoke(ParquetWriteSupport.class); + Method getResourceAsStreamFromModule = module.getClass().getMethod("getResourceAsStream", String.class); + try (InputStream is = (InputStream) getResourceAsStreamFromModule.invoke(module, "/avroschema/" + schemaName)) { + return new Schema.Parser() + .parse(is); + } + } catch (NoSuchMethodException ex) { + // Use JDK8 behavior. + try (InputStream is = ParquetWriteSupport.class.getResourceAsStream("/avroschema/" + schemaName)) { + return new Schema.Parser().parse(is); + } + } } public static ParquetWriteSupport writeTempFile(String schemaName, File outputFolder, diff --git a/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java b/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java index 5293aca0c329b..13b247452348d 100644 --- a/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java +++ b/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java @@ -68,6 +68,7 @@ import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.test.util.ArrowTestDataUtil; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; @@ -75,7 +76,6 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.arrow.vector.util.ArrowTestDataUtil; import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; import org.apache.arrow.vector.util.Text; import org.junit.ClassRule; diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 8f41d2b65b7d1..ec3034d14e271 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -95,6 +95,11 @@ grpc-services test + + io.grpc + grpc-inprocess + test + com.fasterxml.jackson.core @@ -108,6 +113,10 @@ javax.annotation javax.annotation-api + + org.immutables + value + com.google.api.grpc @@ -305,4 +314,32 @@ + + + + jdk11+ + + [11,] + + !m2e.version + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens=org.apache.arrow.flight.core/org.apache.arrow.flight.perf.impl=protobuf.java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + false + + ${project.basedir}/../../../testing/data + + + + + + + + diff --git a/java/flight/flight-core/src/main/java/module-info.java b/java/flight/flight-core/src/main/java/module-info.java new file mode 100644 index 0000000000000..f6bf5b73b0972 --- /dev/null +++ b/java/flight/flight-core/src/main/java/module-info.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.flight.core { + exports org.apache.arrow.flight; + exports org.apache.arrow.flight.auth; + exports org.apache.arrow.flight.auth2; + exports org.apache.arrow.flight.client; + exports org.apache.arrow.flight.impl; + exports org.apache.arrow.flight.sql.impl; + + requires com.fasterxml.jackson.databind; + requires com.google.common; + requires com.google.errorprone.annotations; + requires io.grpc; + requires io.grpc.internal; + requires io.grpc.netty; + requires io.grpc.protobuf; + requires io.grpc.stub; + requires io.netty.common; + requires io.netty.handler; + requires io.netty.transport; + requires org.apache.arrow.format; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires protobuf.java; + requires org.slf4j; +} diff --git a/java/flight/flight-grpc/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java similarity index 100% rename from java/flight/flight-grpc/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java rename to java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java index f9def74b56d1b..11510dbd32058 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Random; -import org.apache.arrow.vector.util.ArrowTestDataUtil; +import org.apache.arrow.vector.test.util.ArrowTestDataUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.function.Executable; diff --git a/java/flight/flight-grpc/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java similarity index 100% rename from java/flight/flight-grpc/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java rename to java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java diff --git a/java/flight/flight-grpc/src/test/protobuf/test.proto b/java/flight/flight-core/src/test/protobuf/test.proto similarity index 100% rename from java/flight/flight-grpc/src/test/protobuf/test.proto rename to java/flight/flight-core/src/test/protobuf/test.proto diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml deleted file mode 100644 index af765f8c436be..0000000000000 --- a/java/flight/flight-grpc/pom.xml +++ /dev/null @@ -1,123 +0,0 @@ - - - - - arrow-flight - org.apache.arrow - 15.0.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - flight-grpc - Arrow Flight GRPC - (Experimental)Contains utility class to expose Flight gRPC service and client - jar - - - 1 - - - - - org.apache.arrow - flight-core - - - io.netty - netty-transport-native-unix-common - - - io.netty - netty-transport-native-kqueue - - - io.netty - netty-transport-native-epoll - - - - - io.grpc - grpc-stub - - - io.grpc - grpc-inprocess - test - - - org.apache.arrow - arrow-memory-core - compile - - - org.apache.arrow - arrow-memory-netty - runtime - - - io.grpc - grpc-protobuf - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - io.grpc - grpc-api - - - - - - - - kr.motd.maven - os-maven-plugin - 1.7.0 - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.6.1 - - com.google.protobuf:protoc:${dep.protobuf-bom.version}:exe:${os.detected.classifier} - false - grpc-java - io.grpc:protoc-gen-grpc-java:${dep.grpc-bom.version}:exe:${os.detected.classifier} - - - - test - - ${basedir}/src/test/protobuf - ${project.build.directory}/generated-test-sources//protobuf - - - compile - compile-custom - - - - - - - - diff --git a/java/flight/flight-grpc/src/test/resources/logback.xml b/java/flight/flight-grpc/src/test/resources/logback.xml deleted file mode 100644 index 4c54d18a210ff..0000000000000 --- a/java/flight/flight-grpc/src/test/resources/logback.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index 3c7e4b3495e5a..5ae1a0a23f3c8 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -50,6 +50,10 @@ org.apache.arrow arrow-memory-core + + org.immutables + value + org.apache.arrow arrow-jdbc @@ -110,4 +114,27 @@ + + + jdk11+ + + [11,] + + !m2e.version + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-reads=org.apache.arrow.flight.sql=org.slf4j --add-reads=org.apache.arrow.flight.core=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + + + diff --git a/java/flight/flight-sql/src/main/java/module-info.java b/java/flight/flight-sql/src/main/java/module-info.java new file mode 100644 index 0000000000000..5514d5b870afd --- /dev/null +++ b/java/flight/flight-sql/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.flight.sql { + exports org.apache.arrow.flight.sql; + exports org.apache.arrow.flight.sql.example; + exports org.apache.arrow.flight.sql.util; + + requires com.google.common; + requires java.sql; + requires org.apache.arrow.flight.core; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires protobuf.java; +} diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java similarity index 98% rename from java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java rename to java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java index 948364a920004..a39736e939f0b 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.flight; +package org.apache.arrow.flight.sql.test; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -38,6 +38,15 @@ import java.util.Optional; import java.util.stream.IntStream; +import org.apache.arrow.flight.CancelFlightInfoRequest; +import org.apache.arrow.flight.FlightClient; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.flight.FlightServer; +import org.apache.arrow.flight.FlightStatusCode; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.RenewFlightEndpointRequest; import org.apache.arrow.flight.sql.FlightSqlClient; import org.apache.arrow.flight.sql.FlightSqlClient.PreparedStatement; import org.apache.arrow.flight.sql.FlightSqlColumnMetadata; @@ -914,7 +923,7 @@ public void testCancelFlightInfo() { FlightInfo info = sqlClient.getSqlInfo(); CancelFlightInfoRequest request = new CancelFlightInfoRequest(info); FlightRuntimeException fre = assertThrows(FlightRuntimeException.class, () -> sqlClient.cancelFlightInfo(request)); - assertEquals(FlightStatusCode.UNIMPLEMENTED, fre.status().code()); + Assertions.assertEquals(FlightStatusCode.UNIMPLEMENTED, fre.status().code()); } @Test diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSqlStreams.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java similarity index 96% rename from java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSqlStreams.java rename to java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java index 1dd925eb53add..1dd96f0fd4e9c 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSqlStreams.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.flight; +package org.apache.arrow.flight.sql.test; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -28,6 +28,15 @@ import java.util.Collections; import java.util.List; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.FlightClient; +import org.apache.arrow.flight.FlightDescriptor; +import org.apache.arrow.flight.FlightEndpoint; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightServer; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.Ticket; import org.apache.arrow.flight.sql.BasicFlightSqlProducer; import org.apache.arrow.flight.sql.FlightSqlClient; import org.apache.arrow.flight.sql.FlightSqlProducer; diff --git a/java/flight/pom.xml b/java/flight/pom.xml index 7ddda94f77b49..9ef01d07a7388 100644 --- a/java/flight/pom.xml +++ b/java/flight/pom.xml @@ -26,7 +26,6 @@ flight-core - flight-grpc flight-sql flight-sql-jdbc-core flight-sql-jdbc-driver diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt index 2aa8d92959e42..369829d7a30d5 100644 --- a/java/gandiva/CMakeLists.txt +++ b/java/gandiva/CMakeLists.txt @@ -84,6 +84,11 @@ if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) ) endif() +set(ARROW_JAVA_JNI_GANDIVA_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${ARROW_JAVA_JNI_ARCH_DIR}") +set(ARROW_JAVA_JNI_GANDIVA_BINDIR + "${CMAKE_INSTALL_PREFIX}/bin/gandiva_jni/${ARROW_JAVA_JNI_ARCH_DIR}") + install(TARGETS arrow_java_jni_gandiva - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_GANDIVA_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_GANDIVA_BINDIR}) diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index e837a09ff8330..330c156a0346b 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -34,6 +34,10 @@ org.apache.arrow arrow-memory-core + + org.immutables + value + org.apache.arrow arrow-memory-netty diff --git a/java/gandiva/src/main/java/module-info.java b/java/gandiva/src/main/java/module-info.java new file mode 100644 index 0000000000000..533717d91f7f0 --- /dev/null +++ b/java/gandiva/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +open module org.apache.arrow.gandiva { + exports org.apache.arrow.gandiva.expression; + exports org.apache.arrow.gandiva.exceptions; + exports org.apache.arrow.gandiva.evaluator; + exports org.apache.arrow.gandiva.ipc; + + requires com.google.common; + requires org.apache.arrow.format; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires org.slf4j; + requires protobuf.java; +} \ No newline at end of file diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java index 2528989f3784b..57748e9c8e1af 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java @@ -71,7 +71,7 @@ private static JniLoader setupInstance() throws GandivaException { private static void loadGandivaLibraryFromJar(final String tmpDir) throws IOException, GandivaException { final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); + LIBRARY_NAME + "/" + getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); final File libraryFile = moveFileFromJarToTemp(tmpDir, libraryToLoad, LIBRARY_NAME); System.load(libraryFile.getAbsolutePath()); } diff --git a/java/performance/src/test/java/org/apache/arrow/adapter/AvroAdapterBenchmarks.java b/java/performance/src/test/java/org/apache/arrow/adapter/AvroAdapterBenchmarks.java index 884647b5af3ea..c07aeffafb10c 100644 --- a/java/performance/src/test/java/org/apache/arrow/adapter/AvroAdapterBenchmarks.java +++ b/java/performance/src/test/java/org/apache/arrow/adapter/AvroAdapterBenchmarks.java @@ -21,10 +21,10 @@ import java.io.ByteArrayOutputStream; import java.util.concurrent.TimeUnit; -import org.apache.arrow.AvroToArrow; -import org.apache.arrow.AvroToArrowConfig; -import org.apache.arrow.AvroToArrowConfigBuilder; -import org.apache.arrow.AvroToArrowVectorIterator; +import org.apache.arrow.adapter.avro.AvroToArrow; +import org.apache.arrow.adapter.avro.AvroToArrowConfig; +import org.apache.arrow.adapter.avro.AvroToArrowConfigBuilder; +import org.apache.arrow.adapter.avro.AvroToArrowVectorIterator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; diff --git a/java/pom.xml b/java/pom.xml index b2513d586268b..6fc4df67af3e7 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1055,7 +1055,6 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.c.jni.dist.dir} ../ @@ -1128,7 +1127,6 @@ -DARROW_SUBSTRAIT=${ARROW_DATASET} -DARROW_USE_CCACHE=ON -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON @@ -1169,7 +1167,6 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/lib/${os.detected.arch}/cmake -DProtobuf_USE_STATIC_LIBS=ON @@ -1248,7 +1245,6 @@ -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON -GNinja @@ -1290,7 +1286,6 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/lib/${os.detected.arch}/cmake diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 8df436bac9aef..0d7eacfe2ddce 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -34,6 +34,10 @@ org.apache.arrow arrow-compression + + org.immutables + value + com.google.guava guava diff --git a/java/tools/src/main/java/module-info.java b/java/tools/src/main/java/module-info.java new file mode 100644 index 0000000000000..6b4329eb84f2a --- /dev/null +++ b/java/tools/src/main/java/module-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.tools { + exports org.apache.arrow.tools; + + requires com.fasterxml.jackson.databind; + requires com.google.common; + requires org.apache.arrow.compression; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires org.slf4j; +} diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java b/java/vector/src/test/java/org/apache/arrow/vector/test/util/ArrowTestDataUtil.java similarity index 97% rename from java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java rename to java/vector/src/test/java/org/apache/arrow/vector/test/util/ArrowTestDataUtil.java index 1c525c0c271ac..901a09e313f59 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/test/util/ArrowTestDataUtil.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.util; +package org.apache.arrow.vector.test.util; import java.nio.file.Path; import java.nio.file.Paths; From 05b8f366e17ee6f21df4746bb6a65be399dfb68d Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 19 Jan 2024 18:43:37 -0500 Subject: [PATCH 52/92] GH-39697: [R] Source build should check if offline (#39699) ### Rationale for this change CRAN. ### What changes are included in this PR? See the commit messages ### Are these changes tested? Existing CI should pass and not be affected. We should confirm that source builds get all features enabled. We should test on macbuilder with this package and with one where we've inserted `download.file <- function(...) stop()` at the top of nixlibs.R to simulate offline behavior. ### Are there any user-facing changes? I hope there is only one user affected. * Closes: #39697 Authored-by: Neal Richardson Signed-off-by: Jacob Wujciak-Jens --- dev/tasks/r/github.linux.offline.build.yml | 2 +- dev/tasks/tasks.yml | 2 +- r/NEWS.md | 2 +- r/configure | 2 +- r/tools/nixlibs.R | 63 +++++++++++++--------- r/vignettes/developers/setup.Rmd | 2 +- 6 files changed, 43 insertions(+), 30 deletions(-) diff --git a/dev/tasks/r/github.linux.offline.build.yml b/dev/tasks/r/github.linux.offline.build.yml index b116accda891c..7a747ac480084 100644 --- a/dev/tasks/r/github.linux.offline.build.yml +++ b/dev/tasks/r/github.linux.offline.build.yml @@ -79,7 +79,7 @@ jobs: shell: Rscript {0} - name: Install env: - TEST_OFFLINE_BUILD: true + ARROW_OFFLINE_BUILD: true LIBARROW_MINIMAL: false {{ macros.github_set_sccache_envvars()|indent(8)}} run: | diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ca45d48bcd470..64620d41993d5 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1272,7 +1272,7 @@ tasks: r_org: library r_image: r-base r_tag: latest - flags: '-e TEST_OFFLINE_BUILD=true' + flags: '-e ARROW_OFFLINE_BUILD=true' test-r-dev-duckdb: ci: github diff --git a/r/NEWS.md b/r/NEWS.md index 9badf4700a36e..22eb5b34ceb0f 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -32,7 +32,7 @@ ## Minor improvements and fixes -* Don't download cmake when TEST_OFFLINE_BUILD=true and update `SystemRequirements` (#39602). +* Don't download cmake when ARROW_OFFLINE_BUILD=true and update `SystemRequirements` (#39602). * Fallback to source build gracefully if binary download fails (#39587). * An error is now thrown instead of warning and pulling the data into R when any of `sub`, `gsub`, `stringr::str_replace`, `stringr::str_replace_all` are diff --git a/r/configure b/r/configure index 029fc004dfc4c..0882ee6719c4b 100755 --- a/r/configure +++ b/r/configure @@ -73,7 +73,7 @@ FORCE_BUNDLED_BUILD=`echo $FORCE_BUNDLED_BUILD | tr '[:upper:]' '[:lower:]'` ARROW_USE_PKG_CONFIG=`echo $ARROW_USE_PKG_CONFIG | tr '[:upper:]' '[:lower:]'` # Just used in testing: whether or not it is ok to download dependencies (in the # bundled build) -TEST_OFFLINE_BUILD=`echo $TEST_OFFLINE_BUILD | tr '[:upper:]' '[:lower:]'` +ARROW_OFFLINE_BUILD=`echo $ARROW_OFFLINE_BUILD | tr '[:upper:]' '[:lower:]'` VERSION=`grep '^Version' DESCRIPTION | sed s/Version:\ //` UNAME=`uname -s` diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index dfe379ebe20df..17c6ab0a8078b 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -120,11 +120,11 @@ validate_checksum <- function(binary_url, libfile, hush = quietly) { # The warnings from system2 if it fails pop up later in the log and thus are # more confusing than they are helpful (so we suppress them) checksum_ok <- suppressWarnings(system2( - "shasum", - args = c("--status", "-a", "512", "-c", checksum_file), - stdout = ifelse(quietly, FALSE, ""), - stderr = ifelse(quietly, FALSE, "") - )) == 0 + "shasum", + args = c("--status", "-a", "512", "-c", checksum_file), + stdout = ifelse(quietly, FALSE, ""), + stderr = ifelse(quietly, FALSE, "") + )) == 0 if (!checksum_ok) { checksum_ok <- suppressWarnings(system2( @@ -565,8 +565,8 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- c(env_var_list, ARROW_DEPENDENCY_SOURCE = "BUNDLED") } - # On macOS, if not otherwise set, let's override Boost_SOURCE to be bundled - # Necessary due to #39590 for CRAN + # On macOS, if not otherwise set, let's override Boost_SOURCE to be bundled + # Necessary due to #39590 for CRAN if (on_macos) { # Using lowercase (e.g. Boost_SOURCE) to match the cmake args we use already. deps_to_bundle <- c("Boost", "lz4") @@ -906,28 +906,13 @@ on_windows <- tolower(Sys.info()[["sysname"]]) == "windows" # For local debugging, set ARROW_R_DEV=TRUE to make this script print more quietly <- !env_is("ARROW_R_DEV", "true") -not_cran <- env_is("NOT_CRAN", "true") - -if (is_release) { - VERSION <- VERSION[1, 1:3] - arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/") -} else { - # Don't override explictily set NOT_CRAN env var, as it is used in CI. - not_cran <- !env_is("NOT_CRAN", "false") - arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/") -} - -if (!is_release && !test_mode) { - VERSION <- find_latest_nightly(VERSION) -} - # To collect dirs to rm on exit, use cleanup() to add dirs # we reset it to avoid errors on reruns in the same session. options(.arrow.cleanup = character()) on.exit(unlink(getOption(".arrow.cleanup"), recursive = TRUE), add = TRUE) -# enable full featured builds for macOS in case of CRAN source builds. -if (not_cran || on_macos) { +not_cran <- env_is("NOT_CRAN", "true") +if (not_cran) { # Set more eager defaults if (env_is("LIBARROW_BINARY", "")) { Sys.setenv(LIBARROW_BINARY = "true") @@ -943,9 +928,37 @@ if (not_cran || on_macos) { build_ok <- !env_is("LIBARROW_BUILD", "false") # Check if we're authorized to download -download_ok <- !test_mode && !env_is("TEST_OFFLINE_BUILD", "true") +download_ok <- !test_mode && !env_is("ARROW_OFFLINE_BUILD", "true") +if (!download_ok) { + lg("Dependency downloading disabled. Unset ARROW_OFFLINE_BUILD to enable", .indent = "***") +} +# If not forbidden from downloading, check if we are offline and turn off downloading. +# The default libarrow source build will download its source dependencies and fail +# if they can't be retrieved. +# But, don't do this if the user has requested a binary or a non-minimal build: +# we should error rather than silently succeeding with a minimal build. +if (download_ok && Sys.getenv("LIBARROW_BINARY") %in% c("false", "") && !env_is("LIBARROW_MINIMAL", "false")) { + download_ok <- try_download("https://apache.jfrog.io/artifactory/arrow/r/", tempfile()) + if (!download_ok) { + lg("Network connection not available", .indent = "***") + } +} + download_libarrow_ok <- download_ok && !env_is("LIBARROW_DOWNLOAD", "false") +# Set binary repos +if (is_release) { + VERSION <- VERSION[1, 1:3] + arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/") +} else { + arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/") +} + +# If we're on a dev version, look for the most recent libarrow binary version +if (download_libarrow_ok && !is_release && !test_mode) { + VERSION <- find_latest_nightly(VERSION) +} + # This "tools/thirdparty_dependencies" path, within the tar file, might exist if # create_package_with_all_dependencies() was run, or if someone has created it # manually before running make build. diff --git a/r/vignettes/developers/setup.Rmd b/r/vignettes/developers/setup.Rmd index 119bc78419410..4c1eab1e6972f 100644 --- a/r/vignettes/developers/setup.Rmd +++ b/r/vignettes/developers/setup.Rmd @@ -280,7 +280,7 @@ withr::with_makevars(list(CPPFLAGS = "", LDFLAGS = ""), remotes::install_github( * See the user-facing [article on installation](../install.html) for a large number of environment variables that determine how the build works and what features get built. -* `TEST_OFFLINE_BUILD`: When set to `true`, the build script will not download +* `ARROW_OFFLINE_BUILD`: When set to `true`, the build script will not download prebuilt the C++ library binary or, if needed, `cmake`. It will turn off any features that require a download, unless they're available in `ARROW_THIRDPARTY_DEPENDENCY_DIR` or the `tools/thirdparty_download/` subfolder. From cbc7349eb9fc49a10691194b0e5573e38a17d0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sun, 21 Jan 2024 16:06:39 +0100 Subject: [PATCH 53/92] MINOR: [Release] Update versions for 16.0.0-SNAPSHOT --- c_glib/meson.build | 2 +- ci/scripts/PKGBUILD | 2 +- cpp/CMakeLists.txt | 2 +- cpp/vcpkg.json | 2 +- csharp/Directory.Build.props | 2 +- .../homebrew-formulae/apache-arrow-glib.rb | 2 +- dev/tasks/homebrew-formulae/apache-arrow.rb | 2 +- docs/source/_static/versions.json | 9 ++++-- go/README.md | 2 +- go/arrow/_examples/helloworld/main.go | 8 ++--- go/arrow/_tools/tmpl/main.go | 2 +- go/arrow/array.go | 4 +-- go/arrow/array/array.go | 6 ++-- go/arrow/array/array_test.go | 10 +++--- go/arrow/array/binary.go | 6 ++-- go/arrow/array/binary_test.go | 6 ++-- go/arrow/array/binarybuilder.go | 8 ++--- go/arrow/array/binarybuilder_test.go | 6 ++-- go/arrow/array/boolean.go | 8 ++--- go/arrow/array/boolean_test.go | 4 +-- go/arrow/array/booleanbuilder.go | 10 +++--- go/arrow/array/booleanbuilder_test.go | 6 ++-- go/arrow/array/bufferbuilder.go | 8 ++--- go/arrow/array/bufferbuilder_byte.go | 2 +- go/arrow/array/bufferbuilder_numeric.gen.go | 6 ++-- .../array/bufferbuilder_numeric.gen.go.tmpl | 6 ++-- go/arrow/array/bufferbuilder_numeric_test.go | 4 +-- go/arrow/array/builder.go | 8 ++--- go/arrow/array/builder_test.go | 4 +-- go/arrow/array/compare.go | 6 ++-- go/arrow/array/compare_test.go | 10 +++--- go/arrow/array/concat.go | 14 ++++---- go/arrow/array/concat_test.go | 10 +++--- go/arrow/array/data.go | 6 ++-- go/arrow/array/data_test.go | 4 +-- go/arrow/array/decimal128.go | 12 +++---- go/arrow/array/decimal128_test.go | 8 ++--- go/arrow/array/decimal256.go | 12 +++---- go/arrow/array/decimal256_test.go | 8 ++--- go/arrow/array/decimal_test.go | 12 +++---- go/arrow/array/dictionary.go | 20 ++++++------ go/arrow/array/dictionary_test.go | 14 ++++---- go/arrow/array/diff.go | 2 +- go/arrow/array/diff_test.go | 10 +++--- go/arrow/array/encoded.go | 12 +++---- go/arrow/array/encoded_test.go | 8 ++--- go/arrow/array/extension.go | 6 ++-- go/arrow/array/extension_test.go | 8 ++--- go/arrow/array/fixed_size_list.go | 10 +++--- go/arrow/array/fixed_size_list_test.go | 6 ++-- go/arrow/array/fixedsize_binary.go | 4 +-- go/arrow/array/fixedsize_binary_test.go | 6 ++-- go/arrow/array/fixedsize_binarybuilder.go | 8 ++--- .../array/fixedsize_binarybuilder_test.go | 4 +-- go/arrow/array/float16.go | 6 ++-- go/arrow/array/float16_builder.go | 12 +++---- go/arrow/array/float16_builder_test.go | 6 ++-- go/arrow/array/interval.go | 10 +++--- go/arrow/array/interval_test.go | 6 ++-- go/arrow/array/json_reader.go | 8 ++--- go/arrow/array/json_reader_test.go | 6 ++-- go/arrow/array/list.go | 10 +++--- go/arrow/array/list_test.go | 6 ++-- go/arrow/array/map.go | 6 ++-- go/arrow/array/map_test.go | 6 ++-- go/arrow/array/null.go | 8 ++--- go/arrow/array/null_test.go | 6 ++-- go/arrow/array/numeric.gen.go | 4 +-- go/arrow/array/numeric.gen.go.tmpl | 4 +-- go/arrow/array/numeric_test.go | 8 ++--- go/arrow/array/numericbuilder.gen.go | 10 +++--- go/arrow/array/numericbuilder.gen.go.tmpl | 10 +++--- go/arrow/array/numericbuilder.gen_test.go | 6 ++-- .../array/numericbuilder.gen_test.go.tmpl | 6 ++-- go/arrow/array/record.go | 8 ++--- go/arrow/array/record_test.go | 6 ++-- go/arrow/array/string.go | 6 ++-- go/arrow/array/string_test.go | 8 ++--- go/arrow/array/struct.go | 10 +++--- go/arrow/array/struct_test.go | 6 ++-- go/arrow/array/table.go | 4 +-- go/arrow/array/table_test.go | 6 ++-- go/arrow/array/timestamp.go | 10 +++--- go/arrow/array/timestamp_test.go | 6 ++-- go/arrow/array/union.go | 12 +++---- go/arrow/array/union_test.go | 6 ++-- go/arrow/array/util.go | 10 +++--- go/arrow/array/util_test.go | 14 ++++---- go/arrow/arrio/arrio.go | 2 +- go/arrow/arrio/arrio_test.go | 10 +++--- go/arrow/avro/avro2parquet/main.go | 8 ++--- go/arrow/avro/reader.go | 8 ++--- go/arrow/avro/reader_test.go | 2 +- go/arrow/avro/reader_types.go | 12 +++---- go/arrow/avro/schema.go | 6 ++-- go/arrow/avro/schema_test.go | 2 +- go/arrow/bitutil/bitmaps.go | 6 ++-- go/arrow/bitutil/bitmaps_test.go | 4 +-- go/arrow/bitutil/bitutil.go | 2 +- go/arrow/bitutil/bitutil_test.go | 4 +-- go/arrow/cdata/cdata.go | 8 ++--- go/arrow/cdata/cdata_exports.go | 10 +++--- go/arrow/cdata/cdata_test.go | 12 +++---- go/arrow/cdata/cdata_test_framework.go | 8 ++--- go/arrow/cdata/exports.go | 4 +-- go/arrow/cdata/import_allocator.go | 2 +- go/arrow/cdata/interface.go | 8 ++--- go/arrow/cdata/test/test_cimport.go | 8 ++--- go/arrow/compute/arithmetic.go | 12 +++---- go/arrow/compute/arithmetic_test.go | 20 ++++++------ go/arrow/compute/cast.go | 10 +++--- go/arrow/compute/cast_test.go | 20 ++++++------ go/arrow/compute/datum.go | 6 ++-- go/arrow/compute/example_test.go | 10 +++--- go/arrow/compute/exec.go | 6 ++-- go/arrow/compute/exec/kernel.go | 8 ++--- go/arrow/compute/exec/kernel_test.go | 12 +++---- go/arrow/compute/exec/span.go | 10 +++--- go/arrow/compute/exec/span_test.go | 16 +++++----- go/arrow/compute/exec/utils.go | 8 ++--- go/arrow/compute/exec/utils_test.go | 8 ++--- go/arrow/compute/exec_internals_test.go | 14 ++++---- go/arrow/compute/exec_test.go | 12 +++---- go/arrow/compute/executor.go | 16 +++++----- go/arrow/compute/expression.go | 16 +++++----- go/arrow/compute/expression_test.go | 10 +++--- go/arrow/compute/exprs/builders.go | 4 +-- go/arrow/compute/exprs/builders_test.go | 4 +-- go/arrow/compute/exprs/exec.go | 18 +++++------ go/arrow/compute/exprs/exec_internal_test.go | 8 ++--- go/arrow/compute/exprs/exec_test.go | 12 +++---- go/arrow/compute/exprs/extension_types.go | 4 +-- go/arrow/compute/exprs/field_refs.go | 10 +++--- go/arrow/compute/exprs/types.go | 4 +-- go/arrow/compute/fieldref.go | 4 +-- go/arrow/compute/fieldref_test.go | 8 ++--- go/arrow/compute/functions.go | 4 +-- go/arrow/compute/functions_test.go | 4 +-- .../internal/kernels/base_arithmetic.go | 10 +++--- .../internal/kernels/base_arithmetic_amd64.go | 6 ++-- .../kernels/base_arithmetic_avx2_amd64.go | 2 +- .../kernels/base_arithmetic_sse4_amd64.go | 2 +- .../kernels/basic_arithmetic_noasm.go | 4 +-- .../compute/internal/kernels/boolean_cast.go | 6 ++-- go/arrow/compute/internal/kernels/cast.go | 6 ++-- .../compute/internal/kernels/cast_numeric.go | 2 +- .../kernels/cast_numeric_avx2_amd64.go | 2 +- .../kernels/cast_numeric_neon_arm64.go | 2 +- .../kernels/cast_numeric_sse4_amd64.go | 2 +- .../compute/internal/kernels/cast_temporal.go | 8 ++--- go/arrow/compute/internal/kernels/helpers.go | 14 ++++---- .../compute/internal/kernels/numeric_cast.go | 14 ++++---- go/arrow/compute/internal/kernels/rounding.go | 10 +++--- .../internal/kernels/scalar_arithmetic.go | 14 ++++---- .../internal/kernels/scalar_boolean.go | 6 ++-- .../kernels/scalar_comparison_amd64.go | 2 +- .../kernels/scalar_comparison_avx2_amd64.go | 2 +- .../kernels/scalar_comparison_noasm.go | 2 +- .../kernels/scalar_comparison_sse4_amd64.go | 2 +- .../internal/kernels/scalar_comparisons.go | 16 +++++----- .../compute/internal/kernels/string_casts.go | 12 +++---- go/arrow/compute/internal/kernels/types.go | 8 ++--- .../compute/internal/kernels/vector_hash.go | 14 ++++---- .../internal/kernels/vector_run_end_encode.go | 16 +++++----- .../internal/kernels/vector_selection.go | 14 ++++---- go/arrow/compute/registry.go | 2 +- go/arrow/compute/registry_test.go | 6 ++-- go/arrow/compute/scalar_bool.go | 6 ++-- go/arrow/compute/scalar_bool_test.go | 10 +++--- go/arrow/compute/scalar_compare.go | 6 ++-- go/arrow/compute/scalar_compare_test.go | 18 +++++------ go/arrow/compute/selection.go | 8 ++--- go/arrow/compute/utils.go | 12 +++---- go/arrow/compute/vector_hash.go | 4 +-- go/arrow/compute/vector_hash_test.go | 12 +++---- go/arrow/compute/vector_run_end_test.go | 14 ++++---- go/arrow/compute/vector_run_ends.go | 4 +-- go/arrow/compute/vector_selection_test.go | 18 +++++------ go/arrow/csv/common.go | 4 +-- go/arrow/csv/reader.go | 14 ++++---- go/arrow/csv/reader_test.go | 14 ++++---- go/arrow/csv/transformer.go | 4 +-- go/arrow/csv/writer.go | 2 +- go/arrow/csv/writer_test.go | 16 +++++----- go/arrow/datatype.go | 2 +- go/arrow/datatype_binary_test.go | 2 +- go/arrow/datatype_extension_test.go | 4 +-- go/arrow/datatype_fixedwidth.go | 2 +- go/arrow/datatype_fixedwidth_test.go | 2 +- go/arrow/datatype_nested.go | 2 +- go/arrow/datatype_null_test.go | 2 +- go/arrow/datatype_viewheader.go | 6 ++-- go/arrow/datatype_viewheader_inline.go | 2 +- go/arrow/datatype_viewheader_inline_go1.19.go | 2 +- go/arrow/datatype_viewheader_inline_tinygo.go | 2 +- go/arrow/decimal128/decimal128.go | 2 +- go/arrow/decimal128/decimal128_test.go | 2 +- go/arrow/decimal256/decimal256.go | 4 +-- go/arrow/decimal256/decimal256_test.go | 2 +- go/arrow/doc.go | 2 +- go/arrow/encoded/ree_utils.go | 2 +- go/arrow/encoded/ree_utils_test.go | 8 ++--- go/arrow/endian/endian.go | 4 +-- go/arrow/example_test.go | 8 ++--- go/arrow/flight/basic_auth_flight_test.go | 2 +- go/arrow/flight/client.go | 2 +- go/arrow/flight/cookie_middleware_test.go | 2 +- go/arrow/flight/example_flight_server_test.go | 2 +- go/arrow/flight/flight_middleware_test.go | 4 +-- go/arrow/flight/flight_test.go | 10 +++--- go/arrow/flight/flightsql/client.go | 12 +++---- go/arrow/flight/flightsql/client_test.go | 12 +++---- go/arrow/flight/flightsql/column_metadata.go | 2 +- go/arrow/flight/flightsql/driver/README.md | 6 ++-- .../flight/flightsql/driver/config_test.go | 2 +- go/arrow/flight/flightsql/driver/driver.go | 10 +++--- .../flight/flightsql/driver/driver_test.go | 14 ++++---- go/arrow/flight/flightsql/driver/utils.go | 4 +-- .../flight/flightsql/driver/utils_test.go | 12 +++---- .../cmd/sqlite_flightsql_server/main.go | 6 ++-- .../flightsql/example/sql_batch_reader.go | 10 +++--- .../flight/flightsql/example/sqlite_info.go | 4 +-- .../flight/flightsql/example/sqlite_server.go | 14 ++++---- .../sqlite_tables_schema_batch_reader.go | 12 +++---- .../flight/flightsql/example/type_info.go | 8 ++--- .../flightsql/schema_ref/reference_schemas.go | 2 +- go/arrow/flight/flightsql/server.go | 16 +++++----- go/arrow/flight/flightsql/server_test.go | 12 +++---- go/arrow/flight/flightsql/sql_info.go | 4 +-- .../flight/flightsql/sqlite_server_test.go | 16 +++++----- go/arrow/flight/flightsql/types.go | 2 +- go/arrow/flight/record_batch_reader.go | 12 +++---- go/arrow/flight/record_batch_writer.go | 6 ++-- go/arrow/flight/server.go | 2 +- go/arrow/flight/server_example_test.go | 2 +- go/arrow/internal/arrdata/arrdata.go | 16 +++++----- go/arrow/internal/arrdata/ioutil.go | 10 +++--- go/arrow/internal/arrjson/arrjson.go | 20 ++++++------ go/arrow/internal/arrjson/arrjson_test.go | 6 ++-- go/arrow/internal/arrjson/option.go | 4 +-- go/arrow/internal/arrjson/reader.go | 10 +++--- go/arrow/internal/arrjson/writer.go | 10 +++--- .../internal/cdata_integration/entrypoints.go | 8 ++--- go/arrow/internal/dictutils/dict.go | 6 ++-- go/arrow/internal/dictutils/dict_test.go | 8 ++--- .../arrow-flight-integration-client/main.go | 2 +- .../arrow-flight-integration-server/main.go | 2 +- .../internal/flight_integration/scenario.go | 18 +++++------ .../internal/testing/gen/random_array_gen.go | 10 +++--- go/arrow/internal/testing/tools/bits_test.go | 2 +- go/arrow/internal/testing/tools/data_types.go | 4 +-- go/arrow/internal/utils.go | 4 +-- go/arrow/ipc/cmd/arrow-cat/main.go | 4 +-- go/arrow/ipc/cmd/arrow-cat/main_test.go | 8 ++--- go/arrow/ipc/cmd/arrow-file-to-stream/main.go | 6 ++-- .../ipc/cmd/arrow-file-to-stream/main_test.go | 4 +-- .../cmd/arrow-json-integration-test/main.go | 12 +++---- .../arrow-json-integration-test/main_test.go | 4 +-- go/arrow/ipc/cmd/arrow-ls/main.go | 4 +-- go/arrow/ipc/cmd/arrow-ls/main_test.go | 8 ++--- go/arrow/ipc/cmd/arrow-stream-to-file/main.go | 6 ++-- .../ipc/cmd/arrow-stream-to-file/main_test.go | 4 +-- go/arrow/ipc/compression.go | 6 ++-- go/arrow/ipc/endian_swap.go | 6 ++-- go/arrow/ipc/endian_swap_test.go | 10 +++--- go/arrow/ipc/file_reader.go | 16 +++++----- go/arrow/ipc/file_test.go | 6 ++-- go/arrow/ipc/file_writer.go | 10 +++--- go/arrow/ipc/ipc.go | 8 ++--- go/arrow/ipc/ipc_test.go | 8 ++--- go/arrow/ipc/message.go | 6 ++-- go/arrow/ipc/message_test.go | 6 ++-- go/arrow/ipc/metadata.go | 10 +++--- go/arrow/ipc/metadata_test.go | 12 +++---- go/arrow/ipc/reader.go | 14 ++++---- go/arrow/ipc/reader_test.go | 6 ++-- go/arrow/ipc/stream_test.go | 6 ++-- go/arrow/ipc/writer.go | 18 +++++------ go/arrow/ipc/writer_test.go | 10 +++--- go/arrow/math/float64.go | 2 +- go/arrow/math/float64_avx2_amd64.go | 2 +- go/arrow/math/float64_neon_arm64.go | 2 +- go/arrow/math/float64_sse4_amd64.go | 2 +- go/arrow/math/float64_test.go | 6 ++-- go/arrow/math/int64.go | 2 +- go/arrow/math/int64_avx2_amd64.go | 2 +- go/arrow/math/int64_neon_arm64.go | 2 +- go/arrow/math/int64_sse4_amd64.go | 2 +- go/arrow/math/int64_test.go | 6 ++-- go/arrow/math/type.go.tmpl | 2 +- go/arrow/math/type_simd_amd64.go.tmpl | 2 +- go/arrow/math/type_simd_arm64.go.tmpl | 2 +- go/arrow/math/type_test.go.tmpl | 6 ++-- go/arrow/math/uint64.go | 2 +- go/arrow/math/uint64_avx2_amd64.go | 2 +- go/arrow/math/uint64_neon_arm64.go | 2 +- go/arrow/math/uint64_sse4_amd64.go | 2 +- go/arrow/math/uint64_test.go | 6 ++-- go/arrow/memory/buffer.go | 2 +- go/arrow/memory/buffer_test.go | 2 +- go/arrow/memory/cgo_allocator.go | 2 +- go/arrow/memory/default_mallocator.go | 2 +- go/arrow/memory/default_mallocator_test.go | 4 +-- go/arrow/memory/mallocator/mallocator_test.go | 2 +- go/arrow/memory/memory_test.go | 2 +- go/arrow/record.go | 2 +- go/arrow/scalar/append.go | 10 +++--- go/arrow/scalar/append_test.go | 10 +++--- go/arrow/scalar/binary.go | 4 +-- go/arrow/scalar/compare.go | 2 +- go/arrow/scalar/nested.go | 8 ++--- go/arrow/scalar/numeric.gen.go | 6 ++-- go/arrow/scalar/numeric.gen_test.go | 4 +-- go/arrow/scalar/numeric.gen_test.go.tmpl | 4 +-- go/arrow/scalar/parse.go | 12 +++---- go/arrow/scalar/scalar.go | 20 ++++++------ go/arrow/scalar/scalar_test.go | 12 +++---- go/arrow/scalar/temporal.go | 2 +- go/arrow/schema.go | 2 +- go/arrow/schema_test.go | 2 +- go/arrow/table.go | 2 +- go/arrow/tensor/numeric.gen.go | 2 +- go/arrow/tensor/numeric.gen.go.tmpl | 4 +-- go/arrow/tensor/numeric.gen_test.go | 8 ++--- go/arrow/tensor/numeric.gen_test.go.tmpl | 8 ++--- go/arrow/tensor/tensor.go | 4 +-- go/arrow/tensor/tensor_test.go | 8 ++--- go/arrow/type_traits.go | 6 ++-- go/arrow/type_traits_boolean.go | 2 +- go/arrow/type_traits_decimal128.go | 4 +-- go/arrow/type_traits_decimal256.go | 4 +-- go/arrow/type_traits_float16.go | 4 +-- go/arrow/type_traits_interval.go | 4 +-- go/arrow/type_traits_numeric.gen.go | 2 +- go/arrow/type_traits_numeric.gen.go.tmpl | 2 +- go/arrow/type_traits_numeric.gen_test.go | 2 +- go/arrow/type_traits_numeric.gen_test.go.tmpl | 2 +- go/arrow/type_traits_test.go | 8 ++--- go/arrow/type_traits_timestamp.go | 2 +- go/arrow/type_traits_view.go | 2 +- go/arrow/util/byte_size.go | 6 ++-- go/arrow/util/byte_size_test.go | 8 ++--- go/go.mod | 2 +- go/internal/bitutils/bit_block_counter.go | 4 +-- .../bitutils/bit_block_counter_test.go | 6 ++-- go/internal/bitutils/bit_run_reader.go | 6 ++-- go/internal/bitutils/bit_run_reader_test.go | 6 ++-- go/internal/bitutils/bit_set_run_reader.go | 4 +-- .../bitutils/bit_set_run_reader_test.go | 6 ++-- go/internal/bitutils/bitmap_generate.go | 2 +- go/internal/bitutils/bitmap_generate_test.go | 2 +- go/internal/hashing/xxh3_memo_table.gen.go | 6 ++-- .../hashing/xxh3_memo_table.gen.go.tmpl | 4 +-- go/internal/types/extension_types.go | 6 ++-- go/internal/types/extension_types_test.go | 10 +++--- go/internal/utils/transpose_ints_def.go | 2 +- go/internal/utils/transpose_ints_test.go | 2 +- go/parquet/cmd/parquet_reader/dumper.go | 6 ++-- go/parquet/cmd/parquet_reader/main.go | 10 +++--- go/parquet/cmd/parquet_schema/main.go | 4 +-- go/parquet/compress/brotli.go | 2 +- go/parquet/compress/compress.go | 2 +- go/parquet/compress/compress_test.go | 2 +- go/parquet/compress/zstd.go | 2 +- go/parquet/doc.go | 6 ++-- go/parquet/encryption_properties.go | 2 +- go/parquet/encryption_properties_test.go | 4 +-- go/parquet/encryption_read_config_test.go | 8 ++--- go/parquet/encryption_write_config_test.go | 8 ++--- go/parquet/file/column_reader.go | 14 ++++---- go/parquet/file/column_reader_test.go | 12 +++---- go/parquet/file/column_reader_types.gen.go | 6 ++-- .../file/column_reader_types.gen.go.tmpl | 4 +-- go/parquet/file/column_writer.go | 16 +++++----- go/parquet/file/column_writer_test.go | 26 +++++++-------- go/parquet/file/column_writer_types.gen.go | 14 ++++---- .../file/column_writer_types.gen.go.tmpl | 8 ++--- go/parquet/file/file_reader.go | 8 ++--- go/parquet/file/file_reader_mmap.go | 2 +- go/parquet/file/file_reader_mmap_windows.go | 2 +- go/parquet/file/file_reader_test.go | 20 ++++++------ go/parquet/file/file_writer.go | 10 +++--- go/parquet/file/file_writer_test.go | 14 ++++---- go/parquet/file/level_conversion.go | 10 +++--- go/parquet/file/level_conversion_test.go | 6 ++-- go/parquet/file/page_reader.go | 14 ++++---- go/parquet/file/page_writer.go | 18 +++++------ go/parquet/file/record_reader.go | 16 +++++----- go/parquet/file/row_group_reader.go | 8 ++--- go/parquet/file/row_group_writer.go | 8 ++--- go/parquet/file/row_group_writer_test.go | 8 ++--- go/parquet/internal/bmi/bmi_test.go | 2 +- .../internal/encoding/boolean_decoder.go | 8 ++--- .../internal/encoding/boolean_encoder.go | 8 ++--- .../internal/encoding/byte_array_decoder.go | 12 +++---- .../internal/encoding/byte_array_encoder.go | 10 +++--- go/parquet/internal/encoding/decoder.go | 20 ++++++------ .../internal/encoding/delta_bit_packing.go | 10 +++--- .../internal/encoding/delta_byte_array.go | 6 ++-- .../encoding/delta_byte_array_test.go | 4 +-- .../encoding/delta_length_byte_array.go | 6 ++-- go/parquet/internal/encoding/encoder.go | 16 +++++----- .../encoding/encoding_benchmarks_test.go | 16 +++++----- go/parquet/internal/encoding/encoding_test.go | 14 ++++---- .../encoding/fixed_len_byte_array_decoder.go | 4 +-- .../encoding/fixed_len_byte_array_encoder.go | 6 ++-- go/parquet/internal/encoding/levels.go | 10 +++--- go/parquet/internal/encoding/levels_test.go | 10 +++--- go/parquet/internal/encoding/memo_table.go | 10 +++--- .../internal/encoding/memo_table_test.go | 10 +++--- .../internal/encoding/memo_table_types.gen.go | 4 +-- .../encoding/memo_table_types.gen.go.tmpl | 2 +- .../encoding/plain_encoder_types.gen.go | 10 +++--- .../encoding/plain_encoder_types.gen.go.tmpl | 8 ++--- .../internal/encoding/typed_encoder.gen.go | 18 +++++------ .../encoding/typed_encoder.gen.go.tmpl | 14 ++++---- go/parquet/internal/encoding/types.go | 10 +++--- go/parquet/internal/encryption/aes.go | 2 +- go/parquet/internal/encryption/decryptor.go | 4 +-- go/parquet/internal/encryption/encryptor.go | 4 +-- go/parquet/internal/testutils/pagebuilder.go | 14 ++++---- .../internal/testutils/primitive_typed.go | 10 +++--- go/parquet/internal/testutils/random.go | 16 +++++----- go/parquet/internal/testutils/random_arrow.go | 8 ++--- go/parquet/internal/testutils/utils.go | 2 +- go/parquet/internal/thrift/helpers.go | 2 +- .../internal/utils/bit_benchmark_test.go | 6 ++-- .../internal/utils/bit_packing_arm64.go | 2 +- go/parquet/internal/utils/bit_reader.go | 8 ++--- go/parquet/internal/utils/bit_reader_test.go | 10 +++--- go/parquet/internal/utils/bit_writer.go | 2 +- go/parquet/internal/utils/bitmap_writer.go | 2 +- .../internal/utils/bitmap_writer_test.go | 4 +-- go/parquet/internal/utils/rle.go | 8 ++--- .../internal/utils/typed_rle_dict.gen.go | 6 ++-- .../internal/utils/typed_rle_dict.gen.go.tmpl | 6 ++-- go/parquet/metadata/app_version.go | 4 +-- go/parquet/metadata/column_chunk.go | 14 ++++---- go/parquet/metadata/file.go | 12 +++---- go/parquet/metadata/metadata_test.go | 6 ++-- go/parquet/metadata/row_group.go | 8 ++--- go/parquet/metadata/stat_compare_test.go | 4 +-- go/parquet/metadata/statistics.go | 18 +++++------ go/parquet/metadata/statistics_test.go | 12 +++---- go/parquet/metadata/statistics_types.gen.go | 18 +++++------ .../metadata/statistics_types.gen.go.tmpl | 14 ++++---- go/parquet/pqarrow/column_readers.go | 20 ++++++------ go/parquet/pqarrow/encode_arrow.go | 20 ++++++------ go/parquet/pqarrow/encode_arrow_test.go | 32 +++++++++---------- go/parquet/pqarrow/encode_dict_compute.go | 16 +++++----- go/parquet/pqarrow/encode_dict_nocompute.go | 4 +-- go/parquet/pqarrow/encode_dictionary_test.go | 16 +++++----- go/parquet/pqarrow/file_reader.go | 14 ++++---- go/parquet/pqarrow/file_reader_test.go | 16 +++++----- go/parquet/pqarrow/file_writer.go | 12 +++---- go/parquet/pqarrow/file_writer_test.go | 10 +++--- go/parquet/pqarrow/helpers.go | 2 +- go/parquet/pqarrow/path_builder.go | 10 +++--- go/parquet/pqarrow/path_builder_test.go | 8 ++--- go/parquet/pqarrow/properties.go | 6 ++-- go/parquet/pqarrow/reader_writer_test.go | 12 +++---- go/parquet/pqarrow/schema.go | 18 +++++------ go/parquet/pqarrow/schema_test.go | 18 +++++------ go/parquet/reader_properties.go | 4 +-- go/parquet/reader_writer_properties_test.go | 6 ++-- go/parquet/schema/column.go | 4 +-- go/parquet/schema/converted_types.go | 2 +- go/parquet/schema/converted_types_test.go | 2 +- go/parquet/schema/helpers.go | 2 +- go/parquet/schema/helpers_test.go | 4 +-- go/parquet/schema/logical_types.go | 8 ++--- go/parquet/schema/logical_types_test.go | 6 ++-- go/parquet/schema/node.go | 4 +-- go/parquet/schema/reflection.go | 6 ++-- go/parquet/schema/reflection_test.go | 6 ++-- go/parquet/schema/schema.go | 4 +-- go/parquet/schema/schema_element_test.go | 4 +-- go/parquet/schema/schema_flatten_test.go | 4 +-- go/parquet/schema/schema_test.go | 6 ++-- go/parquet/types.go | 4 +-- go/parquet/writer_properties.go | 6 ++-- java/adapter/avro/pom.xml | 2 +- java/adapter/jdbc/pom.xml | 2 +- java/adapter/orc/pom.xml | 2 +- java/algorithm/pom.xml | 2 +- java/bom/pom.xml | 2 +- java/c/pom.xml | 2 +- java/compression/pom.xml | 2 +- java/dataset/pom.xml | 2 +- java/flight/flight-core/pom.xml | 2 +- java/flight/flight-integration-tests/pom.xml | 2 +- java/flight/flight-sql-jdbc-core/pom.xml | 2 +- java/flight/flight-sql-jdbc-driver/pom.xml | 2 +- java/flight/flight-sql/pom.xml | 2 +- java/flight/pom.xml | 2 +- java/format/pom.xml | 2 +- java/gandiva/pom.xml | 2 +- .../module-info-compiler-maven-plugin/pom.xml | 2 +- java/maven/pom.xml | 2 +- java/memory/memory-core/pom.xml | 2 +- java/memory/memory-netty-buffer-patch/pom.xml | 2 +- java/memory/memory-netty/pom.xml | 2 +- java/memory/memory-unsafe/pom.xml | 2 +- java/memory/pom.xml | 2 +- java/performance/pom.xml | 2 +- java/pom.xml | 2 +- java/tools/pom.xml | 2 +- java/vector/pom.xml | 2 +- js/package.json | 2 +- matlab/CMakeLists.txt | 2 +- python/CMakeLists.txt | 2 +- python/setup.py | 2 +- r/DESCRIPTION | 2 +- r/NEWS.md | 4 ++- r/pkgdown/assets/versions.json | 8 +++-- ruby/red-arrow-cuda/lib/arrow-cuda/version.rb | 2 +- .../lib/arrow-dataset/version.rb | 2 +- .../lib/arrow-flight-sql/version.rb | 2 +- .../lib/arrow-flight/version.rb | 2 +- ruby/red-arrow/lib/arrow/version.rb | 2 +- ruby/red-gandiva/lib/gandiva/version.rb | 2 +- ruby/red-parquet/lib/parquet/version.rb | 2 +- 522 files changed, 1793 insertions(+), 1782 deletions(-) diff --git a/c_glib/meson.build b/c_glib/meson.build index 7c495d2567d72..ffd41d4d574a7 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -24,7 +24,7 @@ project('arrow-glib', 'c', 'cpp', 'cpp_std=c++17', ]) -version = '15.0.0-SNAPSHOT' +version = '16.0.0-SNAPSHOT' if version.endswith('-SNAPSHOT') version_numbers = version.split('-')[0].split('.') version_tag = version.split('-')[1] diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 674acc99f54a9..50d4fc28c58f3 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -18,7 +18,7 @@ _realname=arrow pkgbase=mingw-w64-${_realname} pkgname="${MINGW_PACKAGE_PREFIX}-${_realname}" -pkgver=14.0.2.9000 +pkgver=15.0.0.9000 pkgrel=8000 pkgdesc="Apache Arrow is a cross-language development platform for in-memory data (mingw-w64)" arch=("any") diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d26e06a146b56..016cd8a1b9ec8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -71,7 +71,7 @@ if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() -set(ARROW_VERSION "15.0.0-SNAPSHOT") +set(ARROW_VERSION "16.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}") diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index c0bf5dce50e32..a0f0aa1008dcd 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -1,6 +1,6 @@ { "name": "arrow", - "version-string": "15.0.0-SNAPSHOT", + "version-string": "16.0.0-SNAPSHOT", "dependencies": [ "abseil", { diff --git a/csharp/Directory.Build.props b/csharp/Directory.Build.props index ae6edda0e2f0e..7cc4b35f2a614 100644 --- a/csharp/Directory.Build.props +++ b/csharp/Directory.Build.props @@ -29,7 +29,7 @@ Apache Arrow library Copyright 2016-2019 The Apache Software Foundation The Apache Software Foundation - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT diff --git a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb index e29354def4c1c..f47c4d737f1a0 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow-glib.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow-glib.rb @@ -29,7 +29,7 @@ class ApacheArrowGlib < Formula desc "GLib bindings for Apache Arrow" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-15.0.0-SNAPSHOT/apache-arrow-15.0.0-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-16.0.0-SNAPSHOT/apache-arrow-16.0.0-SNAPSHOT.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index 14d229b477dc8..881ca50fdecf3 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -29,7 +29,7 @@ class ApacheArrow < Formula desc "Columnar in-memory analytics layer designed to accelerate big data" homepage "https://arrow.apache.org/" - url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-15.0.0-SNAPSHOT/apache-arrow-15.0.0-SNAPSHOT.tar.gz" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-16.0.0-SNAPSHOT/apache-arrow-16.0.0-SNAPSHOT.tar.gz" sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" license "Apache-2.0" head "https://github.com/apache/arrow.git", branch: "main" diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json index 10e179420b803..d647709cda723 100644 --- a/docs/source/_static/versions.json +++ b/docs/source/_static/versions.json @@ -1,15 +1,20 @@ [ { - "name": "15.0 (dev)", + "name": "16.0 (dev)", "version": "dev/", "url": "https://arrow.apache.org/docs/dev/" }, { - "name": "14.0 (stable)", + "name": "15.0 (stable)", "version": "", "url": "https://arrow.apache.org/docs/", "preferred": true }, + { + "name": "14.0", + "version": "14.0/", + "url": "https://arrow.apache.org/docs/14.0/" + }, { "name": "13.0", "version": "13.0/", diff --git a/go/README.md b/go/README.md index c45bcd756f81c..4a9e151ddf234 100644 --- a/go/README.md +++ b/go/README.md @@ -20,7 +20,7 @@ Apache Arrow for Go =================== -[![Go Reference](https://pkg.go.dev/badge/github.com/apache/arrow/go/v15.svg)](https://pkg.go.dev/github.com/apache/arrow/go/v15) +[![Go Reference](https://pkg.go.dev/badge/github.com/apache/arrow/go/v16.svg)](https://pkg.go.dev/github.com/apache/arrow/go/v16) [Apache Arrow][arrow] is a cross-language development platform for in-memory data. It specifies a standardized language-independent columnar memory format diff --git a/go/arrow/_examples/helloworld/main.go b/go/arrow/_examples/helloworld/main.go index 156a52b86da1c..9952cc9497389 100644 --- a/go/arrow/_examples/helloworld/main.go +++ b/go/arrow/_examples/helloworld/main.go @@ -19,10 +19,10 @@ package main import ( "os" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/math" + "github.com/apache/arrow/go/v16/arrow/memory" ) func main() { diff --git a/go/arrow/_tools/tmpl/main.go b/go/arrow/_tools/tmpl/main.go index c591c3b96803c..3c60e52d317dd 100644 --- a/go/arrow/_tools/tmpl/main.go +++ b/go/arrow/_tools/tmpl/main.go @@ -28,7 +28,7 @@ import ( "strings" "text/template" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/internal/json" ) const Ext = ".tmpl" diff --git a/go/arrow/array.go b/go/arrow/array.go index eed859cf46649..ca83989ab9bfc 100644 --- a/go/arrow/array.go +++ b/go/arrow/array.go @@ -19,8 +19,8 @@ package arrow import ( "fmt" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // ArrayData is the underlying memory and metadata of an Arrow array, corresponding diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index 5aacc8f99a4ee..c8d82cc79d32d 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -19,9 +19,9 @@ package array import ( "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) const ( diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go index bbfbee83585da..9e422bb9d545f 100644 --- a/go/arrow/array/array_test.go +++ b/go/arrow/array/array_test.go @@ -19,11 +19,11 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/binary.go b/go/arrow/array/binary.go index 9e26de7a6d820..f1023d40d6a1f 100644 --- a/go/arrow/array/binary.go +++ b/go/arrow/array/binary.go @@ -23,9 +23,9 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) type BinaryLike interface { diff --git a/go/arrow/array/binary_test.go b/go/arrow/array/binary_test.go index 8a11ec7dab8c4..5b3d6f0d8ba95 100644 --- a/go/arrow/array/binary_test.go +++ b/go/arrow/array/binary_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/binarybuilder.go b/go/arrow/array/binarybuilder.go index 21ad576508e9e..9de7921e29dfc 100644 --- a/go/arrow/array/binarybuilder.go +++ b/go/arrow/array/binarybuilder.go @@ -25,10 +25,10 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // A BinaryBuilder is used to build a Binary array using the Append methods. diff --git a/go/arrow/array/binarybuilder_test.go b/go/arrow/array/binarybuilder_test.go index 96be73da6516d..589396f2c5a04 100644 --- a/go/arrow/array/binarybuilder_test.go +++ b/go/arrow/array/binarybuilder_test.go @@ -20,9 +20,9 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go index 43bac64a4c990..ce9d4136122cd 100644 --- a/go/arrow/array/boolean.go +++ b/go/arrow/array/boolean.go @@ -21,10 +21,10 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // A type which represents an immutable sequence of boolean values. diff --git a/go/arrow/array/boolean_test.go b/go/arrow/array/boolean_test.go index bcd17ee5967d6..165fb470d6f89 100644 --- a/go/arrow/array/boolean_test.go +++ b/go/arrow/array/boolean_test.go @@ -22,8 +22,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/booleanbuilder.go b/go/arrow/array/booleanbuilder.go index cd0cffd5e43e2..b09d5f08f8401 100644 --- a/go/arrow/array/booleanbuilder.go +++ b/go/arrow/array/booleanbuilder.go @@ -23,11 +23,11 @@ import ( "strconv" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) type BooleanBuilder struct { diff --git a/go/arrow/array/booleanbuilder_test.go b/go/arrow/array/booleanbuilder_test.go index f6f6c80dab37f..25b6678155fa5 100644 --- a/go/arrow/array/booleanbuilder_test.go +++ b/go/arrow/array/booleanbuilder_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/bufferbuilder.go b/go/arrow/array/bufferbuilder.go index 13741ba8926ac..61d0b2d65ba88 100644 --- a/go/arrow/array/bufferbuilder.go +++ b/go/arrow/array/bufferbuilder.go @@ -20,10 +20,10 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" ) type bufBuilder interface { diff --git a/go/arrow/array/bufferbuilder_byte.go b/go/arrow/array/bufferbuilder_byte.go index 9b2b559ba9f68..82a132c1c3b06 100644 --- a/go/arrow/array/bufferbuilder_byte.go +++ b/go/arrow/array/bufferbuilder_byte.go @@ -16,7 +16,7 @@ package array -import "github.com/apache/arrow/go/v15/arrow/memory" +import "github.com/apache/arrow/go/v16/arrow/memory" type byteBufferBuilder struct { bufferBuilder diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go b/go/arrow/array/bufferbuilder_numeric.gen.go index a7961166c0edd..923168728a03f 100644 --- a/go/arrow/array/bufferbuilder_numeric.gen.go +++ b/go/arrow/array/bufferbuilder_numeric.gen.go @@ -19,9 +19,9 @@ package array import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" ) type int64BufferBuilder struct { diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl b/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl index 845d7ef01c89a..ae6b2d8e0dea6 100644 --- a/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl +++ b/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl @@ -17,9 +17,9 @@ package array import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" ) {{range .In}} diff --git a/go/arrow/array/bufferbuilder_numeric_test.go b/go/arrow/array/bufferbuilder_numeric_test.go index b51e9ae9207ea..4d218fd103e92 100644 --- a/go/arrow/array/bufferbuilder_numeric_test.go +++ b/go/arrow/array/bufferbuilder_numeric_test.go @@ -20,8 +20,8 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/endian" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go index 279804a1cdb9f..4751f57cc5b9c 100644 --- a/go/arrow/array/builder.go +++ b/go/arrow/array/builder.go @@ -20,10 +20,10 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) const ( diff --git a/go/arrow/array/builder_test.go b/go/arrow/array/builder_test.go index 7bec86d86cc8b..ab3507e225544 100644 --- a/go/arrow/array/builder_test.go +++ b/go/arrow/array/builder_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/compare.go b/go/arrow/array/compare.go index 372293a61d6cb..6c806b06b98cf 100644 --- a/go/arrow/array/compare.go +++ b/go/arrow/array/compare.go @@ -20,9 +20,9 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/internal/bitutils" ) // RecordEqual reports whether the two provided records are equal. diff --git a/go/arrow/array/compare_test.go b/go/arrow/array/compare_test.go index 4fc9cf50e8643..995eb35e7d250 100644 --- a/go/arrow/array/compare_test.go +++ b/go/arrow/array/compare_test.go @@ -22,11 +22,11 @@ import ( "sort" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/concat.go b/go/arrow/array/concat.go index f0bc2855eb1e4..9e0cb53abc54b 100644 --- a/go/arrow/array/concat.go +++ b/go/arrow/array/concat.go @@ -23,13 +23,13 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/encoded" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/encoded" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/utils" ) // Concatenate creates a new arrow.Array which is the concatenation of the diff --git a/go/arrow/array/concat_test.go b/go/arrow/array/concat_test.go index a3686b45700aa..23d0d4b5ca8e7 100644 --- a/go/arrow/array/concat_test.go +++ b/go/arrow/array/concat_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/array/data.go b/go/arrow/array/data.go index ddd9cf0c895d5..9ef1da6e14729 100644 --- a/go/arrow/array/data.go +++ b/go/arrow/array/data.go @@ -22,9 +22,9 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" ) // Data represents the memory and metadata of an Arrow array. diff --git a/go/arrow/array/data_test.go b/go/arrow/array/data_test.go index dd4793a7cdbfa..63af01ffdad2d 100644 --- a/go/arrow/array/data_test.go +++ b/go/arrow/array/data_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal128.go b/go/arrow/array/decimal128.go index 16a492db09c67..0dca320cda959 100644 --- a/go/arrow/array/decimal128.go +++ b/go/arrow/array/decimal128.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // A type which represents an immutable sequence of 128-bit decimal values. diff --git a/go/arrow/array/decimal128_test.go b/go/arrow/array/decimal128_test.go index 8c26d00cdc18e..836a6987df69f 100644 --- a/go/arrow/array/decimal128_test.go +++ b/go/arrow/array/decimal128_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal256.go b/go/arrow/array/decimal256.go index 8f72e414d1959..452ac96625bc8 100644 --- a/go/arrow/array/decimal256.go +++ b/go/arrow/array/decimal256.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // Decimal256 is a type that represents an immutable sequence of 256-bit decimal values. diff --git a/go/arrow/array/decimal256_test.go b/go/arrow/array/decimal256_test.go index 6085d6b5a6a59..4f0c441210643 100644 --- a/go/arrow/array/decimal256_test.go +++ b/go/arrow/array/decimal256_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/decimal_test.go b/go/arrow/array/decimal_test.go index 67900447be1cf..d63c807fa9dbf 100644 --- a/go/arrow/array/decimal_test.go +++ b/go/arrow/array/decimal_test.go @@ -21,12 +21,12 @@ import ( "math/big" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/dictionary.go b/go/arrow/array/dictionary.go index bbde4e4f1e5bd..35f0fb20492ba 100644 --- a/go/arrow/array/dictionary.go +++ b/go/arrow/array/dictionary.go @@ -25,16 +25,16 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/hashing" + "github.com/apache/arrow/go/v16/internal/json" + "github.com/apache/arrow/go/v16/internal/utils" ) // Dictionary represents the type for dictionary-encoded data with a data diff --git a/go/arrow/array/dictionary_test.go b/go/arrow/array/dictionary_test.go index f32cc9555f9bc..2986b2ae70c04 100644 --- a/go/arrow/array/dictionary_test.go +++ b/go/arrow/array/dictionary_test.go @@ -24,13 +24,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/array/diff.go b/go/arrow/array/diff.go index 6bf6372531fd7..fa1f404578c39 100644 --- a/go/arrow/array/diff.go +++ b/go/arrow/array/diff.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) // Edit represents one entry in the edit script to compare two arrays. diff --git a/go/arrow/array/diff_test.go b/go/arrow/array/diff_test.go index 17539c38d282f..341ee3ea89c61 100644 --- a/go/arrow/array/diff_test.go +++ b/go/arrow/array/diff_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" + "github.com/apache/arrow/go/v16/internal/types" ) type diffTestCase struct { diff --git a/go/arrow/array/encoded.go b/go/arrow/array/encoded.go index 8ca1416b92ab3..48ce582274b93 100644 --- a/go/arrow/array/encoded.go +++ b/go/arrow/array/encoded.go @@ -23,12 +23,12 @@ import ( "reflect" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/encoded" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/encoded" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" + "github.com/apache/arrow/go/v16/internal/utils" ) // RunEndEncoded represents an array containing two children: diff --git a/go/arrow/array/encoded_test.go b/go/arrow/array/encoded_test.go index 5bfac7a1a96e6..b19ed9c3cca3d 100644 --- a/go/arrow/array/encoded_test.go +++ b/go/arrow/array/encoded_test.go @@ -20,10 +20,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/arrow/array/extension.go b/go/arrow/array/extension.go index 021b8e7bc451b..b1d4178821c81 100644 --- a/go/arrow/array/extension.go +++ b/go/arrow/array/extension.go @@ -20,9 +20,9 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // ExtensionArray is the interface that needs to be implemented to handle diff --git a/go/arrow/array/extension_test.go b/go/arrow/array/extension_test.go index a8e2b0dfd59bb..21dbdf6bd6c6d 100644 --- a/go/arrow/array/extension_test.go +++ b/go/arrow/array/extension_test.go @@ -19,10 +19,10 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/fixed_size_list.go b/go/arrow/array/fixed_size_list.go index 5923d68590b15..29ec328401865 100644 --- a/go/arrow/array/fixed_size_list.go +++ b/go/arrow/array/fixed_size_list.go @@ -22,11 +22,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // FixedSizeList represents an immutable sequence of N array values. diff --git a/go/arrow/array/fixed_size_list_test.go b/go/arrow/array/fixed_size_list_test.go index 5c01199ddf987..64c2c65b9e2b4 100644 --- a/go/arrow/array/fixed_size_list_test.go +++ b/go/arrow/array/fixed_size_list_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/fixedsize_binary.go b/go/arrow/array/fixedsize_binary.go index 6cdaeace939fd..b5c8994373dce 100644 --- a/go/arrow/array/fixedsize_binary.go +++ b/go/arrow/array/fixedsize_binary.go @@ -22,8 +22,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/internal/json" ) // A type which represents an immutable sequence of fixed-length binary strings. diff --git a/go/arrow/array/fixedsize_binary_test.go b/go/arrow/array/fixedsize_binary_test.go index 785725537cbdd..07a5c36f71a16 100644 --- a/go/arrow/array/fixedsize_binary_test.go +++ b/go/arrow/array/fixedsize_binary_test.go @@ -21,9 +21,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestFixedSizeBinary(t *testing.T) { diff --git a/go/arrow/array/fixedsize_binarybuilder.go b/go/arrow/array/fixedsize_binarybuilder.go index 230a65fd2d352..33548dcfa1975 100644 --- a/go/arrow/array/fixedsize_binarybuilder.go +++ b/go/arrow/array/fixedsize_binarybuilder.go @@ -23,10 +23,10 @@ import ( "reflect" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods. diff --git a/go/arrow/array/fixedsize_binarybuilder_test.go b/go/arrow/array/fixedsize_binarybuilder_test.go index 8e4a0ac1e46a7..8f82e27b4b876 100644 --- a/go/arrow/array/fixedsize_binarybuilder_test.go +++ b/go/arrow/array/fixedsize_binarybuilder_test.go @@ -19,8 +19,8 @@ package array import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/float16.go b/go/arrow/array/float16.go index 4260f8e3774b4..6fe35f2401949 100644 --- a/go/arrow/array/float16.go +++ b/go/arrow/array/float16.go @@ -20,9 +20,9 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/internal/json" ) // A type which represents an immutable sequence of Float16 values. diff --git a/go/arrow/array/float16_builder.go b/go/arrow/array/float16_builder.go index 033b9fa2d8028..dafa8a5602352 100644 --- a/go/arrow/array/float16_builder.go +++ b/go/arrow/array/float16_builder.go @@ -23,12 +23,12 @@ import ( "strconv" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) type Float16Builder struct { diff --git a/go/arrow/array/float16_builder_test.go b/go/arrow/array/float16_builder_test.go index f8c5890179869..3826861bc4a4c 100644 --- a/go/arrow/array/float16_builder_test.go +++ b/go/arrow/array/float16_builder_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/interval.go b/go/arrow/array/interval.go index 2a5529f1c30f7..7af9806dc15f7 100644 --- a/go/arrow/array/interval.go +++ b/go/arrow/array/interval.go @@ -23,11 +23,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) func NewIntervalData(data arrow.ArrayData) arrow.Array { diff --git a/go/arrow/array/interval_test.go b/go/arrow/array/interval_test.go index f83fc52dfa34e..6b6be3ef7a3e4 100644 --- a/go/arrow/array/interval_test.go +++ b/go/arrow/array/interval_test.go @@ -20,9 +20,9 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/json_reader.go b/go/arrow/array/json_reader.go index 2f05d4b70dd76..c8462798ad554 100644 --- a/go/arrow/array/json_reader.go +++ b/go/arrow/array/json_reader.go @@ -22,10 +22,10 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) type Option func(config) diff --git a/go/arrow/array/json_reader_test.go b/go/arrow/array/json_reader_test.go index 7f12bf211dd04..f8398e91b562f 100644 --- a/go/arrow/array/json_reader_test.go +++ b/go/arrow/array/json_reader_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/list.go b/go/arrow/array/list.go index 9d959b5e43b78..d383a73a196a0 100644 --- a/go/arrow/array/list.go +++ b/go/arrow/array/list.go @@ -22,11 +22,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) type ListLike interface { diff --git a/go/arrow/array/list_test.go b/go/arrow/array/list_test.go index 011b5d7426b22..37f1d3f489232 100644 --- a/go/arrow/array/list_test.go +++ b/go/arrow/array/list_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/map.go b/go/arrow/array/map.go index fe07a68785067..da80ab016e727 100644 --- a/go/arrow/array/map.go +++ b/go/arrow/array/map.go @@ -20,9 +20,9 @@ import ( "bytes" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // Map represents an immutable sequence of Key/Value structs. It is a diff --git a/go/arrow/array/map_test.go b/go/arrow/array/map_test.go index a7ecc032682bc..8cd874f0aadb3 100644 --- a/go/arrow/array/map_test.go +++ b/go/arrow/array/map_test.go @@ -20,9 +20,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go index 2735a88a92cb3..19b7c15f498f1 100644 --- a/go/arrow/array/null.go +++ b/go/arrow/array/null.go @@ -23,10 +23,10 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // Null represents an immutable, degenerate array with no physical storage. diff --git a/go/arrow/array/null_test.go b/go/arrow/array/null_test.go index 5d230ec5cec71..d4855e244a083 100644 --- a/go/arrow/array/null_test.go +++ b/go/arrow/array/null_test.go @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go index 59c9a979768d5..0c7346ddf1964 100644 --- a/go/arrow/array/numeric.gen.go +++ b/go/arrow/array/numeric.gen.go @@ -23,8 +23,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/internal/json" ) // A type which represents an immutable sequence of int64 values. diff --git a/go/arrow/array/numeric.gen.go.tmpl b/go/arrow/array/numeric.gen.go.tmpl index 027456009daad..2ccb8d8b83679 100644 --- a/go/arrow/array/numeric.gen.go.tmpl +++ b/go/arrow/array/numeric.gen.go.tmpl @@ -21,8 +21,8 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/internal/json" ) {{range .In}} diff --git a/go/arrow/array/numeric_test.go b/go/arrow/array/numeric_test.go index 91dd724c8da50..34cab789259fd 100644 --- a/go/arrow/array/numeric_test.go +++ b/go/arrow/array/numeric_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go index 52b189d6ed453..5f79b87c73788 100644 --- a/go/arrow/array/numericbuilder.gen.go +++ b/go/arrow/array/numericbuilder.gen.go @@ -27,11 +27,11 @@ import ( "sync/atomic" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) type Int64Builder struct { diff --git a/go/arrow/array/numericbuilder.gen.go.tmpl b/go/arrow/array/numericbuilder.gen.go.tmpl index 82ac35465d424..95bc9accbe8ec 100644 --- a/go/arrow/array/numericbuilder.gen.go.tmpl +++ b/go/arrow/array/numericbuilder.gen.go.tmpl @@ -17,11 +17,11 @@ package array import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) {{range .In}} diff --git a/go/arrow/array/numericbuilder.gen_test.go b/go/arrow/array/numericbuilder.gen_test.go index e1f72773403d8..10751e42de21a 100644 --- a/go/arrow/array/numericbuilder.gen_test.go +++ b/go/arrow/array/numericbuilder.gen_test.go @@ -21,9 +21,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numericbuilder.gen_test.go.tmpl b/go/arrow/array/numericbuilder.gen_test.go.tmpl index eddd884e2eddf..41f68715fddbc 100644 --- a/go/arrow/array/numericbuilder.gen_test.go.tmpl +++ b/go/arrow/array/numericbuilder.gen_test.go.tmpl @@ -19,9 +19,9 @@ package array_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/record.go b/go/arrow/array/record.go index f25e7c9a874b3..b4a03410c4fbf 100644 --- a/go/arrow/array/record.go +++ b/go/arrow/array/record.go @@ -22,10 +22,10 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // RecordReader reads a stream of records. diff --git a/go/arrow/array/record_test.go b/go/arrow/array/record_test.go index 7d438d1f1f81e..c961c04af2c23 100644 --- a/go/arrow/array/record_test.go +++ b/go/arrow/array/record_test.go @@ -21,9 +21,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/string.go b/go/arrow/array/string.go index c8517ba3056df..6453bcaf7bdc5 100644 --- a/go/arrow/array/string.go +++ b/go/arrow/array/string.go @@ -23,9 +23,9 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) type StringLike interface { diff --git a/go/arrow/array/string_test.go b/go/arrow/array/string_test.go index 3df56a2675252..8006bc4740b4d 100644 --- a/go/arrow/array/string_test.go +++ b/go/arrow/array/string_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go index 5e723c9dc7553..dc6fab689735b 100644 --- a/go/arrow/array/struct.go +++ b/go/arrow/array/struct.go @@ -23,11 +23,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // Struct represents an ordered sequence of relative types. diff --git a/go/arrow/array/struct_test.go b/go/arrow/array/struct_test.go index 1b0dc5a3e4b19..895d9508dcb8b 100644 --- a/go/arrow/array/struct_test.go +++ b/go/arrow/array/struct_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/table.go b/go/arrow/array/table.go index b34650e182de4..197179b5ca4c3 100644 --- a/go/arrow/array/table.go +++ b/go/arrow/array/table.go @@ -23,8 +23,8 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) // NewColumnSlice returns a new zero-copy slice of the column with the indicated diff --git a/go/arrow/array/table_test.go b/go/arrow/array/table_test.go index 9535ae6b089bb..6ce69f66d67ac 100644 --- a/go/arrow/array/table_test.go +++ b/go/arrow/array/table_test.go @@ -22,9 +22,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestChunked(t *testing.T) { diff --git a/go/arrow/array/timestamp.go b/go/arrow/array/timestamp.go index 0cc46a127fc51..21a02dd402099 100644 --- a/go/arrow/array/timestamp.go +++ b/go/arrow/array/timestamp.go @@ -24,11 +24,11 @@ import ( "sync/atomic" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) // Timestamp represents an immutable sequence of arrow.Timestamp values. diff --git a/go/arrow/array/timestamp_test.go b/go/arrow/array/timestamp_test.go index c172ad811dc37..5a6e470c58804 100644 --- a/go/arrow/array/timestamp_test.go +++ b/go/arrow/array/timestamp_test.go @@ -20,9 +20,9 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/union.go b/go/arrow/array/union.go index 1af3e70472065..2400b9f88e0f6 100644 --- a/go/arrow/array/union.go +++ b/go/arrow/array/union.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/json" ) // Union is a convenience interface to encompass both Sparse and Dense diff --git a/go/arrow/array/union_test.go b/go/arrow/array/union_test.go index e876f5def26d3..45373b2e96d6a 100644 --- a/go/arrow/array/union_test.go +++ b/go/arrow/array/union_test.go @@ -21,9 +21,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/array/util.go b/go/arrow/array/util.go index c9b730b040611..f4781ce4e3cbc 100644 --- a/go/arrow/array/util.go +++ b/go/arrow/array/util.go @@ -22,11 +22,11 @@ import ( "io" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/hashing" + "github.com/apache/arrow/go/v16/internal/json" ) func min(a, b int) int { diff --git a/go/arrow/array/util_test.go b/go/arrow/array/util_test.go index 84a6debdf3946..262f45d356933 100644 --- a/go/arrow/array/util_test.go +++ b/go/arrow/array/util_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/arrio/arrio.go b/go/arrow/arrio/arrio.go index 51cf6dc46d44a..6825b14cf403a 100644 --- a/go/arrow/arrio/arrio.go +++ b/go/arrow/arrio/arrio.go @@ -22,7 +22,7 @@ import ( "errors" "io" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) // Reader is the interface that wraps the Read method. diff --git a/go/arrow/arrio/arrio_test.go b/go/arrow/arrio/arrio_test.go index c80d5d2569d67..e05385eed2d17 100644 --- a/go/arrow/arrio/arrio_test.go +++ b/go/arrow/arrio/arrio_test.go @@ -22,11 +22,11 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) type copyKind int diff --git a/go/arrow/avro/avro2parquet/main.go b/go/arrow/avro/avro2parquet/main.go index 45377b46a444c..f1c5eb36e9142 100644 --- a/go/arrow/avro/avro2parquet/main.go +++ b/go/arrow/avro/avro2parquet/main.go @@ -26,10 +26,10 @@ import ( "runtime/pprof" "time" - "github.com/apache/arrow/go/v15/arrow/avro" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - pq "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v16/arrow/avro" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + pq "github.com/apache/arrow/go/v16/parquet/pqarrow" ) var ( diff --git a/go/arrow/avro/reader.go b/go/arrow/avro/reader.go index e72a5632bdd6e..19a9023c57a68 100644 --- a/go/arrow/avro/reader.go +++ b/go/arrow/avro/reader.go @@ -23,10 +23,10 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/hamba/avro/v2/ocf" "github.com/tidwall/sjson" diff --git a/go/arrow/avro/reader_test.go b/go/arrow/avro/reader_test.go index e94d4f48fb933..375c653eee85b 100644 --- a/go/arrow/avro/reader_test.go +++ b/go/arrow/avro/reader_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" hamba "github.com/hamba/avro/v2" ) diff --git a/go/arrow/avro/reader_types.go b/go/arrow/avro/reader_types.go index 974fea1f14e5a..484155a43a794 100644 --- a/go/arrow/avro/reader_types.go +++ b/go/arrow/avro/reader_types.go @@ -23,12 +23,12 @@ import ( "fmt" "math/big" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" ) type dataLoader struct { diff --git a/go/arrow/avro/schema.go b/go/arrow/avro/schema.go index 32e37096c68f2..c9bb4388b8c15 100644 --- a/go/arrow/avro/schema.go +++ b/go/arrow/avro/schema.go @@ -22,9 +22,9 @@ import ( "math" "strconv" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/internal/types" avro "github.com/hamba/avro/v2" ) diff --git a/go/arrow/avro/schema_test.go b/go/arrow/avro/schema_test.go index 08a3fe1ed7440..ca97debfbad4c 100644 --- a/go/arrow/avro/schema_test.go +++ b/go/arrow/avro/schema_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" hamba "github.com/hamba/avro/v2" ) diff --git a/go/arrow/bitutil/bitmaps.go b/go/arrow/bitutil/bitmaps.go index 887a1920bc933..3e211c213014b 100644 --- a/go/arrow/bitutil/bitmaps.go +++ b/go/arrow/bitutil/bitmaps.go @@ -22,9 +22,9 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" ) // BitmapReader is a simple bitmap reader for a byte slice. diff --git a/go/arrow/bitutil/bitmaps_test.go b/go/arrow/bitutil/bitmaps_test.go index c926bff39e09a..918db3c7ad437 100644 --- a/go/arrow/bitutil/bitmaps_test.go +++ b/go/arrow/bitutil/bitmaps_test.go @@ -22,8 +22,8 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/bitutil/bitutil.go b/go/arrow/bitutil/bitutil.go index dc510a8b374c4..82747ee1417b8 100644 --- a/go/arrow/bitutil/bitutil.go +++ b/go/arrow/bitutil/bitutil.go @@ -22,7 +22,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/memory" ) var ( diff --git a/go/arrow/bitutil/bitutil_test.go b/go/arrow/bitutil/bitutil_test.go index 189c8541f4925..752da59a0c7b6 100644 --- a/go/arrow/bitutil/bitutil_test.go +++ b/go/arrow/bitutil/bitutil_test.go @@ -21,8 +21,8 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/testing/tools" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/cdata/cdata.go b/go/arrow/cdata/cdata.go index 64cc8456e8153..7e8f678f7e2d9 100644 --- a/go/arrow/cdata/cdata.go +++ b/go/arrow/cdata/cdata.go @@ -46,10 +46,10 @@ import ( "syscall" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/cdata/cdata_exports.go b/go/arrow/cdata/cdata_exports.go index 9c7c238ffb7b4..d59c87712eedf 100644 --- a/go/arrow/cdata/cdata_exports.go +++ b/go/arrow/cdata/cdata_exports.go @@ -45,11 +45,11 @@ import ( "strings" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/internal" + "github.com/apache/arrow/go/v16/arrow/ipc" ) func encodeCMetadata(keys, values []string) []byte { diff --git a/go/arrow/cdata/cdata_test.go b/go/arrow/cdata/cdata_test.go index 607cfe53217a6..354e560ea31ed 100644 --- a/go/arrow/cdata/cdata_test.go +++ b/go/arrow/cdata/cdata_test.go @@ -35,12 +35,12 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/cdata/cdata_test_framework.go b/go/arrow/cdata/cdata_test_framework.go index 1251b20201e41..2a4eec9b0e782 100644 --- a/go/arrow/cdata/cdata_test_framework.go +++ b/go/arrow/cdata/cdata_test_framework.go @@ -69,10 +69,10 @@ import ( "runtime/cgo" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal" + "github.com/apache/arrow/go/v16/arrow/memory/mallocator" ) const ( diff --git a/go/arrow/cdata/exports.go b/go/arrow/cdata/exports.go index 7353df62d113a..f499908976e59 100644 --- a/go/arrow/cdata/exports.go +++ b/go/arrow/cdata/exports.go @@ -20,8 +20,8 @@ import ( "runtime/cgo" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" ) // #include diff --git a/go/arrow/cdata/import_allocator.go b/go/arrow/cdata/import_allocator.go index cf1c6a961ff37..2ba8429cdbc7f 100644 --- a/go/arrow/cdata/import_allocator.go +++ b/go/arrow/cdata/import_allocator.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) // #include "arrow/c/helpers.h" diff --git a/go/arrow/cdata/interface.go b/go/arrow/cdata/interface.go index d55a068aa1564..3d9e7cb8ddd22 100644 --- a/go/arrow/cdata/interface.go +++ b/go/arrow/cdata/interface.go @@ -22,10 +22,10 @@ package cdata import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/cdata/test/test_cimport.go b/go/arrow/cdata/test/test_cimport.go index 147c3691f0c71..8fd161c225eef 100644 --- a/go/arrow/cdata/test/test_cimport.go +++ b/go/arrow/cdata/test/test_cimport.go @@ -23,10 +23,10 @@ import ( "fmt" "runtime" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/cdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/cdata" + "github.com/apache/arrow/go/v16/arrow/memory" ) // #include diff --git a/go/arrow/compute/arithmetic.go b/go/arrow/compute/arithmetic.go index 052d79610bcba..ebc3124e43688 100644 --- a/go/arrow/compute/arithmetic.go +++ b/go/arrow/compute/arithmetic.go @@ -22,12 +22,12 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/scalar" ) type ( diff --git a/go/arrow/compute/arithmetic_test.go b/go/arrow/compute/arithmetic_test.go index 34c1bc6d98d65..f39d5a45a4c30 100644 --- a/go/arrow/compute/arithmetic_test.go +++ b/go/arrow/compute/arithmetic_test.go @@ -26,16 +26,16 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/klauspost/cpuid/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/arrow/compute/cast.go b/go/arrow/compute/cast.go index 385c0e6858968..f4b4a039ffc3c 100644 --- a/go/arrow/compute/cast.go +++ b/go/arrow/compute/cast.go @@ -23,11 +23,11 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/cast_test.go b/go/arrow/compute/cast_test.go index 10957a45167aa..149ff19ca4ef6 100644 --- a/go/arrow/compute/cast_test.go +++ b/go/arrow/compute/cast_test.go @@ -26,16 +26,16 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/datum.go b/go/arrow/compute/datum.go index 388cfa10156d5..43ed13fcfcc69 100644 --- a/go/arrow/compute/datum.go +++ b/go/arrow/compute/datum.go @@ -21,9 +21,9 @@ package compute import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/scalar" ) //go:generate go run golang.org/x/tools/cmd/stringer -type=DatumKind -linecomment diff --git a/go/arrow/compute/example_test.go b/go/arrow/compute/example_test.go index e2b07b7e191ae..e2cc91c4b1aa9 100644 --- a/go/arrow/compute/example_test.go +++ b/go/arrow/compute/example_test.go @@ -23,11 +23,11 @@ import ( "fmt" "log" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/memory" ) // This example demonstrates how to register a custom scalar function. diff --git a/go/arrow/compute/exec.go b/go/arrow/compute/exec.go index eba47e64bd509..31e7657b5a3aa 100644 --- a/go/arrow/compute/exec.go +++ b/go/arrow/compute/exec.go @@ -22,9 +22,9 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) func haveChunkedArray(values []Datum) bool { diff --git a/go/arrow/compute/exec/kernel.go b/go/arrow/compute/exec/kernel.go index 766857f63e565..20cc617a1ae27 100644 --- a/go/arrow/compute/exec/kernel.go +++ b/go/arrow/compute/exec/kernel.go @@ -24,10 +24,10 @@ import ( "hash/maphash" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/exec/kernel_test.go b/go/arrow/compute/exec/kernel_test.go index 4df6b42ff9408..a07b993c6adf1 100644 --- a/go/arrow/compute/exec/kernel_test.go +++ b/go/arrow/compute/exec/kernel_test.go @@ -22,12 +22,12 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec/span.go b/go/arrow/compute/exec/span.go index 55753de9e0e73..6f9bb240e3469 100644 --- a/go/arrow/compute/exec/span.go +++ b/go/arrow/compute/exec/span.go @@ -23,11 +23,11 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" ) // BufferSpan is a lightweight Buffer holder for ArraySpans that does not diff --git a/go/arrow/compute/exec/span_test.go b/go/arrow/compute/exec/span_test.go index 474c005b44642..a814d150831d6 100644 --- a/go/arrow/compute/exec/span_test.go +++ b/go/arrow/compute/exec/span_test.go @@ -24,14 +24,14 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec/utils.go b/go/arrow/compute/exec/utils.go index 1b5e69a502cfd..868161010bf59 100644 --- a/go/arrow/compute/exec/utils.go +++ b/go/arrow/compute/exec/utils.go @@ -24,10 +24,10 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/exp/constraints" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/exec/utils_test.go b/go/arrow/compute/exec/utils_test.go index 345d6dcf3b4c4..2504286b5e240 100644 --- a/go/arrow/compute/exec/utils_test.go +++ b/go/arrow/compute/exec/utils_test.go @@ -21,10 +21,10 @@ package exec_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exec_internals_test.go b/go/arrow/compute/exec_internals_test.go index bae32268862ff..0a6be416076fc 100644 --- a/go/arrow/compute/exec_internals_test.go +++ b/go/arrow/compute/exec_internals_test.go @@ -24,13 +24,13 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/exec_test.go b/go/arrow/compute/exec_test.go index a37f67c03e8ce..0fbfa55e548f0 100644 --- a/go/arrow/compute/exec_test.go +++ b/go/arrow/compute/exec_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/executor.go b/go/arrow/compute/executor.go index b8144a4a8de87..53c9ec155528b 100644 --- a/go/arrow/compute/executor.go +++ b/go/arrow/compute/executor.go @@ -25,14 +25,14 @@ import ( "runtime" "sync" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" ) // ExecCtx holds simple contextual information for execution diff --git a/go/arrow/compute/expression.go b/go/arrow/compute/expression.go index fbb6c502d98ab..a81cb24b5a575 100644 --- a/go/arrow/compute/expression.go +++ b/go/arrow/compute/expression.go @@ -28,14 +28,14 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" ) var hashSeed = maphash.MakeSeed() diff --git a/go/arrow/compute/expression_test.go b/go/arrow/compute/expression_test.go index b3b44ae1faa61..8964cb8af0696 100644 --- a/go/arrow/compute/expression_test.go +++ b/go/arrow/compute/expression_test.go @@ -22,11 +22,11 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/exprs/builders.go b/go/arrow/compute/exprs/builders.go index ebe1d4c27385b..8b273a7a6bb1d 100644 --- a/go/arrow/compute/exprs/builders.go +++ b/go/arrow/compute/exprs/builders.go @@ -25,8 +25,8 @@ import ( "strings" "unicode" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/exprs/builders_test.go b/go/arrow/compute/exprs/builders_test.go index 69501622359d5..d78eb7f1c7cda 100644 --- a/go/arrow/compute/exprs/builders_test.go +++ b/go/arrow/compute/exprs/builders_test.go @@ -21,8 +21,8 @@ package exprs_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exprs" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exprs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/substrait-io/substrait-go/expr" diff --git a/go/arrow/compute/exprs/exec.go b/go/arrow/compute/exprs/exec.go index d43703f5c1c8b..6a780be32f019 100644 --- a/go/arrow/compute/exprs/exec.go +++ b/go/arrow/compute/exprs/exec.go @@ -23,15 +23,15 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/exprs/exec_internal_test.go b/go/arrow/compute/exprs/exec_internal_test.go index 680bf36f11958..2525d1c8c7cae 100644 --- a/go/arrow/compute/exprs/exec_internal_test.go +++ b/go/arrow/compute/exprs/exec_internal_test.go @@ -23,10 +23,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/exprs/exec_test.go b/go/arrow/compute/exprs/exec_test.go index 65cafc8e62ddb..cf7aac9a9b2e8 100644 --- a/go/arrow/compute/exprs/exec_test.go +++ b/go/arrow/compute/exprs/exec_test.go @@ -23,12 +23,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exprs" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exprs" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/substrait-io/substrait-go/expr" diff --git a/go/arrow/compute/exprs/extension_types.go b/go/arrow/compute/exprs/extension_types.go index db7992b8f089f..24031eea66def 100644 --- a/go/arrow/compute/exprs/extension_types.go +++ b/go/arrow/compute/exprs/extension_types.go @@ -24,8 +24,8 @@ import ( "reflect" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" ) type simpleExtensionTypeFactory[P comparable] struct { diff --git a/go/arrow/compute/exprs/field_refs.go b/go/arrow/compute/exprs/field_refs.go index e95e3c8c9abb9..941159bd47443 100644 --- a/go/arrow/compute/exprs/field_refs.go +++ b/go/arrow/compute/exprs/field_refs.go @@ -21,11 +21,11 @@ package exprs import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/substrait-io/substrait-go/expr" ) diff --git a/go/arrow/compute/exprs/types.go b/go/arrow/compute/exprs/types.go index 87e08233cc00a..dee0ad9eb91ab 100644 --- a/go/arrow/compute/exprs/types.go +++ b/go/arrow/compute/exprs/types.go @@ -24,8 +24,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute" "github.com/substrait-io/substrait-go/expr" "github.com/substrait-io/substrait-go/extensions" "github.com/substrait-io/substrait-go/types" diff --git a/go/arrow/compute/fieldref.go b/go/arrow/compute/fieldref.go index 036e1e355ed75..ab6d856f85f0d 100644 --- a/go/arrow/compute/fieldref.go +++ b/go/arrow/compute/fieldref.go @@ -27,8 +27,8 @@ import ( "unicode" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" ) var ( diff --git a/go/arrow/compute/fieldref_test.go b/go/arrow/compute/fieldref_test.go index 72985012e4a92..1ddb3267ff1b3 100644 --- a/go/arrow/compute/fieldref_test.go +++ b/go/arrow/compute/fieldref_test.go @@ -19,10 +19,10 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/compute/functions.go b/go/arrow/compute/functions.go index f35d9facaf2d3..e1c7b66d822e7 100644 --- a/go/arrow/compute/functions.go +++ b/go/arrow/compute/functions.go @@ -23,8 +23,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" ) type Function interface { diff --git a/go/arrow/compute/functions_test.go b/go/arrow/compute/functions_test.go index 1e6bbd598e0c9..f07b5dc1eb589 100644 --- a/go/arrow/compute/functions_test.go +++ b/go/arrow/compute/functions_test.go @@ -21,8 +21,8 @@ package compute_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic.go b/go/arrow/compute/internal/kernels/base_arithmetic.go index b795c04c39ead..009251da87dc7 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic.go @@ -24,11 +24,11 @@ import ( "math/bits" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/internal/debug" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go index 51b1866fb68fa..7800a3abd7bb3 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_amd64.go @@ -21,9 +21,9 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/debug" "golang.org/x/exp/constraints" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go index 6814e834b4bf3..b49ebf0060e01 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go b/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go index 633ec5f4f18e5..da02db2c90f78 100644 --- a/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/base_arithmetic_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go b/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go index 2c1559fe0f0fd..f77408c51d756 100644 --- a/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go +++ b/go/arrow/compute/internal/kernels/basic_arithmetic_noasm.go @@ -19,8 +19,8 @@ package kernels import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/boolean_cast.go b/go/arrow/compute/internal/kernels/boolean_cast.go index 6109d25790940..30ff952795180 100644 --- a/go/arrow/compute/internal/kernels/boolean_cast.go +++ b/go/arrow/compute/internal/kernels/boolean_cast.go @@ -22,9 +22,9 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" ) func isNonZero[T arrow.FixedWidthType](ctx *exec.KernelCtx, in []T, out []byte) error { diff --git a/go/arrow/compute/internal/kernels/cast.go b/go/arrow/compute/internal/kernels/cast.go index bc4ee3abd128c..a2676c7f7182a 100644 --- a/go/arrow/compute/internal/kernels/cast.go +++ b/go/arrow/compute/internal/kernels/cast.go @@ -19,9 +19,9 @@ package kernels import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute/exec" ) type CastOptions struct { diff --git a/go/arrow/compute/internal/kernels/cast_numeric.go b/go/arrow/compute/internal/kernels/cast_numeric.go index 2e893c7205f6a..b77f2177a04f3 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric.go +++ b/go/arrow/compute/internal/kernels/cast_numeric.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) var castNumericUnsafe func(itype, otype arrow.Type, in, out []byte, len int) = castNumericGo diff --git a/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go b/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go index eafa4b41001ae..1609983740d00 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go b/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go index 75ad79fd0e7f3..7e00d3075cfcb 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_neon_arm64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go b/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go index d04f393c4c099..4f0f6c2bc424e 100644 --- a/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/cast_numeric_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/cast_temporal.go b/go/arrow/compute/internal/kernels/cast_temporal.go index 48e2bfb6cada1..3e6edb6a38918 100644 --- a/go/arrow/compute/internal/kernels/cast_temporal.go +++ b/go/arrow/compute/internal/kernels/cast_temporal.go @@ -24,10 +24,10 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) const millisecondsInDay = 86400000 diff --git a/go/arrow/compute/internal/kernels/helpers.go b/go/arrow/compute/internal/kernels/helpers.go index 1ac09ba43bfb5..d47885d2f4084 100644 --- a/go/arrow/compute/internal/kernels/helpers.go +++ b/go/arrow/compute/internal/kernels/helpers.go @@ -22,13 +22,13 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" + "github.com/apache/arrow/go/v16/internal/bitutils" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/numeric_cast.go b/go/arrow/compute/internal/kernels/numeric_cast.go index d31edfdd3087c..4667abd4751ae 100644 --- a/go/arrow/compute/internal/kernels/numeric_cast.go +++ b/go/arrow/compute/internal/kernels/numeric_cast.go @@ -23,13 +23,13 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/internal/bitutils" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/rounding.go b/go/arrow/compute/internal/kernels/rounding.go index 345c779085fe8..e9d547d6f1e9f 100644 --- a/go/arrow/compute/internal/kernels/rounding.go +++ b/go/arrow/compute/internal/kernels/rounding.go @@ -22,11 +22,11 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/scalar" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/compute/internal/kernels/scalar_arithmetic.go b/go/arrow/compute/internal/kernels/scalar_arithmetic.go index f1ed21065e404..14bf36ee7ba07 100644 --- a/go/arrow/compute/internal/kernels/scalar_arithmetic.go +++ b/go/arrow/compute/internal/kernels/scalar_arithmetic.go @@ -22,13 +22,13 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/scalar" ) // scalar kernel that ignores (assumed all-null inputs) and returns null diff --git a/go/arrow/compute/internal/kernels/scalar_boolean.go b/go/arrow/compute/internal/kernels/scalar_boolean.go index 0707c92e6a198..6005d45c102fb 100644 --- a/go/arrow/compute/internal/kernels/scalar_boolean.go +++ b/go/arrow/compute/internal/kernels/scalar_boolean.go @@ -19,9 +19,9 @@ package kernels import ( - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/scalar" ) type computeWordFN func(leftTrue, leftFalse, rightTrue, rightFalse uint64) (outValid, outData uint64) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go index 52cd2c31a2aa4..93862ce617eae 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" "golang.org/x/sys/cpu" ) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go index cf9fc1eeedbb8..3f0066dc59159 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_avx2_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go b/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go index b36524baa126b..a4811981953bd 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_noasm.go @@ -18,7 +18,7 @@ package kernels -import "github.com/apache/arrow/go/v15/arrow" +import "github.com/apache/arrow/go/v16/arrow" func genCompareKernel[T arrow.NumericType](op CompareOperator) *CompareData { return genGoCompareKernel(getCmpOp[T](op)) diff --git a/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go b/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go index f8b36a1e4be76..a47668c3e725c 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go +++ b/go/arrow/compute/internal/kernels/scalar_comparison_sse4_amd64.go @@ -21,7 +21,7 @@ package kernels import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) //go:noescape diff --git a/go/arrow/compute/internal/kernels/scalar_comparisons.go b/go/arrow/compute/internal/kernels/scalar_comparisons.go index 29e6db29cb267..1c8418020afa1 100644 --- a/go/arrow/compute/internal/kernels/scalar_comparisons.go +++ b/go/arrow/compute/internal/kernels/scalar_comparisons.go @@ -23,14 +23,14 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/scalar" + "github.com/apache/arrow/go/v16/internal/bitutils" ) type binaryKernel func(left, right, out []byte, offset int) diff --git a/go/arrow/compute/internal/kernels/string_casts.go b/go/arrow/compute/internal/kernels/string_casts.go index d9cf52320b3aa..21bfe7bd87e00 100644 --- a/go/arrow/compute/internal/kernels/string_casts.go +++ b/go/arrow/compute/internal/kernels/string_casts.go @@ -23,12 +23,12 @@ import ( "strconv" "unicode/utf8" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/internal/bitutils" ) func validateUtf8Fsb(input *exec.ArraySpan) error { diff --git a/go/arrow/compute/internal/kernels/types.go b/go/arrow/compute/internal/kernels/types.go index 481eab36059f8..0237e1a9994b6 100644 --- a/go/arrow/compute/internal/kernels/types.go +++ b/go/arrow/compute/internal/kernels/types.go @@ -21,10 +21,10 @@ package kernels import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/scalar" ) var ( diff --git a/go/arrow/compute/internal/kernels/vector_hash.go b/go/arrow/compute/internal/kernels/vector_hash.go index f6c9a7f39db93..5c9ed0a8c712e 100644 --- a/go/arrow/compute/internal/kernels/vector_hash.go +++ b/go/arrow/compute/internal/kernels/vector_hash.go @@ -21,13 +21,13 @@ package kernels import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/hashing" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/hashing" ) type HashState interface { diff --git a/go/arrow/compute/internal/kernels/vector_run_end_encode.go b/go/arrow/compute/internal/kernels/vector_run_end_encode.go index 017b9712025b7..6275a61510993 100644 --- a/go/arrow/compute/internal/kernels/vector_run_end_encode.go +++ b/go/arrow/compute/internal/kernels/vector_run_end_encode.go @@ -24,14 +24,14 @@ import ( "sort" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" ) type RunEndEncodeState struct { diff --git a/go/arrow/compute/internal/kernels/vector_selection.go b/go/arrow/compute/internal/kernels/vector_selection.go index f08bb4100bf88..7734b7b123eaa 100644 --- a/go/arrow/compute/internal/kernels/vector_selection.go +++ b/go/arrow/compute/internal/kernels/vector_selection.go @@ -22,13 +22,13 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" ) type NullSelectionBehavior int8 diff --git a/go/arrow/compute/registry.go b/go/arrow/compute/registry.go index 379e0ccbe86a9..dc89fe1a621e0 100644 --- a/go/arrow/compute/registry.go +++ b/go/arrow/compute/registry.go @@ -21,7 +21,7 @@ package compute import ( "sync" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/registry_test.go b/go/arrow/compute/registry_test.go index b725091090434..16e51514f8b2a 100644 --- a/go/arrow/compute/registry_test.go +++ b/go/arrow/compute/registry_test.go @@ -23,9 +23,9 @@ import ( "errors" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exec" "github.com/stretchr/testify/assert" "golang.org/x/exp/slices" ) diff --git a/go/arrow/compute/scalar_bool.go b/go/arrow/compute/scalar_bool.go index 1f28a6e2bfcb2..a7e85dfeb479c 100644 --- a/go/arrow/compute/scalar_bool.go +++ b/go/arrow/compute/scalar_bool.go @@ -21,9 +21,9 @@ package compute import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/scalar_bool_test.go b/go/arrow/compute/scalar_bool_test.go index bd4f3c5c0df2b..dd9bfbbd8a689 100644 --- a/go/arrow/compute/scalar_bool_test.go +++ b/go/arrow/compute/scalar_bool_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/compute/scalar_compare.go b/go/arrow/compute/scalar_compare.go index 24a4191a10999..6150e610b9a3d 100644 --- a/go/arrow/compute/scalar_compare.go +++ b/go/arrow/compute/scalar_compare.go @@ -21,9 +21,9 @@ package compute import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" ) type compareFunction struct { diff --git a/go/arrow/compute/scalar_compare_test.go b/go/arrow/compute/scalar_compare_test.go index 1fa0591692ecb..0e08fc11dcbca 100644 --- a/go/arrow/compute/scalar_compare_test.go +++ b/go/arrow/compute/scalar_compare_test.go @@ -24,15 +24,15 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/selection.go b/go/arrow/compute/selection.go index 90bc5280ef2a7..77d76b95863c2 100644 --- a/go/arrow/compute/selection.go +++ b/go/arrow/compute/selection.go @@ -22,10 +22,10 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" "golang.org/x/sync/errgroup" ) diff --git a/go/arrow/compute/utils.go b/go/arrow/compute/utils.go index 9de6523fd0b5f..9c9b63955eb02 100644 --- a/go/arrow/compute/utils.go +++ b/go/arrow/compute/utils.go @@ -24,12 +24,12 @@ import ( "math" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/compute/vector_hash.go b/go/arrow/compute/vector_hash.go index 144c123380fe2..e7d7aaa5a7562 100644 --- a/go/arrow/compute/vector_hash.go +++ b/go/arrow/compute/vector_hash.go @@ -21,8 +21,8 @@ package compute import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/vector_hash_test.go b/go/arrow/compute/vector_hash_test.go index c37db584805d0..ba604f31e9fca 100644 --- a/go/arrow/compute/vector_hash_test.go +++ b/go/arrow/compute/vector_hash_test.go @@ -23,12 +23,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/compute/vector_run_end_test.go b/go/arrow/compute/vector_run_end_test.go index 51c0f834ceb6a..2c4170e07cd1c 100644 --- a/go/arrow/compute/vector_run_end_test.go +++ b/go/arrow/compute/vector_run_end_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/compute/vector_run_ends.go b/go/arrow/compute/vector_run_ends.go index 3e47c67de3c35..8ce369241739f 100644 --- a/go/arrow/compute/vector_run_ends.go +++ b/go/arrow/compute/vector_run_ends.go @@ -21,8 +21,8 @@ package compute import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" ) var ( diff --git a/go/arrow/compute/vector_selection_test.go b/go/arrow/compute/vector_selection_test.go index 4e38bc995cdfc..c49f751fd0526 100644 --- a/go/arrow/compute/vector_selection_test.go +++ b/go/arrow/compute/vector_selection_test.go @@ -24,15 +24,15 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/compute/exec" - "github.com/apache/arrow/go/v15/arrow/compute/internal/kernels" - "github.com/apache/arrow/go/v15/arrow/internal/testing/gen" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/compute/exec" + "github.com/apache/arrow/go/v16/arrow/compute/internal/kernels" + "github.com/apache/arrow/go/v16/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/csv/common.go b/go/arrow/csv/common.go index 31ca61f323d36..173d8559c7e26 100644 --- a/go/arrow/csv/common.go +++ b/go/arrow/csv/common.go @@ -23,8 +23,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" ) var ( diff --git a/go/arrow/csv/reader.go b/go/arrow/csv/reader.go index e58b426d837fb..05e79e6500fc4 100644 --- a/go/arrow/csv/reader.go +++ b/go/arrow/csv/reader.go @@ -29,13 +29,13 @@ import ( "time" "unicode/utf8" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" ) // Reader wraps encoding/csv.Reader and creates array.Records from a schema. diff --git a/go/arrow/csv/reader_test.go b/go/arrow/csv/reader_test.go index dfcb6625bd7cc..7e131c6dfe0b7 100644 --- a/go/arrow/csv/reader_test.go +++ b/go/arrow/csv/reader_test.go @@ -25,13 +25,13 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/csv" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/csv" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/csv/transformer.go b/go/arrow/csv/transformer.go index 78b16446d4def..6f66fdb474f63 100644 --- a/go/arrow/csv/transformer.go +++ b/go/arrow/csv/transformer.go @@ -25,8 +25,8 @@ import ( "math/big" "strconv" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" ) func (w *Writer) transformColToStringArr(typ arrow.DataType, col arrow.Array, stringsReplacer func(string)string) []string { diff --git a/go/arrow/csv/writer.go b/go/arrow/csv/writer.go index b939b72984b0f..d99f4c86c3785 100644 --- a/go/arrow/csv/writer.go +++ b/go/arrow/csv/writer.go @@ -22,7 +22,7 @@ import ( "strconv" "sync" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) // Writer wraps encoding/csv.Writer and writes arrow.Record based on a schema. diff --git a/go/arrow/csv/writer_test.go b/go/arrow/csv/writer_test.go index b1bd3251c5622..9ca5e063989a4 100644 --- a/go/arrow/csv/writer_test.go +++ b/go/arrow/csv/writer_test.go @@ -26,14 +26,14 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/csv" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/csv" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" "github.com/google/uuid" ) diff --git a/go/arrow/datatype.go b/go/arrow/datatype.go index b2f2329e5ccaa..f8fffe5d36399 100644 --- a/go/arrow/datatype.go +++ b/go/arrow/datatype.go @@ -21,7 +21,7 @@ import ( "hash/maphash" "strings" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) // Type is a logical type. They can be expressed as diff --git a/go/arrow/datatype_binary_test.go b/go/arrow/datatype_binary_test.go index 083d69ee3e5d4..48c4f0c839079 100644 --- a/go/arrow/datatype_binary_test.go +++ b/go/arrow/datatype_binary_test.go @@ -19,7 +19,7 @@ package arrow_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) func TestBinaryType(t *testing.T) { diff --git a/go/arrow/datatype_extension_test.go b/go/arrow/datatype_extension_test.go index 9811800400506..92d04b5c3db7a 100644 --- a/go/arrow/datatype_extension_test.go +++ b/go/arrow/datatype_extension_test.go @@ -20,8 +20,8 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/arrow/datatype_fixedwidth.go b/go/arrow/datatype_fixedwidth.go index 9248dcb8c9a35..21edd41d232e4 100644 --- a/go/arrow/datatype_fixedwidth.go +++ b/go/arrow/datatype_fixedwidth.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/internal/json" "golang.org/x/xerrors" ) diff --git a/go/arrow/datatype_fixedwidth_test.go b/go/arrow/datatype_fixedwidth_test.go index d6caa21e1a255..2ef05554ad86c 100644 --- a/go/arrow/datatype_fixedwidth_test.go +++ b/go/arrow/datatype_fixedwidth_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/datatype_nested.go b/go/arrow/datatype_nested.go index 9a8873a50eb1a..1a31eb6312ba9 100644 --- a/go/arrow/datatype_nested.go +++ b/go/arrow/datatype_nested.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) type ( diff --git a/go/arrow/datatype_null_test.go b/go/arrow/datatype_null_test.go index 57cddfadb8ef8..b12d1fc60db6a 100644 --- a/go/arrow/datatype_null_test.go +++ b/go/arrow/datatype_null_test.go @@ -19,7 +19,7 @@ package arrow_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) func TestNullType(t *testing.T) { diff --git a/go/arrow/datatype_viewheader.go b/go/arrow/datatype_viewheader.go index 54b9256b34604..bbd7f977db809 100644 --- a/go/arrow/datatype_viewheader.go +++ b/go/arrow/datatype_viewheader.go @@ -20,9 +20,9 @@ import ( "bytes" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" ) const ( diff --git a/go/arrow/datatype_viewheader_inline.go b/go/arrow/datatype_viewheader_inline.go index 89ac1d06adcdf..9ffc0a1f98339 100644 --- a/go/arrow/datatype_viewheader_inline.go +++ b/go/arrow/datatype_viewheader_inline.go @@ -21,7 +21,7 @@ package arrow import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) func (sh *ViewHeader) InlineString() (data string) { diff --git a/go/arrow/datatype_viewheader_inline_go1.19.go b/go/arrow/datatype_viewheader_inline_go1.19.go index aec66009d9492..3933f44df065c 100644 --- a/go/arrow/datatype_viewheader_inline_go1.19.go +++ b/go/arrow/datatype_viewheader_inline_go1.19.go @@ -22,7 +22,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) func (sh *ViewHeader) InlineString() (data string) { diff --git a/go/arrow/datatype_viewheader_inline_tinygo.go b/go/arrow/datatype_viewheader_inline_tinygo.go index bff63a273a722..3c0dea4a72587 100644 --- a/go/arrow/datatype_viewheader_inline_tinygo.go +++ b/go/arrow/datatype_viewheader_inline_tinygo.go @@ -22,7 +22,7 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) func (sh *ViewHeader) InlineString() (data string) { diff --git a/go/arrow/decimal128/decimal128.go b/go/arrow/decimal128/decimal128.go index 7ce8cd51b0717..68b4d66e1bd38 100644 --- a/go/arrow/decimal128/decimal128.go +++ b/go/arrow/decimal128/decimal128.go @@ -23,7 +23,7 @@ import ( "math/big" "math/bits" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) const ( diff --git a/go/arrow/decimal128/decimal128_test.go b/go/arrow/decimal128/decimal128_test.go index 05af1f557f1f9..a36ae0622aa98 100644 --- a/go/arrow/decimal128/decimal128_test.go +++ b/go/arrow/decimal128/decimal128_test.go @@ -22,7 +22,7 @@ import ( "math/big" "testing" - "github.com/apache/arrow/go/v15/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal128" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/decimal256/decimal256.go b/go/arrow/decimal256/decimal256.go index 65be0df9dd424..cb1a41c84b5a6 100644 --- a/go/arrow/decimal256/decimal256.go +++ b/go/arrow/decimal256/decimal256.go @@ -23,8 +23,8 @@ import ( "math/big" "math/bits" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) const ( diff --git a/go/arrow/decimal256/decimal256_test.go b/go/arrow/decimal256/decimal256_test.go index 3be6a1944406f..3e0a97e430b45 100644 --- a/go/arrow/decimal256/decimal256_test.go +++ b/go/arrow/decimal256/decimal256_test.go @@ -23,7 +23,7 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/decimal256" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/doc.go b/go/arrow/doc.go index c8da330e4f8a1..2f7c8c2acf1ce 100644 --- a/go/arrow/doc.go +++ b/go/arrow/doc.go @@ -36,7 +36,7 @@ To build with tinygo include the noasm build tag. */ package arrow -const PkgVersion = "15.0.0-SNAPSHOT" +const PkgVersion = "16.0.0-SNAPSHOT" //go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl //go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl diff --git a/go/arrow/encoded/ree_utils.go b/go/arrow/encoded/ree_utils.go index 2ff4e7cadd38b..daa265369ca7b 100644 --- a/go/arrow/encoded/ree_utils.go +++ b/go/arrow/encoded/ree_utils.go @@ -20,7 +20,7 @@ import ( "math" "sort" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) // FindPhysicalIndex performs a binary search on the run-ends to return diff --git a/go/arrow/encoded/ree_utils_test.go b/go/arrow/encoded/ree_utils_test.go index 57d11e9b4edbe..454ab2810008c 100644 --- a/go/arrow/encoded/ree_utils_test.go +++ b/go/arrow/encoded/ree_utils_test.go @@ -21,10 +21,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/encoded" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/encoded" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/endian/endian.go b/go/arrow/endian/endian.go index d4865025b1fb0..7b7ce57f5d561 100644 --- a/go/arrow/endian/endian.go +++ b/go/arrow/endian/endian.go @@ -17,8 +17,8 @@ package endian import ( - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" ) type Endianness flatbuf.Endianness diff --git a/go/arrow/example_test.go b/go/arrow/example_test.go index b28cc9093739f..9b7ae11d451ad 100644 --- a/go/arrow/example_test.go +++ b/go/arrow/example_test.go @@ -20,10 +20,10 @@ import ( "fmt" "log" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/tensor" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/tensor" ) // This example demonstrates how to build an array of int64 values using a builder and Append. diff --git a/go/arrow/flight/basic_auth_flight_test.go b/go/arrow/flight/basic_auth_flight_test.go index 4097bf02edeff..e1cc6c1cd7b3c 100755 --- a/go/arrow/flight/basic_auth_flight_test.go +++ b/go/arrow/flight/basic_auth_flight_test.go @@ -22,7 +22,7 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/client.go b/go/arrow/flight/client.go index 4f44bdc5ebd58..312c8a76b6f0e 100644 --- a/go/arrow/flight/client.go +++ b/go/arrow/flight/client.go @@ -26,7 +26,7 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v16/arrow/flight/gen/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" diff --git a/go/arrow/flight/cookie_middleware_test.go b/go/arrow/flight/cookie_middleware_test.go index bdfe4fb32c1a5..0adf4927652d4 100644 --- a/go/arrow/flight/cookie_middleware_test.go +++ b/go/arrow/flight/cookie_middleware_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/go/arrow/flight/example_flight_server_test.go b/go/arrow/flight/example_flight_server_test.go index d3a71b3b67ade..af8094a953867 100755 --- a/go/arrow/flight/example_flight_server_test.go +++ b/go/arrow/flight/example_flight_server_test.go @@ -23,7 +23,7 @@ import ( "io" "log" - "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/flight_middleware_test.go b/go/arrow/flight/flight_middleware_test.go index e6faa1b7df1d2..a99436c59d8ed 100755 --- a/go/arrow/flight/flight_middleware_test.go +++ b/go/arrow/flight/flight_middleware_test.go @@ -23,8 +23,8 @@ import ( sync "sync" "testing" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/go/arrow/flight/flight_test.go b/go/arrow/flight/flight_test.go index cd37658603d5b..d023704276eab 100755 --- a/go/arrow/flight/flight_test.go +++ b/go/arrow/flight/flight_test.go @@ -23,11 +23,11 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/go/arrow/flight/flightsql/client.go b/go/arrow/flight/flightsql/client.go index 89784b483b01b..928118cf299c1 100644 --- a/go/arrow/flight/flightsql/client.go +++ b/go/arrow/flight/flightsql/client.go @@ -22,12 +22,12 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + pb "github.com/apache/arrow/go/v16/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" diff --git a/go/arrow/flight/flightsql/client_test.go b/go/arrow/flight/flightsql/client_test.go index 3efe4ba4049b8..a4fb83f984f1d 100644 --- a/go/arrow/flight/flightsql/client_test.go +++ b/go/arrow/flight/flightsql/client_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + pb "github.com/apache/arrow/go/v16/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "google.golang.org/grpc" diff --git a/go/arrow/flight/flightsql/column_metadata.go b/go/arrow/flight/flightsql/column_metadata.go index 8e0a69b99d055..875acdb1e435d 100644 --- a/go/arrow/flight/flightsql/column_metadata.go +++ b/go/arrow/flight/flightsql/column_metadata.go @@ -19,7 +19,7 @@ package flightsql import ( "strconv" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) const ( diff --git a/go/arrow/flight/flightsql/driver/README.md b/go/arrow/flight/flightsql/driver/README.md index fade2cc5c4eec..7fe24e7bd3f3b 100644 --- a/go/arrow/flight/flightsql/driver/README.md +++ b/go/arrow/flight/flightsql/driver/README.md @@ -36,7 +36,7 @@ connection pooling, transactions combined with ease of use (see (#usage)). ## Prerequisites * Go 1.17+ -* Installation via `go get -u github.com/apache/arrow/go/v15/arrow/flight/flightsql` +* Installation via `go get -u github.com/apache/arrow/go/v16/arrow/flight/flightsql` * Backend speaking FlightSQL --------------------------------------- @@ -55,7 +55,7 @@ import ( "database/sql" "time" - _ "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + _ "github.com/apache/arrow/go/v16/arrow/flight/flightsql" ) // Open the connection to an SQLite backend @@ -141,7 +141,7 @@ import ( "log" "time" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" ) func main() { diff --git a/go/arrow/flight/flightsql/driver/config_test.go b/go/arrow/flight/flightsql/driver/config_test.go index 6b24b535585d9..b3c3433362d05 100644 --- a/go/arrow/flight/flightsql/driver/config_test.go +++ b/go/arrow/flight/flightsql/driver/config_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/driver" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/driver" ) func TestConfigTLSRegistry(t *testing.T) { diff --git a/go/arrow/flight/flightsql/driver/driver.go b/go/arrow/flight/flightsql/driver/driver.go index f74bfa378a303..852a97fb4d3ca 100644 --- a/go/arrow/flight/flightsql/driver/driver.go +++ b/go/arrow/flight/flightsql/driver/driver.go @@ -25,11 +25,11 @@ import ( "sort" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/memory" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/go/arrow/flight/flightsql/driver/driver_test.go b/go/arrow/flight/flightsql/driver/driver_test.go index 24eb5ee6812c0..58193626f51dc 100644 --- a/go/arrow/flight/flightsql/driver/driver_test.go +++ b/go/arrow/flight/flightsql/driver/driver_test.go @@ -33,13 +33,13 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/driver" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/driver" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v16/arrow/memory" ) const defaultTableName = "drivertest" diff --git a/go/arrow/flight/flightsql/driver/utils.go b/go/arrow/flight/flightsql/driver/utils.go index 022e9da4925f9..f7bd2a2e02113 100644 --- a/go/arrow/flight/flightsql/driver/utils.go +++ b/go/arrow/flight/flightsql/driver/utils.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" ) // *** GRPC helpers *** diff --git a/go/arrow/flight/flightsql/driver/utils_test.go b/go/arrow/flight/flightsql/driver/utils_test.go index 8f5ea031adf28..6b1adfed47503 100644 --- a/go/arrow/flight/flightsql/driver/utils_test.go +++ b/go/arrow/flight/flightsql/driver/utils_test.go @@ -22,12 +22,12 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go b/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go index 1a50422f826f3..209e2287890b5 100644 --- a/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go +++ b/go/arrow/flight/flightsql/example/cmd/sqlite_flightsql_server/main.go @@ -27,9 +27,9 @@ import ( "os" "strconv" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/example" ) func main() { diff --git a/go/arrow/flight/flightsql/example/sql_batch_reader.go b/go/arrow/flight/flightsql/example/sql_batch_reader.go index 8c87021672de5..27a71153ffca9 100644 --- a/go/arrow/flight/flightsql/example/sql_batch_reader.go +++ b/go/arrow/flight/flightsql/example/sql_batch_reader.go @@ -26,11 +26,11 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/wrapperspb" diff --git a/go/arrow/flight/flightsql/example/sqlite_info.go b/go/arrow/flight/flightsql/example/sqlite_info.go index c993af97f7449..a097fa1954435 100644 --- a/go/arrow/flight/flightsql/example/sqlite_info.go +++ b/go/arrow/flight/flightsql/example/sqlite_info.go @@ -20,8 +20,8 @@ package example import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" ) func SqlInfoResultMap() flightsql.SqlInfoResultMap { diff --git a/go/arrow/flight/flightsql/example/sqlite_server.go b/go/arrow/flight/flightsql/example/sqlite_server.go index 24cadd957e584..d87efbc9844b1 100644 --- a/go/arrow/flight/flightsql/example/sqlite_server.go +++ b/go/arrow/flight/flightsql/example/sqlite_server.go @@ -45,13 +45,13 @@ import ( "strings" "sync" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" diff --git a/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go b/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go index e07a98fffdbf7..08b217a3eadc1 100644 --- a/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go +++ b/go/arrow/flight/flightsql/example/sqlite_tables_schema_batch_reader.go @@ -25,12 +25,12 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" sqlite3 "modernc.org/sqlite/lib" ) diff --git a/go/arrow/flight/flightsql/example/type_info.go b/go/arrow/flight/flightsql/example/type_info.go index 0180fc1f23af5..03f906a8f9624 100644 --- a/go/arrow/flight/flightsql/example/type_info.go +++ b/go/arrow/flight/flightsql/example/type_info.go @@ -22,10 +22,10 @@ package example import ( "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v16/arrow/memory" ) func GetTypeInfoResult(mem memory.Allocator) arrow.Record { diff --git a/go/arrow/flight/flightsql/schema_ref/reference_schemas.go b/go/arrow/flight/flightsql/schema_ref/reference_schemas.go index b65688306473b..eb756686187d0 100644 --- a/go/arrow/flight/flightsql/schema_ref/reference_schemas.go +++ b/go/arrow/flight/flightsql/schema_ref/reference_schemas.go @@ -18,7 +18,7 @@ // by FlightSQL servers and clients. package schema_ref -import "github.com/apache/arrow/go/v15/arrow" +import "github.com/apache/arrow/go/v16/arrow" var ( Catalogs = arrow.NewSchema( diff --git a/go/arrow/flight/flightsql/server.go b/go/arrow/flight/flightsql/server.go index 2ec02e2829962..b825f121f3a16 100644 --- a/go/arrow/flight/flightsql/server.go +++ b/go/arrow/flight/flightsql/server.go @@ -20,14 +20,14 @@ import ( "context" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/schema_ref" + pb "github.com/apache/arrow/go/v16/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" diff --git a/go/arrow/flight/flightsql/server_test.go b/go/arrow/flight/flightsql/server_test.go index 956a1714c671c..22bbe3f8154b2 100644 --- a/go/arrow/flight/flightsql/server_test.go +++ b/go/arrow/flight/flightsql/server_test.go @@ -22,12 +22,12 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + pb "github.com/apache/arrow/go/v16/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/go/arrow/flight/flightsql/sql_info.go b/go/arrow/flight/flightsql/sql_info.go index 5f78e1facd581..ed6cf5f2cc2d9 100644 --- a/go/arrow/flight/flightsql/sql_info.go +++ b/go/arrow/flight/flightsql/sql_info.go @@ -17,8 +17,8 @@ package flightsql import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" ) const ( diff --git a/go/arrow/flight/flightsql/sqlite_server_test.go b/go/arrow/flight/flightsql/sqlite_server_test.go index e6fa798c55cf1..da8369e91b458 100644 --- a/go/arrow/flight/flightsql/sqlite_server_test.go +++ b/go/arrow/flight/flightsql/sqlite_server_test.go @@ -26,14 +26,14 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/example" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/example" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "google.golang.org/grpc/codes" diff --git a/go/arrow/flight/flightsql/types.go b/go/arrow/flight/flightsql/types.go index 5a26414d8c232..d89e68f028bb8 100644 --- a/go/arrow/flight/flightsql/types.go +++ b/go/arrow/flight/flightsql/types.go @@ -17,7 +17,7 @@ package flightsql import ( - pb "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + pb "github.com/apache/arrow/go/v16/arrow/flight/gen/flight" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) diff --git a/go/arrow/flight/record_batch_reader.go b/go/arrow/flight/record_batch_reader.go index f137c75172368..f0741139fd510 100644 --- a/go/arrow/flight/record_batch_reader.go +++ b/go/arrow/flight/record_batch_reader.go @@ -21,12 +21,12 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) // DataStreamReader is an interface for receiving flight data messages on a stream diff --git a/go/arrow/flight/record_batch_writer.go b/go/arrow/flight/record_batch_writer.go index 426dca6c3e907..6fbec8d7bd9ce 100644 --- a/go/arrow/flight/record_batch_writer.go +++ b/go/arrow/flight/record_batch_writer.go @@ -19,9 +19,9 @@ package flight import ( "bytes" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) // DataStreamWriter is an interface that represents an Arrow Flight stream diff --git a/go/arrow/flight/server.go b/go/arrow/flight/server.go index 3e1da64dcf8c4..c70aceabcfe8e 100644 --- a/go/arrow/flight/server.go +++ b/go/arrow/flight/server.go @@ -22,7 +22,7 @@ import ( "os" "os/signal" - "github.com/apache/arrow/go/v15/arrow/flight/gen/flight" + "github.com/apache/arrow/go/v16/arrow/flight/gen/flight" "google.golang.org/grpc" ) diff --git a/go/arrow/flight/server_example_test.go b/go/arrow/flight/server_example_test.go index 4e2e4f4c6b587..f6cb9ce59b99a 100644 --- a/go/arrow/flight/server_example_test.go +++ b/go/arrow/flight/server_example_test.go @@ -21,7 +21,7 @@ import ( "fmt" "net" - "github.com/apache/arrow/go/v15/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/health" diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go index 985388094eb51..bb04bcdd373ef 100644 --- a/go/arrow/internal/arrdata/arrdata.go +++ b/go/arrow/internal/arrdata/arrdata.go @@ -21,14 +21,14 @@ import ( "fmt" "sort" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" ) var ( diff --git a/go/arrow/internal/arrdata/ioutil.go b/go/arrow/internal/arrdata/ioutil.go index a6becc2151ef3..5e09fe01c8af5 100644 --- a/go/arrow/internal/arrdata/ioutil.go +++ b/go/arrow/internal/arrdata/ioutil.go @@ -23,11 +23,11 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) // CheckArrowFile checks whether a given ARROW file contains the expected list of records. diff --git a/go/arrow/internal/arrjson/arrjson.go b/go/arrow/internal/arrjson/arrjson.go index 49f711cdacd76..fdc0d4e2f1cab 100644 --- a/go/arrow/internal/arrjson/arrjson.go +++ b/go/arrow/internal/arrjson/arrjson.go @@ -26,16 +26,16 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" ) type Schema struct { diff --git a/go/arrow/internal/arrjson/arrjson_test.go b/go/arrow/internal/arrjson/arrjson_test.go index 164210cbc230d..6b5b414d49802 100644 --- a/go/arrow/internal/arrjson/arrjson_test.go +++ b/go/arrow/internal/arrjson/arrjson_test.go @@ -22,9 +22,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/internal/arrjson/option.go b/go/arrow/internal/arrjson/option.go index 47d40f857c54e..e310ce727aa80 100644 --- a/go/arrow/internal/arrjson/option.go +++ b/go/arrow/internal/arrjson/option.go @@ -17,8 +17,8 @@ package arrjson import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" ) type config struct { diff --git a/go/arrow/internal/arrjson/reader.go b/go/arrow/internal/arrjson/reader.go index 226fa1b1919f9..dff9534b782d4 100644 --- a/go/arrow/internal/arrjson/reader.go +++ b/go/arrow/internal/arrjson/reader.go @@ -20,11 +20,11 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/internal/json" ) type Reader struct { diff --git a/go/arrow/internal/arrjson/writer.go b/go/arrow/internal/arrjson/writer.go index fdd36e97d9b42..37ddcbec28921 100644 --- a/go/arrow/internal/arrjson/writer.go +++ b/go/arrow/internal/arrjson/writer.go @@ -20,11 +20,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/internal/json" ) const ( diff --git a/go/arrow/internal/cdata_integration/entrypoints.go b/go/arrow/internal/cdata_integration/entrypoints.go index a40db8316f848..384bcb886d8c2 100644 --- a/go/arrow/internal/cdata_integration/entrypoints.go +++ b/go/arrow/internal/cdata_integration/entrypoints.go @@ -25,10 +25,10 @@ import ( "runtime" "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/cdata" - "github.com/apache/arrow/go/v15/arrow/internal/arrjson" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/cdata" + "github.com/apache/arrow/go/v16/arrow/internal/arrjson" + "github.com/apache/arrow/go/v16/arrow/memory" ) // #include diff --git a/go/arrow/internal/dictutils/dict.go b/go/arrow/internal/dictutils/dict.go index 5c0bf54dafcbd..ae7fe4d5c88c6 100644 --- a/go/arrow/internal/dictutils/dict.go +++ b/go/arrow/internal/dictutils/dict.go @@ -21,9 +21,9 @@ import ( "fmt" "hash/maphash" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" ) type Kind int8 diff --git a/go/arrow/internal/dictutils/dict_test.go b/go/arrow/internal/dictutils/dict_test.go index 9d2f7ae4782c7..86ce6709798c5 100644 --- a/go/arrow/internal/dictutils/dict_test.go +++ b/go/arrow/internal/dictutils/dict_test.go @@ -20,10 +20,10 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestDictMemo(t *testing.T) { diff --git a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go index 1afb53c5294f4..e53e035315b58 100755 --- a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go +++ b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-client/main.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - "github.com/apache/arrow/go/v15/arrow/internal/flight_integration" + "github.com/apache/arrow/go/v16/arrow/internal/flight_integration" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) diff --git a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go index d382ff9e88670..8def9993795d1 100644 --- a/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go +++ b/go/arrow/internal/flight_integration/cmd/arrow-flight-integration-server/main.go @@ -23,7 +23,7 @@ import ( "os" "syscall" - "github.com/apache/arrow/go/v15/arrow/internal/flight_integration" + "github.com/apache/arrow/go/v16/arrow/internal/flight_integration" ) var ( diff --git a/go/arrow/internal/flight_integration/scenario.go b/go/arrow/internal/flight_integration/scenario.go index 0b12d22cc7ed7..91658a694ecab 100644 --- a/go/arrow/internal/flight_integration/scenario.go +++ b/go/arrow/internal/flight_integration/scenario.go @@ -31,15 +31,15 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql" - "github.com/apache/arrow/go/v15/arrow/flight/flightsql/schema_ref" - "github.com/apache/arrow/go/v15/arrow/internal/arrjson" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql" + "github.com/apache/arrow/go/v16/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow/go/v16/arrow/internal/arrjson" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" "golang.org/x/xerrors" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/go/arrow/internal/testing/gen/random_array_gen.go b/go/arrow/internal/testing/gen/random_array_gen.go index 3e5a11fe88d2d..16ff7a0b41e94 100644 --- a/go/arrow/internal/testing/gen/random_array_gen.go +++ b/go/arrow/internal/testing/gen/random_array_gen.go @@ -19,11 +19,11 @@ package gen import ( "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" ) diff --git a/go/arrow/internal/testing/tools/bits_test.go b/go/arrow/internal/testing/tools/bits_test.go index 6ad8ac5b12b68..b33f1a173a34d 100644 --- a/go/arrow/internal/testing/tools/bits_test.go +++ b/go/arrow/internal/testing/tools/bits_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v16/arrow/internal/testing/tools" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/internal/testing/tools/data_types.go b/go/arrow/internal/testing/tools/data_types.go index 161cfa431491d..490b9474232d7 100644 --- a/go/arrow/internal/testing/tools/data_types.go +++ b/go/arrow/internal/testing/tools/data_types.go @@ -21,8 +21,8 @@ package tools import ( "reflect" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/float16" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/internal/utils.go b/go/arrow/internal/utils.go index 7b5df167ea432..f3831f6961bec 100644 --- a/go/arrow/internal/utils.go +++ b/go/arrow/internal/utils.go @@ -17,8 +17,8 @@ package internal import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" ) const CurMetadataVersion = flatbuf.MetadataVersionV5 diff --git a/go/arrow/ipc/cmd/arrow-cat/main.go b/go/arrow/ipc/cmd/arrow-cat/main.go index 0251b08c09bff..8fe55054c44a9 100644 --- a/go/arrow/ipc/cmd/arrow-cat/main.go +++ b/go/arrow/ipc/cmd/arrow-cat/main.go @@ -63,8 +63,8 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-cat/main_test.go b/go/arrow/ipc/cmd/arrow-cat/main_test.go index b6528b85b416d..d2a28271ce98e 100644 --- a/go/arrow/ipc/cmd/arrow-cat/main_test.go +++ b/go/arrow/ipc/cmd/arrow-cat/main_test.go @@ -23,10 +23,10 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestCatStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-file-to-stream/main.go b/go/arrow/ipc/cmd/arrow-file-to-stream/main.go index 6187b94465211..d47d4019775f2 100644 --- a/go/arrow/ipc/cmd/arrow-file-to-stream/main.go +++ b/go/arrow/ipc/cmd/arrow-file-to-stream/main.go @@ -24,9 +24,9 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go b/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go index 2f66a8b48c59d..edd49e5acbf98 100644 --- a/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go +++ b/go/arrow/ipc/cmd/arrow-file-to-stream/main_test.go @@ -21,8 +21,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestFileToStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-json-integration-test/main.go b/go/arrow/ipc/cmd/arrow-json-integration-test/main.go index 7db2fa1d1c5db..0c55c38cf991f 100644 --- a/go/arrow/ipc/cmd/arrow-json-integration-test/main.go +++ b/go/arrow/ipc/cmd/arrow-json-integration-test/main.go @@ -22,12 +22,12 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/arrjson" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/internal/arrjson" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/internal/types" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go b/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go index eb702f0aa2bc5..1d1fd76d97738 100644 --- a/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go +++ b/go/arrow/ipc/cmd/arrow-json-integration-test/main_test.go @@ -20,8 +20,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestIntegration(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-ls/main.go b/go/arrow/ipc/cmd/arrow-ls/main.go index 4230ae2449985..a4634c65df19f 100644 --- a/go/arrow/ipc/cmd/arrow-ls/main.go +++ b/go/arrow/ipc/cmd/arrow-ls/main.go @@ -61,8 +61,8 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-ls/main_test.go b/go/arrow/ipc/cmd/arrow-ls/main_test.go index bda1c2dc4d3dc..36c293ffc0e0a 100644 --- a/go/arrow/ipc/cmd/arrow-ls/main_test.go +++ b/go/arrow/ipc/cmd/arrow-ls/main_test.go @@ -23,10 +23,10 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestLsStream(t *testing.T) { diff --git a/go/arrow/ipc/cmd/arrow-stream-to-file/main.go b/go/arrow/ipc/cmd/arrow-stream-to-file/main.go index 7ed3f6a281d9f..3fbb534145c2b 100644 --- a/go/arrow/ipc/cmd/arrow-stream-to-file/main.go +++ b/go/arrow/ipc/cmd/arrow-stream-to-file/main.go @@ -24,9 +24,9 @@ import ( "log" "os" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) func main() { diff --git a/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go b/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go index 73196551054ac..d5545763f1169 100644 --- a/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go +++ b/go/arrow/ipc/cmd/arrow-stream-to-file/main_test.go @@ -21,8 +21,8 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestStreamToFile(t *testing.T) { diff --git a/go/arrow/ipc/compression.go b/go/arrow/ipc/compression.go index 06a9cf67cfb6b..7d9c19093ad7f 100644 --- a/go/arrow/ipc/compression.go +++ b/go/arrow/ipc/compression.go @@ -19,9 +19,9 @@ package ipc import ( "io" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/klauspost/compress/zstd" "github.com/pierrec/lz4/v4" ) diff --git a/go/arrow/ipc/endian_swap.go b/go/arrow/ipc/endian_swap.go index 35ba0e4e764f9..244a5718c9da7 100644 --- a/go/arrow/ipc/endian_swap.go +++ b/go/arrow/ipc/endian_swap.go @@ -21,9 +21,9 @@ import ( "fmt" "math/bits" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" ) // swap the endianness of the array's buffers as needed in-place to save diff --git a/go/arrow/ipc/endian_swap_test.go b/go/arrow/ipc/endian_swap_test.go index 1c724103f4441..8667e1cf5d28d 100644 --- a/go/arrow/ipc/endian_swap_test.go +++ b/go/arrow/ipc/endian_swap_test.go @@ -20,11 +20,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go index 7bc7f6ebfaa09..d0118279afa4d 100644 --- a/go/arrow/ipc/file_reader.go +++ b/go/arrow/ipc/file_reader.go @@ -23,14 +23,14 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/internal" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" ) // FileReader is an Arrow file reader. diff --git a/go/arrow/ipc/file_test.go b/go/arrow/ipc/file_test.go index 5f4dac1f899bb..87c3f4ddd4a65 100644 --- a/go/arrow/ipc/file_test.go +++ b/go/arrow/ipc/file_test.go @@ -21,9 +21,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestFile(t *testing.T) { diff --git a/go/arrow/ipc/file_writer.go b/go/arrow/ipc/file_writer.go index 55e4d7c2dc22c..5812745e7d759 100644 --- a/go/arrow/ipc/file_writer.go +++ b/go/arrow/ipc/file_writer.go @@ -21,11 +21,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" ) // PayloadWriter is an interface for injecting a different payloadwriter diff --git a/go/arrow/ipc/ipc.go b/go/arrow/ipc/ipc.go index 8cf56e8331fcb..16ca907350934 100644 --- a/go/arrow/ipc/ipc.go +++ b/go/arrow/ipc/ipc.go @@ -19,10 +19,10 @@ package ipc import ( "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" ) const ( diff --git a/go/arrow/ipc/ipc_test.go b/go/arrow/ipc/ipc_test.go index d02d8734b56d4..247cad04ee409 100644 --- a/go/arrow/ipc/ipc_test.go +++ b/go/arrow/ipc/ipc_test.go @@ -29,10 +29,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestArrow12072(t *testing.T) { diff --git a/go/arrow/ipc/message.go b/go/arrow/ipc/message.go index 5295c5df30137..b1616c891fd50 100644 --- a/go/arrow/ipc/message.go +++ b/go/arrow/ipc/message.go @@ -22,9 +22,9 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" ) // MetadataVersion represents the Arrow metadata version. diff --git a/go/arrow/ipc/message_test.go b/go/arrow/ipc/message_test.go index 912d112229f0c..2ad3334c1e0d6 100644 --- a/go/arrow/ipc/message_test.go +++ b/go/arrow/ipc/message_test.go @@ -22,9 +22,9 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestMessageReaderBodyInAllocator(t *testing.T) { diff --git a/go/arrow/ipc/metadata.go b/go/arrow/ipc/metadata.go index 7a7f9b3e212b7..62167bcecfb69 100644 --- a/go/arrow/ipc/metadata.go +++ b/go/arrow/ipc/metadata.go @@ -23,11 +23,11 @@ import ( "io" "sort" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" flatbuffers "github.com/google/flatbuffers/go" ) diff --git a/go/arrow/ipc/metadata_test.go b/go/arrow/ipc/metadata_test.go index a35068813ba56..339059db5efdc 100644 --- a/go/arrow/ipc/metadata_test.go +++ b/go/arrow/ipc/metadata_test.go @@ -21,12 +21,12 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" flatbuffers "github.com/google/flatbuffers/go" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/ipc/reader.go b/go/arrow/ipc/reader.go index 826062fafdb46..9b0810c50e67f 100644 --- a/go/arrow/ipc/reader.go +++ b/go/arrow/ipc/reader.go @@ -23,13 +23,13 @@ import ( "io" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" ) // Reader reads records from an io.Reader. diff --git a/go/arrow/ipc/reader_test.go b/go/arrow/ipc/reader_test.go index 42bb3fea3e963..35ada57018701 100644 --- a/go/arrow/ipc/reader_test.go +++ b/go/arrow/ipc/reader_test.go @@ -22,9 +22,9 @@ import ( "io" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/ipc/stream_test.go b/go/arrow/ipc/stream_test.go index 201bfb82a5677..fc0cdce62da95 100644 --- a/go/arrow/ipc/stream_test.go +++ b/go/arrow/ipc/stream_test.go @@ -22,9 +22,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow/internal/arrdata" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/internal/arrdata" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" ) func TestStream(t *testing.T) { diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go index 31ce53a0f1af7..588273166aa99 100644 --- a/go/arrow/ipc/writer.go +++ b/go/arrow/ipc/writer.go @@ -26,15 +26,15 @@ import ( "sync" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/internal/dictutils" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/dictutils" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" ) type swriter struct { diff --git a/go/arrow/ipc/writer_test.go b/go/arrow/ipc/writer_test.go index ea7592554c888..a1bf92efbcc8a 100644 --- a/go/arrow/ipc/writer_test.go +++ b/go/arrow/ipc/writer_test.go @@ -24,11 +24,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/internal/flatbuf" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/internal/flatbuf" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/arrow/math/float64.go b/go/arrow/math/float64.go index 8d72ecf1b2668..f99cba737f8dc 100644 --- a/go/arrow/math/float64.go +++ b/go/arrow/math/float64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) type Float64Funcs struct { diff --git a/go/arrow/math/float64_avx2_amd64.go b/go/arrow/math/float64_avx2_amd64.go index e78d4affe9f6f..0d98573720627 100644 --- a/go/arrow/math/float64_avx2_amd64.go +++ b/go/arrow/math/float64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_neon_arm64.go b/go/arrow/math/float64_neon_arm64.go index a5013960d9ec7..91b018ff655ec 100755 --- a/go/arrow/math/float64_neon_arm64.go +++ b/go/arrow/math/float64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_sse4_amd64.go b/go/arrow/math/float64_sse4_amd64.go index cdd88f91c2c2b..8ec3a46f676b1 100644 --- a/go/arrow/math/float64_sse4_amd64.go +++ b/go/arrow/math/float64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/float64_test.go b/go/arrow/math/float64_test.go index 3a7c247c5df8c..f734f35dcb13c 100644 --- a/go/arrow/math/float64_test.go +++ b/go/arrow/math/float64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/math" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/int64.go b/go/arrow/math/int64.go index b8236dfbb6552..9abc1f591d68c 100644 --- a/go/arrow/math/int64.go +++ b/go/arrow/math/int64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) type Int64Funcs struct { diff --git a/go/arrow/math/int64_avx2_amd64.go b/go/arrow/math/int64_avx2_amd64.go index 5c16e790c1e55..84a6d60d5f2c6 100644 --- a/go/arrow/math/int64_avx2_amd64.go +++ b/go/arrow/math/int64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_neon_arm64.go b/go/arrow/math/int64_neon_arm64.go index 00ec48b6e4735..03ad1b43d80a7 100755 --- a/go/arrow/math/int64_neon_arm64.go +++ b/go/arrow/math/int64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_sse4_amd64.go b/go/arrow/math/int64_sse4_amd64.go index a894bad2ef6a3..c3d76991b074b 100644 --- a/go/arrow/math/int64_sse4_amd64.go +++ b/go/arrow/math/int64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/int64_test.go b/go/arrow/math/int64_test.go index 3781fd3570785..dd5b78d107737 100644 --- a/go/arrow/math/int64_test.go +++ b/go/arrow/math/int64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/math" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/type.go.tmpl b/go/arrow/math/type.go.tmpl index f1a4dfa1362f8..7ec4dda4feadf 100644 --- a/go/arrow/math/type.go.tmpl +++ b/go/arrow/math/type.go.tmpl @@ -17,7 +17,7 @@ package math import ( - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) {{$def := .D}} diff --git a/go/arrow/math/type_simd_amd64.go.tmpl b/go/arrow/math/type_simd_amd64.go.tmpl index 77dee758b05cd..336496f39d51f 100644 --- a/go/arrow/math/type_simd_amd64.go.tmpl +++ b/go/arrow/math/type_simd_amd64.go.tmpl @@ -21,7 +21,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) {{$name := printf "%s_%s" .In.Type .D.arch}} diff --git a/go/arrow/math/type_simd_arm64.go.tmpl b/go/arrow/math/type_simd_arm64.go.tmpl index 77dee758b05cd..336496f39d51f 100755 --- a/go/arrow/math/type_simd_arm64.go.tmpl +++ b/go/arrow/math/type_simd_arm64.go.tmpl @@ -21,7 +21,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) {{$name := printf "%s_%s" .In.Type .D.arch}} diff --git a/go/arrow/math/type_test.go.tmpl b/go/arrow/math/type_test.go.tmpl index 969377e454e52..9c412327b7a85 100644 --- a/go/arrow/math/type_test.go.tmpl +++ b/go/arrow/math/type_test.go.tmpl @@ -19,9 +19,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/math" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/math/uint64.go b/go/arrow/math/uint64.go index b9a70360224ea..6da7427879d49 100644 --- a/go/arrow/math/uint64.go +++ b/go/arrow/math/uint64.go @@ -19,7 +19,7 @@ package math import ( - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) type Uint64Funcs struct { diff --git a/go/arrow/math/uint64_avx2_amd64.go b/go/arrow/math/uint64_avx2_amd64.go index c1f48efefd8eb..01b29f5e9b27b 100644 --- a/go/arrow/math/uint64_avx2_amd64.go +++ b/go/arrow/math/uint64_avx2_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_neon_arm64.go b/go/arrow/math/uint64_neon_arm64.go index 37ca4ecad04e7..df7bcc7340361 100755 --- a/go/arrow/math/uint64_neon_arm64.go +++ b/go/arrow/math/uint64_neon_arm64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_sse4_amd64.go b/go/arrow/math/uint64_sse4_amd64.go index 7163c1de171de..031a36635fb46 100644 --- a/go/arrow/math/uint64_sse4_amd64.go +++ b/go/arrow/math/uint64_sse4_amd64.go @@ -24,7 +24,7 @@ package math import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow/array" ) //go:noescape diff --git a/go/arrow/math/uint64_test.go b/go/arrow/math/uint64_test.go index e8ba42b59aa10..d332e8e766c74 100644 --- a/go/arrow/math/uint64_test.go +++ b/go/arrow/math/uint64_test.go @@ -21,9 +21,9 @@ package math_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/math" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/math" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/buffer.go b/go/arrow/memory/buffer.go index ffdb41e3dbe9e..f45d0d55da602 100644 --- a/go/arrow/memory/buffer.go +++ b/go/arrow/memory/buffer.go @@ -19,7 +19,7 @@ package memory import ( "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) // Buffer is a wrapper type for a buffer of bytes. diff --git a/go/arrow/memory/buffer_test.go b/go/arrow/memory/buffer_test.go index 92bb071d0abde..b97fe4b3ae4ab 100644 --- a/go/arrow/memory/buffer_test.go +++ b/go/arrow/memory/buffer_test.go @@ -19,7 +19,7 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/cgo_allocator.go b/go/arrow/memory/cgo_allocator.go index af25d1899a6fe..9b8bb467a2c4f 100644 --- a/go/arrow/memory/cgo_allocator.go +++ b/go/arrow/memory/cgo_allocator.go @@ -22,7 +22,7 @@ package memory import ( "runtime" - cga "github.com/apache/arrow/go/v15/arrow/memory/internal/cgoalloc" + cga "github.com/apache/arrow/go/v16/arrow/memory/internal/cgoalloc" ) // CgoArrowAllocator is an allocator which exposes the C++ memory pool class diff --git a/go/arrow/memory/default_mallocator.go b/go/arrow/memory/default_mallocator.go index 4a9ef942fd08d..414f2c2d87e62 100644 --- a/go/arrow/memory/default_mallocator.go +++ b/go/arrow/memory/default_mallocator.go @@ -19,7 +19,7 @@ package memory import ( - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v16/arrow/memory/mallocator" ) // DefaultAllocator is a default implementation of Allocator and can be used anywhere diff --git a/go/arrow/memory/default_mallocator_test.go b/go/arrow/memory/default_mallocator_test.go index 5a38e8b4e843c..904230cba503c 100644 --- a/go/arrow/memory/default_mallocator_test.go +++ b/go/arrow/memory/default_mallocator_test.go @@ -21,8 +21,8 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/mallocator/mallocator_test.go b/go/arrow/memory/mallocator/mallocator_test.go index 91b74383ed494..fa98208f66ecc 100644 --- a/go/arrow/memory/mallocator/mallocator_test.go +++ b/go/arrow/memory/mallocator/mallocator_test.go @@ -23,7 +23,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/arrow/memory/mallocator" + "github.com/apache/arrow/go/v16/arrow/memory/mallocator" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/memory/memory_test.go b/go/arrow/memory/memory_test.go index adaa8359369be..bedabe364f842 100644 --- a/go/arrow/memory/memory_test.go +++ b/go/arrow/memory/memory_test.go @@ -19,7 +19,7 @@ package memory_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/record.go b/go/arrow/record.go index 885af38034938..28974ea1dc0ba 100644 --- a/go/arrow/record.go +++ b/go/arrow/record.go @@ -16,7 +16,7 @@ package arrow -import "github.com/apache/arrow/go/v15/internal/json" +import "github.com/apache/arrow/go/v16/internal/json" // Record is a collection of equal-length arrays matching a particular Schema. // Also known as a RecordBatch in the spec and in some implementations. diff --git a/go/arrow/scalar/append.go b/go/arrow/scalar/append.go index fe8dd32e9eae5..f87e16dcf4603 100644 --- a/go/arrow/scalar/append.go +++ b/go/arrow/scalar/append.go @@ -21,11 +21,11 @@ package scalar import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/scalar/append_test.go b/go/arrow/scalar/append_test.go index e509bca1fb7b0..3b24dff579e0d 100644 --- a/go/arrow/scalar/append_test.go +++ b/go/arrow/scalar/append_test.go @@ -23,11 +23,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/testing/tools" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/testing/tools" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/scalar/binary.go b/go/arrow/scalar/binary.go index b6abe9cba7b1c..763583916982e 100644 --- a/go/arrow/scalar/binary.go +++ b/go/arrow/scalar/binary.go @@ -21,8 +21,8 @@ import ( "fmt" "unicode/utf8" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" ) type BinaryScalar interface { diff --git a/go/arrow/scalar/compare.go b/go/arrow/scalar/compare.go index b4a3fe08d26fe..c5ccb73b77c52 100644 --- a/go/arrow/scalar/compare.go +++ b/go/arrow/scalar/compare.go @@ -16,7 +16,7 @@ package scalar -import "github.com/apache/arrow/go/v15/arrow" +import "github.com/apache/arrow/go/v16/arrow" // Equals returns true if two scalars are equal, which means they have the same // datatype, validity and value. diff --git a/go/arrow/scalar/nested.go b/go/arrow/scalar/nested.go index cf89dc9fbdc17..f22290ecfb373 100644 --- a/go/arrow/scalar/nested.go +++ b/go/arrow/scalar/nested.go @@ -21,10 +21,10 @@ import ( "errors" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/scalar/numeric.gen.go b/go/arrow/scalar/numeric.gen.go index da088162f4017..d5875df1d622a 100644 --- a/go/arrow/scalar/numeric.gen.go +++ b/go/arrow/scalar/numeric.gen.go @@ -24,9 +24,9 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" ) type Int8 struct { diff --git a/go/arrow/scalar/numeric.gen_test.go b/go/arrow/scalar/numeric.gen_test.go index 807857c1c98c2..c13f36129e54f 100644 --- a/go/arrow/scalar/numeric.gen_test.go +++ b/go/arrow/scalar/numeric.gen_test.go @@ -21,8 +21,8 @@ package scalar_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/scalar/numeric.gen_test.go.tmpl b/go/arrow/scalar/numeric.gen_test.go.tmpl index 7f45898a20d29..98fc3d11a18a4 100644 --- a/go/arrow/scalar/numeric.gen_test.go.tmpl +++ b/go/arrow/scalar/numeric.gen_test.go.tmpl @@ -19,8 +19,8 @@ package scalar_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/scalar/parse.go b/go/arrow/scalar/parse.go index 5002f98a65c42..50269211bb9df 100644 --- a/go/arrow/scalar/parse.go +++ b/go/arrow/scalar/parse.go @@ -25,12 +25,12 @@ import ( "strings" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" ) type TypeToScalar interface { diff --git a/go/arrow/scalar/scalar.go b/go/arrow/scalar/scalar.go index 9744c07fb05a3..1f4d9d007c3ed 100644 --- a/go/arrow/scalar/scalar.go +++ b/go/arrow/scalar/scalar.go @@ -26,16 +26,16 @@ import ( "strconv" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/encoded" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/internal/debug" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/encoded" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/xerrors" ) diff --git a/go/arrow/scalar/scalar_test.go b/go/arrow/scalar/scalar_test.go index e85f160624d18..e20e3089d6efc 100644 --- a/go/arrow/scalar/scalar_test.go +++ b/go/arrow/scalar/scalar_test.go @@ -25,12 +25,12 @@ import ( "testing" "time" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/scalar" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/scalar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/arrow/scalar/temporal.go b/go/arrow/scalar/temporal.go index ee13c84429e8e..ad42188c26997 100644 --- a/go/arrow/scalar/temporal.go +++ b/go/arrow/scalar/temporal.go @@ -22,7 +22,7 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) func temporalToString(s TemporalScalar) string { diff --git a/go/arrow/schema.go b/go/arrow/schema.go index 7a05bb1888972..d4943cc4ae632 100644 --- a/go/arrow/schema.go +++ b/go/arrow/schema.go @@ -21,7 +21,7 @@ import ( "sort" "strings" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/endian" ) type Metadata struct { diff --git a/go/arrow/schema_test.go b/go/arrow/schema_test.go index fd94620aee650..a01ba839fa525 100644 --- a/go/arrow/schema_test.go +++ b/go/arrow/schema_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/endian" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/table.go b/go/arrow/table.go index 82dc283706b65..f0728108d94b9 100644 --- a/go/arrow/table.go +++ b/go/arrow/table.go @@ -20,7 +20,7 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) // Table represents a logical sequence of chunked arrays of equal length. It is diff --git a/go/arrow/tensor/numeric.gen.go b/go/arrow/tensor/numeric.gen.go index d207f0bfa2c9c..eac0fe5cc7fcb 100644 --- a/go/arrow/tensor/numeric.gen.go +++ b/go/arrow/tensor/numeric.gen.go @@ -19,7 +19,7 @@ package tensor import ( - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) // Int8 is an n-dim array of int8s. diff --git a/go/arrow/tensor/numeric.gen.go.tmpl b/go/arrow/tensor/numeric.gen.go.tmpl index e03f986da3f8f..0813035ac06e1 100644 --- a/go/arrow/tensor/numeric.gen.go.tmpl +++ b/go/arrow/tensor/numeric.gen.go.tmpl @@ -17,8 +17,8 @@ package tensor import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" ) {{range .In}} diff --git a/go/arrow/tensor/numeric.gen_test.go b/go/arrow/tensor/numeric.gen_test.go index 3a7c3570c216b..9750d6681ca9a 100644 --- a/go/arrow/tensor/numeric.gen_test.go +++ b/go/arrow/tensor/numeric.gen_test.go @@ -23,10 +23,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/tensor" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/tensor" ) func TestTensorInt8(t *testing.T) { diff --git a/go/arrow/tensor/numeric.gen_test.go.tmpl b/go/arrow/tensor/numeric.gen_test.go.tmpl index 34f01b40d8084..7110f87e9c454 100644 --- a/go/arrow/tensor/numeric.gen_test.go.tmpl +++ b/go/arrow/tensor/numeric.gen_test.go.tmpl @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/tensor" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/tensor" ) {{range .In}} diff --git a/go/arrow/tensor/tensor.go b/go/arrow/tensor/tensor.go index 1f2ed7e82141b..f96720f0d0f8f 100644 --- a/go/arrow/tensor/tensor.go +++ b/go/arrow/tensor/tensor.go @@ -21,8 +21,8 @@ import ( "fmt" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) // Interface represents an n-dimensional array of numerical data. diff --git a/go/arrow/tensor/tensor_test.go b/go/arrow/tensor/tensor_test.go index 552c4c1c223e0..e6a2547b7beef 100644 --- a/go/arrow/tensor/tensor_test.go +++ b/go/arrow/tensor/tensor_test.go @@ -21,10 +21,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/tensor" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/tensor" ) func TestTensor(t *testing.T) { diff --git a/go/arrow/type_traits.go b/go/arrow/type_traits.go index 67fa8a266b35f..54e1bbdc31162 100644 --- a/go/arrow/type_traits.go +++ b/go/arrow/type_traits.go @@ -20,9 +20,9 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" "golang.org/x/exp/constraints" ) diff --git a/go/arrow/type_traits_boolean.go b/go/arrow/type_traits_boolean.go index c164d45954fc0..6b22c264c6676 100644 --- a/go/arrow/type_traits_boolean.go +++ b/go/arrow/type_traits_boolean.go @@ -17,7 +17,7 @@ package arrow import ( - "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/bitutil" ) type booleanTraits struct{} diff --git a/go/arrow/type_traits_decimal128.go b/go/arrow/type_traits_decimal128.go index d600ba29c1186..8d9bcc8c3f66e 100644 --- a/go/arrow/type_traits_decimal128.go +++ b/go/arrow/type_traits_decimal128.go @@ -19,8 +19,8 @@ package arrow import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/endian" ) // Decimal128 traits diff --git a/go/arrow/type_traits_decimal256.go b/go/arrow/type_traits_decimal256.go index fded46a0a52d0..73fd9b81ffe69 100644 --- a/go/arrow/type_traits_decimal256.go +++ b/go/arrow/type_traits_decimal256.go @@ -19,8 +19,8 @@ package arrow import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/endian" ) // Decimal256 traits diff --git a/go/arrow/type_traits_float16.go b/go/arrow/type_traits_float16.go index 5369ad352f839..2340ac567b62c 100644 --- a/go/arrow/type_traits_float16.go +++ b/go/arrow/type_traits_float16.go @@ -19,8 +19,8 @@ package arrow import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/float16" ) // Float16 traits diff --git a/go/arrow/type_traits_interval.go b/go/arrow/type_traits_interval.go index ca530a72323ff..79fd029310bc9 100644 --- a/go/arrow/type_traits_interval.go +++ b/go/arrow/type_traits_interval.go @@ -19,8 +19,8 @@ package arrow import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/internal/debug" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/internal/debug" ) var ( diff --git a/go/arrow/type_traits_numeric.gen.go b/go/arrow/type_traits_numeric.gen.go index 06412466032f9..34d1d7843ce14 100644 --- a/go/arrow/type_traits_numeric.gen.go +++ b/go/arrow/type_traits_numeric.gen.go @@ -22,7 +22,7 @@ import ( "math" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/endian" ) var ( diff --git a/go/arrow/type_traits_numeric.gen.go.tmpl b/go/arrow/type_traits_numeric.gen.go.tmpl index e98f59528c6aa..af471738b6cb6 100644 --- a/go/arrow/type_traits_numeric.gen.go.tmpl +++ b/go/arrow/type_traits_numeric.gen.go.tmpl @@ -20,7 +20,7 @@ import ( "math" "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/endian" ) var ( diff --git a/go/arrow/type_traits_numeric.gen_test.go b/go/arrow/type_traits_numeric.gen_test.go index ac2d0726ed6fa..a830a7a0fdb34 100644 --- a/go/arrow/type_traits_numeric.gen_test.go +++ b/go/arrow/type_traits_numeric.gen_test.go @@ -22,7 +22,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) func TestInt64Traits(t *testing.T) { diff --git a/go/arrow/type_traits_numeric.gen_test.go.tmpl b/go/arrow/type_traits_numeric.gen_test.go.tmpl index 4948f42cf542e..b033d57e5d890 100644 --- a/go/arrow/type_traits_numeric.gen_test.go.tmpl +++ b/go/arrow/type_traits_numeric.gen_test.go.tmpl @@ -20,7 +20,7 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) {{- range .In}} diff --git a/go/arrow/type_traits_test.go b/go/arrow/type_traits_test.go index 0ae88b4bd82b6..869da426fad96 100644 --- a/go/arrow/type_traits_test.go +++ b/go/arrow/type_traits_test.go @@ -22,10 +22,10 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/float16" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/float16" ) func TestBooleanTraits(t *testing.T) { diff --git a/go/arrow/type_traits_timestamp.go b/go/arrow/type_traits_timestamp.go index 8e9970a719f54..3117f983edd0b 100644 --- a/go/arrow/type_traits_timestamp.go +++ b/go/arrow/type_traits_timestamp.go @@ -19,7 +19,7 @@ package arrow import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/endian" ) var TimestampTraits timestampTraits diff --git a/go/arrow/type_traits_view.go b/go/arrow/type_traits_view.go index be3f15fed69ae..8a56702900e9f 100644 --- a/go/arrow/type_traits_view.go +++ b/go/arrow/type_traits_view.go @@ -19,7 +19,7 @@ package arrow import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/endian" ) var ViewHeaderTraits viewHeaderTraits diff --git a/go/arrow/util/byte_size.go b/go/arrow/util/byte_size.go index 6d6fc021f8005..840af58ec0614 100644 --- a/go/arrow/util/byte_size.go +++ b/go/arrow/util/byte_size.go @@ -17,9 +17,9 @@ package util import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" ) func isArrayDataNil(arrayData arrow.ArrayData) bool { diff --git a/go/arrow/util/byte_size_test.go b/go/arrow/util/byte_size_test.go index a218c69558fe9..a0302e72a8a34 100644 --- a/go/arrow/util/byte_size_test.go +++ b/go/arrow/util/byte_size_test.go @@ -20,10 +20,10 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/arrow/util" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/util" "github.com/stretchr/testify/assert" ) diff --git a/go/go.mod b/go/go.mod index 73a1cb7e7738b..c07abaf3c9888 100644 --- a/go/go.mod +++ b/go/go.mod @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -module github.com/apache/arrow/go/v15 +module github.com/apache/arrow/go/v16 go 1.20 diff --git a/go/internal/bitutils/bit_block_counter.go b/go/internal/bitutils/bit_block_counter.go index 50996b10e8851..a53a1d9c1cb47 100644 --- a/go/internal/bitutils/bit_block_counter.go +++ b/go/internal/bitutils/bit_block_counter.go @@ -21,8 +21,8 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/internal/utils" ) func loadWord(byt []byte) uint64 { diff --git a/go/internal/bitutils/bit_block_counter_test.go b/go/internal/bitutils/bit_block_counter_test.go index 790105c290182..441bb3c4b6e68 100644 --- a/go/internal/bitutils/bit_block_counter_test.go +++ b/go/internal/bitutils/bit_block_counter_test.go @@ -19,9 +19,9 @@ package bitutils_test import ( "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" "github.com/stretchr/testify/assert" "golang.org/x/exp/rand" ) diff --git a/go/internal/bitutils/bit_run_reader.go b/go/internal/bitutils/bit_run_reader.go index f09149d7ec5df..5d0e92e76c734 100644 --- a/go/internal/bitutils/bit_run_reader.go +++ b/go/internal/bitutils/bit_run_reader.go @@ -22,9 +22,9 @@ import ( "math/bits" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/internal/utils" ) // BitRun represents a run of bits with the same value of length Len diff --git a/go/internal/bitutils/bit_run_reader_test.go b/go/internal/bitutils/bit_run_reader_test.go index 7db76768a9476..da90860c56986 100644 --- a/go/internal/bitutils/bit_run_reader_test.go +++ b/go/internal/bitutils/bit_run_reader_test.go @@ -21,9 +21,9 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/internal/bitutils" "github.com/stretchr/testify/assert" ) diff --git a/go/internal/bitutils/bit_set_run_reader.go b/go/internal/bitutils/bit_set_run_reader.go index 374b8d4aab39a..47d09881d5a13 100644 --- a/go/internal/bitutils/bit_set_run_reader.go +++ b/go/internal/bitutils/bit_set_run_reader.go @@ -20,8 +20,8 @@ import ( "encoding/binary" "math/bits" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/internal/utils" ) // IsMultipleOf64 returns whether v is a multiple of 64. diff --git a/go/internal/bitutils/bit_set_run_reader_test.go b/go/internal/bitutils/bit_set_run_reader_test.go index 832993671ef6d..136b9fad92c59 100644 --- a/go/internal/bitutils/bit_set_run_reader_test.go +++ b/go/internal/bitutils/bit_set_run_reader_test.go @@ -20,9 +20,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/utils" "github.com/stretchr/testify/suite" ) diff --git a/go/internal/bitutils/bitmap_generate.go b/go/internal/bitutils/bitmap_generate.go index 08b5fceab57d3..d88aeee901194 100644 --- a/go/internal/bitutils/bitmap_generate.go +++ b/go/internal/bitutils/bitmap_generate.go @@ -16,7 +16,7 @@ package bitutils -import "github.com/apache/arrow/go/v15/arrow/bitutil" +import "github.com/apache/arrow/go/v16/arrow/bitutil" // GenerateBits writes sequential bits to a bitmap. Bits preceding the // initial start offset are preserved, bits following the bitmap may diff --git a/go/internal/bitutils/bitmap_generate_test.go b/go/internal/bitutils/bitmap_generate_test.go index c9a6203864a20..e5e68f477037a 100644 --- a/go/internal/bitutils/bitmap_generate_test.go +++ b/go/internal/bitutils/bitmap_generate_test.go @@ -19,7 +19,7 @@ package bitutils_test import ( "testing" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/bitutils" "golang.org/x/exp/rand" ) diff --git a/go/internal/hashing/xxh3_memo_table.gen.go b/go/internal/hashing/xxh3_memo_table.gen.go index 39b82cdeff9a2..ca27fdf42ebb4 100644 --- a/go/internal/hashing/xxh3_memo_table.gen.go +++ b/go/internal/hashing/xxh3_memo_table.gen.go @@ -21,9 +21,9 @@ package hashing import ( "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/internal/utils" ) type payloadInt8 struct { diff --git a/go/internal/hashing/xxh3_memo_table.gen.go.tmpl b/go/internal/hashing/xxh3_memo_table.gen.go.tmpl index 527008ad63c3c..b1b52f3bcfcf9 100644 --- a/go/internal/hashing/xxh3_memo_table.gen.go.tmpl +++ b/go/internal/hashing/xxh3_memo_table.gen.go.tmpl @@ -17,8 +17,8 @@ package hashing import ( - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/internal/utils" ) {{range .In}} diff --git a/go/internal/types/extension_types.go b/go/internal/types/extension_types.go index e24c89efc7b8b..f2b3c4a02ec59 100644 --- a/go/internal/types/extension_types.go +++ b/go/internal/types/extension_types.go @@ -24,9 +24,9 @@ import ( "reflect" "strings" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/internal/json" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/internal/json" "github.com/google/uuid" "golang.org/x/xerrors" ) diff --git a/go/internal/types/extension_types_test.go b/go/internal/types/extension_types_test.go index f93f1000c9e4a..e6132b611d455 100644 --- a/go/internal/types/extension_types_test.go +++ b/go/internal/types/extension_types_test.go @@ -20,11 +20,11 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/json" + "github.com/apache/arrow/go/v16/internal/types" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/internal/utils/transpose_ints_def.go b/go/internal/utils/transpose_ints_def.go index c52598d7148ea..2680429acf8cf 100644 --- a/go/internal/utils/transpose_ints_def.go +++ b/go/internal/utils/transpose_ints_def.go @@ -19,7 +19,7 @@ package utils import ( "errors" - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) //go:generate go run ../../arrow/_tools/tmpl -i -data=transpose_ints.tmpldata -d arch=avx2 transpose_ints_simd.go.tmpl=transpose_ints_avx2_amd64.go diff --git a/go/internal/utils/transpose_ints_test.go b/go/internal/utils/transpose_ints_test.go index 73b2bbce3fc14..16e3832868e50 100644 --- a/go/internal/utils/transpose_ints_test.go +++ b/go/internal/utils/transpose_ints_test.go @@ -22,7 +22,7 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/internal/utils" ) var ( diff --git a/go/parquet/cmd/parquet_reader/dumper.go b/go/parquet/cmd/parquet_reader/dumper.go index 4cb2ea4a96fee..11248502119c4 100644 --- a/go/parquet/cmd/parquet_reader/dumper.go +++ b/go/parquet/cmd/parquet_reader/dumper.go @@ -22,9 +22,9 @@ import ( "reflect" "time" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/schema" ) const defaultBatchSize = 128 diff --git a/go/parquet/cmd/parquet_reader/main.go b/go/parquet/cmd/parquet_reader/main.go index 0d651d8c294c5..c315aeeb32e79 100644 --- a/go/parquet/cmd/parquet_reader/main.go +++ b/go/parquet/cmd/parquet_reader/main.go @@ -25,11 +25,11 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/internal/json" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/docopt/docopt-go" ) diff --git a/go/parquet/cmd/parquet_schema/main.go b/go/parquet/cmd/parquet_schema/main.go index 01d541e8ac925..8c59358248cee 100644 --- a/go/parquet/cmd/parquet_schema/main.go +++ b/go/parquet/cmd/parquet_schema/main.go @@ -20,8 +20,8 @@ import ( "fmt" "os" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/docopt/docopt-go" ) diff --git a/go/parquet/compress/brotli.go b/go/parquet/compress/brotli.go index a1199d9711435..8a7e92a1403c3 100644 --- a/go/parquet/compress/brotli.go +++ b/go/parquet/compress/brotli.go @@ -21,7 +21,7 @@ import ( "io" "github.com/andybalholm/brotli" - "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v16/parquet/internal/debug" ) type brotliCodec struct{} diff --git a/go/parquet/compress/compress.go b/go/parquet/compress/compress.go index f61147eb1ea10..dc45b6ee9311f 100644 --- a/go/parquet/compress/compress.go +++ b/go/parquet/compress/compress.go @@ -23,7 +23,7 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" ) // Compression is an alias to the thrift compression codec enum type for easy use diff --git a/go/parquet/compress/compress_test.go b/go/parquet/compress/compress_test.go index d1c55b15bc3cc..e2ff8871c3b59 100644 --- a/go/parquet/compress/compress_test.go +++ b/go/parquet/compress/compress_test.go @@ -22,7 +22,7 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/compress" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/compress/zstd.go b/go/parquet/compress/zstd.go index fd8ec81953359..02ffd2eae568a 100644 --- a/go/parquet/compress/zstd.go +++ b/go/parquet/compress/zstd.go @@ -20,7 +20,7 @@ import ( "io" "sync" - "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v16/parquet/internal/debug" "github.com/klauspost/compress/zstd" ) diff --git a/go/parquet/doc.go b/go/parquet/doc.go index afeee00587ef5..dfe20b6ffd86a 100644 --- a/go/parquet/doc.go +++ b/go/parquet/doc.go @@ -29,9 +29,9 @@ // Install // // You can download the library and cli utilities via: -// go get -u github.com/apache/arrow/go/v15/parquet -// go install github.com/apache/arrow/go/v15/parquet/cmd/parquet_reader@latest -// go install github.com/apache/arrow/go/v15/parquet/cmd/parquet_schema@latest +// go get -u github.com/apache/arrow/go/v16/parquet +// go install github.com/apache/arrow/go/v16/parquet/cmd/parquet_reader@latest +// go install github.com/apache/arrow/go/v16/parquet/cmd/parquet_schema@latest // // Modules // diff --git a/go/parquet/encryption_properties.go b/go/parquet/encryption_properties.go index 0eadc5fb0451c..dcb00ab7d09ca 100644 --- a/go/parquet/encryption_properties.go +++ b/go/parquet/encryption_properties.go @@ -20,7 +20,7 @@ import ( "crypto/rand" "unicode/utf8" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" ) // Constants that will be used as the default values with encryption/decryption diff --git a/go/parquet/encryption_properties_test.go b/go/parquet/encryption_properties_test.go index ab028927c5ecb..e73435897435f 100644 --- a/go/parquet/encryption_properties_test.go +++ b/go/parquet/encryption_properties_test.go @@ -19,8 +19,8 @@ package parquet_test import ( "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/encryption_read_config_test.go b/go/parquet/encryption_read_config_test.go index 53b7ba3c621c0..8e981fbb12b55 100644 --- a/go/parquet/encryption_read_config_test.go +++ b/go/parquet/encryption_read_config_test.go @@ -23,10 +23,10 @@ import ( "path" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/encryption_write_config_test.go b/go/parquet/encryption_write_config_test.go index 558b89208c706..7075fd6366cc7 100644 --- a/go/parquet/encryption_write_config_test.go +++ b/go/parquet/encryption_write_config_test.go @@ -23,10 +23,10 @@ import ( "path/filepath" "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/file/column_reader.go b/go/parquet/file/column_reader.go index 342fb3b198abe..40d93a35c3407 100644 --- a/go/parquet/file/column_reader.go +++ b/go/parquet/file/column_reader.go @@ -21,13 +21,13 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/column_reader_test.go b/go/parquet/file/column_reader_test.go index a6725bc02fee0..f19039797fabe 100755 --- a/go/parquet/file/column_reader_test.go +++ b/go/parquet/file/column_reader_test.go @@ -24,12 +24,12 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/testutils" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/file/column_reader_types.gen.go b/go/parquet/file/column_reader_types.gen.go index 3fb113780f811..d8ddef4bd1081 100644 --- a/go/parquet/file/column_reader_types.gen.go +++ b/go/parquet/file/column_reader_types.gen.go @@ -21,9 +21,9 @@ package file import ( "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" ) // Int32ColumnChunkReader is the Typed Column chunk reader instance for reading diff --git a/go/parquet/file/column_reader_types.gen.go.tmpl b/go/parquet/file/column_reader_types.gen.go.tmpl index 261b5f0bfacab..153c4f7527ae9 100644 --- a/go/parquet/file/column_reader_types.gen.go.tmpl +++ b/go/parquet/file/column_reader_types.gen.go.tmpl @@ -17,8 +17,8 @@ package file import ( - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" ) {{range .In}} diff --git a/go/parquet/file/column_writer.go b/go/parquet/file/column_writer.go index 0b0d1145d604e..ac857d17e632d 100755 --- a/go/parquet/file/column_writer.go +++ b/go/parquet/file/column_writer.go @@ -21,14 +21,14 @@ import ( "encoding/binary" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" ) //go:generate go run ../../arrow/_tools/tmpl/main.go -i -data=../internal/encoding/physical_types.tmpldata column_writer_types.gen.go.tmpl diff --git a/go/parquet/file/column_writer_test.go b/go/parquet/file/column_writer_test.go index 05c1cadebf6cf..8011ac2487995 100755 --- a/go/parquet/file/column_writer_test.go +++ b/go/parquet/file/column_writer_test.go @@ -24,19 +24,19 @@ import ( "sync" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - arrutils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + arrutils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/testutils" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" diff --git a/go/parquet/file/column_writer_types.gen.go b/go/parquet/file/column_writer_types.gen.go index b4d7954639319..2766d3aa4aca2 100644 --- a/go/parquet/file/column_writer_types.gen.go +++ b/go/parquet/file/column_writer_types.gen.go @@ -22,13 +22,13 @@ import ( "errors" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/column_writer_types.gen.go.tmpl b/go/parquet/file/column_writer_types.gen.go.tmpl index 70bcfe679eb92..fcd3750012af5 100644 --- a/go/parquet/file/column_writer_types.gen.go.tmpl +++ b/go/parquet/file/column_writer_types.gen.go.tmpl @@ -19,10 +19,10 @@ package file import ( "fmt" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" ) {{range .In}} diff --git a/go/parquet/file/file_reader.go b/go/parquet/file/file_reader.go index 29162e4a4ec27..b0c8cbf9c7244 100644 --- a/go/parquet/file/file_reader.go +++ b/go/parquet/file/file_reader.go @@ -25,10 +25,10 @@ import ( "runtime" "sync" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + "github.com/apache/arrow/go/v16/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/file_reader_mmap.go b/go/parquet/file/file_reader_mmap.go index 03e12adf08c8c..43fb22f07bf99 100644 --- a/go/parquet/file/file_reader_mmap.go +++ b/go/parquet/file/file_reader_mmap.go @@ -22,7 +22,7 @@ package file import ( "io" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/exp/mmap" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/file_reader_mmap_windows.go b/go/parquet/file/file_reader_mmap_windows.go index 06a9e97160fe0..bf6436accee02 100644 --- a/go/parquet/file/file_reader_mmap_windows.go +++ b/go/parquet/file/file_reader_mmap_windows.go @@ -22,7 +22,7 @@ package file import ( "errors" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/parquet" ) func mmapOpen(filename string) (parquet.ReaderAtSeeker, error) { diff --git a/go/parquet/file/file_reader_test.go b/go/parquet/file/file_reader_test.go index f3248925cf5b9..c68f30b7ba7f4 100644 --- a/go/parquet/file/file_reader_test.go +++ b/go/parquet/file/file_reader_test.go @@ -25,16 +25,16 @@ import ( "path" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/thrift" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" libthrift "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/file/file_writer.go b/go/parquet/file/file_writer.go index 1d7f7840dac50..a2cf397cbc80b 100644 --- a/go/parquet/file/file_writer.go +++ b/go/parquet/file/file_writer.go @@ -21,11 +21,11 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" ) // Writer is the primary interface for writing a parquet file diff --git a/go/parquet/file/file_writer_test.go b/go/parquet/file/file_writer_test.go index f32e403a8d534..434c9852c5823 100644 --- a/go/parquet/file/file_writer_test.go +++ b/go/parquet/file/file_writer_test.go @@ -22,13 +22,13 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/internal/testutils" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/file/level_conversion.go b/go/parquet/file/level_conversion.go index 251468658ae30..9eb33fae361b3 100755 --- a/go/parquet/file/level_conversion.go +++ b/go/parquet/file/level_conversion.go @@ -22,11 +22,11 @@ import ( "math/bits" "unsafe" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/bmi" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/schema" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/bmi" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/level_conversion_test.go b/go/parquet/file/level_conversion_test.go index 54e52c5e7abb6..4b72fac859866 100644 --- a/go/parquet/file/level_conversion_test.go +++ b/go/parquet/file/level_conversion_test.go @@ -20,9 +20,9 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/parquet/internal/bmi" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/parquet/internal/bmi" + "github.com/apache/arrow/go/v16/parquet/internal/utils" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/file/page_reader.go b/go/parquet/file/page_reader.go index 01f253aff8864..0acd841762c20 100644 --- a/go/parquet/file/page_reader.go +++ b/go/parquet/file/page_reader.go @@ -23,13 +23,13 @@ import ( "sync" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/thrift" + "github.com/apache/arrow/go/v16/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/page_writer.go b/go/parquet/file/page_writer.go index c16476fbb232c..97e352d77043c 100644 --- a/go/parquet/file/page_writer.go +++ b/go/parquet/file/page_writer.go @@ -20,15 +20,15 @@ import ( "bytes" "sync" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/thrift" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + "github.com/apache/arrow/go/v16/parquet/metadata" libthrift "github.com/apache/thrift/lib/go/thrift" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/record_reader.go b/go/parquet/file/record_reader.go index 5698f49d9f2bb..cbc66dc7746c9 100755 --- a/go/parquet/file/record_reader.go +++ b/go/parquet/file/record_reader.go @@ -23,14 +23,14 @@ import ( "unsafe" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_reader.go b/go/parquet/file/row_group_reader.go index 3c1c1edb0b484..43f1a69ece553 100644 --- a/go/parquet/file/row_group_reader.go +++ b/go/parquet/file/row_group_reader.go @@ -20,10 +20,10 @@ import ( "fmt" "sync" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + "github.com/apache/arrow/go/v16/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_writer.go b/go/parquet/file/row_group_writer.go index 74f4becb55e08..7ab23e2291020 100644 --- a/go/parquet/file/row_group_writer.go +++ b/go/parquet/file/row_group_writer.go @@ -17,10 +17,10 @@ package file import ( - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + "github.com/apache/arrow/go/v16/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/file/row_group_writer_test.go b/go/parquet/file/row_group_writer_test.go index 2ea317cb91001..d8ebba1b83fc0 100644 --- a/go/parquet/file/row_group_writer_test.go +++ b/go/parquet/file/row_group_writer_test.go @@ -20,10 +20,10 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/bmi/bmi_test.go b/go/parquet/internal/bmi/bmi_test.go index a5278dfef2211..e229b40050ca2 100644 --- a/go/parquet/internal/bmi/bmi_test.go +++ b/go/parquet/internal/bmi/bmi_test.go @@ -20,7 +20,7 @@ import ( "fmt" "testing" - "github.com/apache/arrow/go/v15/parquet/internal/bmi" + "github.com/apache/arrow/go/v16/parquet/internal/bmi" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/encoding/boolean_decoder.go b/go/parquet/internal/encoding/boolean_decoder.go index 353f443855952..8f386ec752beb 100644 --- a/go/parquet/internal/encoding/boolean_decoder.go +++ b/go/parquet/internal/encoding/boolean_decoder.go @@ -23,10 +23,10 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow/bitutil" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/utils" ) // PlainBooleanDecoder is for the Plain Encoding type, there is no diff --git a/go/parquet/internal/encoding/boolean_encoder.go b/go/parquet/internal/encoding/boolean_encoder.go index 3e01bde369d8b..33a414562ce57 100644 --- a/go/parquet/internal/encoding/boolean_encoder.go +++ b/go/parquet/internal/encoding/boolean_encoder.go @@ -19,10 +19,10 @@ package encoding import ( "encoding/binary" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/debug" + "github.com/apache/arrow/go/v16/parquet/internal/utils" ) const ( diff --git a/go/parquet/internal/encoding/byte_array_decoder.go b/go/parquet/internal/encoding/byte_array_decoder.go index 0c1c858fb48bb..ef58fd83329a4 100644 --- a/go/parquet/internal/encoding/byte_array_decoder.go +++ b/go/parquet/internal/encoding/byte_array_decoder.go @@ -19,12 +19,12 @@ package encoding import ( "encoding/binary" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - pqutils "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + pqutils "github.com/apache/arrow/go/v16/parquet/internal/utils" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/byte_array_encoder.go b/go/parquet/internal/encoding/byte_array_encoder.go index 9270b5531768c..456c6ccb5270a 100644 --- a/go/parquet/internal/encoding/byte_array_encoder.go +++ b/go/parquet/internal/encoding/byte_array_encoder.go @@ -21,11 +21,11 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" ) // PlainByteArrayEncoder encodes byte arrays according to the spec for Plain encoding diff --git a/go/parquet/internal/encoding/decoder.go b/go/parquet/internal/encoding/decoder.go index acb57fbce7806..2ba5152a99a8f 100644 --- a/go/parquet/internal/encoding/decoder.go +++ b/go/parquet/internal/encoding/decoder.go @@ -20,16 +20,16 @@ import ( "bytes" "reflect" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/debug" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/delta_bit_packing.go b/go/parquet/internal/encoding/delta_bit_packing.go index 6ac799f1c179d..cd4f6d5f81716 100644 --- a/go/parquet/internal/encoding/delta_bit_packing.go +++ b/go/parquet/internal/encoding/delta_bit_packing.go @@ -23,11 +23,11 @@ import ( "math/bits" "reflect" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/utils" ) // see the deltaBitPack encoder for a description of the encoding format that is diff --git a/go/parquet/internal/encoding/delta_byte_array.go b/go/parquet/internal/encoding/delta_byte_array.go index 18bd12015a430..db31fa7c2bb4d 100644 --- a/go/parquet/internal/encoding/delta_byte_array.go +++ b/go/parquet/internal/encoding/delta_byte_array.go @@ -17,9 +17,9 @@ package encoding import ( - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/delta_byte_array_test.go b/go/parquet/internal/encoding/delta_byte_array_test.go index 0a206796f742a..eda7c8c35cdc2 100644 --- a/go/parquet/internal/encoding/delta_byte_array_test.go +++ b/go/parquet/internal/encoding/delta_byte_array_test.go @@ -18,8 +18,8 @@ package encoding import ( "fmt" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" "github.com/stretchr/testify/assert" "testing" ) diff --git a/go/parquet/internal/encoding/delta_length_byte_array.go b/go/parquet/internal/encoding/delta_length_byte_array.go index 183eb453ca0a3..1906f4cb217b1 100644 --- a/go/parquet/internal/encoding/delta_length_byte_array.go +++ b/go/parquet/internal/encoding/delta_length_byte_array.go @@ -17,9 +17,9 @@ package encoding import ( - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/encoder.go b/go/parquet/internal/encoding/encoder.go index 7023309397a3b..8539b5f84b69c 100644 --- a/go/parquet/internal/encoding/encoder.go +++ b/go/parquet/internal/encoding/encoder.go @@ -21,14 +21,14 @@ import ( "math/bits" "reflect" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + "github.com/apache/arrow/go/v16/parquet/schema" ) //go:generate go run ../../../arrow/_tools/tmpl/main.go -i -data=physical_types.tmpldata plain_encoder_types.gen.go.tmpl typed_encoder.gen.go.tmpl diff --git a/go/parquet/internal/encoding/encoding_benchmarks_test.go b/go/parquet/internal/encoding/encoding_benchmarks_test.go index e0645e9de54e4..db95ede315f27 100644 --- a/go/parquet/internal/encoding/encoding_benchmarks_test.go +++ b/go/parquet/internal/encoding/encoding_benchmarks_test.go @@ -21,14 +21,14 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/hashing" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/internal/testutils" + "github.com/apache/arrow/go/v16/parquet/schema" ) const ( diff --git a/go/parquet/internal/encoding/encoding_test.go b/go/parquet/internal/encoding/encoding_test.go index 48e2316b0f897..d6fb38f7083f6 100644 --- a/go/parquet/internal/encoding/encoding_test.go +++ b/go/parquet/internal/encoding/encoding_test.go @@ -26,13 +26,13 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/internal/testutils" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go b/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go index 2054e1bb85f21..63b8b469c4b0b 100644 --- a/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go +++ b/go/parquet/internal/encoding/fixed_len_byte_array_decoder.go @@ -19,8 +19,8 @@ package encoding import ( "math" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go b/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go index 39202c8e25d9f..5e45d3f36b662 100644 --- a/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go +++ b/go/parquet/internal/encoding/fixed_len_byte_array_encoder.go @@ -19,9 +19,9 @@ package encoding import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/parquet" ) // PlainFixedLenByteArrayEncoder writes the raw bytes of the byte array diff --git a/go/parquet/internal/encoding/levels.go b/go/parquet/internal/encoding/levels.go index 2a6dc24933714..976df4e810e41 100644 --- a/go/parquet/internal/encoding/levels.go +++ b/go/parquet/internal/encoding/levels.go @@ -24,11 +24,11 @@ import ( "math/bits" "github.com/JohnCGriffin/overflow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/utils" ) // LevelEncoder is for handling the encoding of Definition and Repetition levels diff --git a/go/parquet/internal/encoding/levels_test.go b/go/parquet/internal/encoding/levels_test.go index 304ce32b3106d..6b31f4afe22b8 100644 --- a/go/parquet/internal/encoding/levels_test.go +++ b/go/parquet/internal/encoding/levels_test.go @@ -21,11 +21,11 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/internal/encoding/memo_table.go b/go/parquet/internal/encoding/memo_table.go index a36ad32973c96..df775b4cad709 100644 --- a/go/parquet/internal/encoding/memo_table.go +++ b/go/parquet/internal/encoding/memo_table.go @@ -20,11 +20,11 @@ import ( "math" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/hashing" + "github.com/apache/arrow/go/v16/parquet" ) //go:generate go run ../../../arrow/_tools/tmpl/main.go -i -data=physical_types.tmpldata memo_table_types.gen.go.tmpl diff --git a/go/parquet/internal/encoding/memo_table_test.go b/go/parquet/internal/encoding/memo_table_test.go index 1b9337010f855..5c0ad40c3920a 100644 --- a/go/parquet/internal/encoding/memo_table_test.go +++ b/go/parquet/internal/encoding/memo_table_test.go @@ -20,11 +20,11 @@ import ( "math" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/hashing" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/hashing" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/internal/encoding/memo_table_types.gen.go b/go/parquet/internal/encoding/memo_table_types.gen.go index 4da2721437814..dd165efaaa157 100644 --- a/go/parquet/internal/encoding/memo_table_types.gen.go +++ b/go/parquet/internal/encoding/memo_table_types.gen.go @@ -19,8 +19,8 @@ package encoding import ( - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" ) // standard map based implementation of memo tables which can be more efficient diff --git a/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl b/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl index 75335f25ff1f7..88f8d40d47ba7 100644 --- a/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl +++ b/go/parquet/internal/encoding/memo_table_types.gen.go.tmpl @@ -17,7 +17,7 @@ package encoding import ( - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/parquet" ) // standard map based implementation of memo tables which can be more efficient diff --git a/go/parquet/internal/encoding/plain_encoder_types.gen.go b/go/parquet/internal/encoding/plain_encoder_types.gen.go index a41f754f62a88..e8da35d52719f 100644 --- a/go/parquet/internal/encoding/plain_encoder_types.gen.go +++ b/go/parquet/internal/encoding/plain_encoder_types.gen.go @@ -24,11 +24,11 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl b/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl index 74f63e78bccf3..b60d71b2555fd 100644 --- a/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl +++ b/go/parquet/internal/encoding/plain_encoder_types.gen.go.tmpl @@ -20,10 +20,10 @@ import ( "encoding/binary" "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/internal/bitutils" ) var ( diff --git a/go/parquet/internal/encoding/typed_encoder.gen.go b/go/parquet/internal/encoding/typed_encoder.gen.go index 04db72178f3ee..77452da00b29b 100644 --- a/go/parquet/internal/encoding/typed_encoder.gen.go +++ b/go/parquet/internal/encoding/typed_encoder.gen.go @@ -22,15 +22,15 @@ import ( "fmt" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl b/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl index ceb755caa0b46..78ade64e750aa 100644 --- a/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl +++ b/go/parquet/internal/encoding/typed_encoder.gen.go.tmpl @@ -17,13 +17,13 @@ package encoding import ( - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/internal/bitutils" ) // fully typed encoder interfaces to enable writing against encoder/decoders diff --git a/go/parquet/internal/encoding/types.go b/go/parquet/internal/encoding/types.go index f8d860c88a059..18d101a65966f 100644 --- a/go/parquet/internal/encoding/types.go +++ b/go/parquet/internal/encoding/types.go @@ -20,11 +20,11 @@ import ( "io" "sync" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/encryption/aes.go b/go/parquet/internal/encryption/aes.go index b6e9130ef81cd..526669e35016d 100644 --- a/go/parquet/internal/encryption/aes.go +++ b/go/parquet/internal/encryption/aes.go @@ -29,7 +29,7 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/parquet" ) // important constants for handling the aes encryption diff --git a/go/parquet/internal/encryption/decryptor.go b/go/parquet/internal/encryption/decryptor.go index 658559e6082c4..2ea4130beb4e8 100644 --- a/go/parquet/internal/encryption/decryptor.go +++ b/go/parquet/internal/encryption/decryptor.go @@ -19,8 +19,8 @@ package encryption import ( "io" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" ) // FileDecryptor is an interface used by the filereader for decrypting an diff --git a/go/parquet/internal/encryption/encryptor.go b/go/parquet/internal/encryption/encryptor.go index bdbae4740a44f..d9616c16a9c64 100644 --- a/go/parquet/internal/encryption/encryptor.go +++ b/go/parquet/internal/encryption/encryptor.go @@ -19,8 +19,8 @@ package encryption import ( "io" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" ) // FileEncryptor is the interface for constructing encryptors for the different diff --git a/go/parquet/internal/testutils/pagebuilder.go b/go/parquet/internal/testutils/pagebuilder.go index 525921d9631f9..0b1fea64ad828 100644 --- a/go/parquet/internal/testutils/pagebuilder.go +++ b/go/parquet/internal/testutils/pagebuilder.go @@ -22,13 +22,13 @@ import ( "io" "reflect" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/mock" ) diff --git a/go/parquet/internal/testutils/primitive_typed.go b/go/parquet/internal/testutils/primitive_typed.go index 50627b2e275ff..ef50ace7fdf79 100644 --- a/go/parquet/internal/testutils/primitive_typed.go +++ b/go/parquet/internal/testutils/primitive_typed.go @@ -20,11 +20,11 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" ) type PrimitiveTypedTest struct { diff --git a/go/parquet/internal/testutils/random.go b/go/parquet/internal/testutils/random.go index 4d697693510d8..3139e7f7c94f0 100644 --- a/go/parquet/internal/testutils/random.go +++ b/go/parquet/internal/testutils/random.go @@ -24,14 +24,14 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/endian" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/endian" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/pqarrow" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" diff --git a/go/parquet/internal/testutils/random_arrow.go b/go/parquet/internal/testutils/random_arrow.go index 7dd2a3e8b77e3..91fa473adcb03 100644 --- a/go/parquet/internal/testutils/random_arrow.go +++ b/go/parquet/internal/testutils/random_arrow.go @@ -17,10 +17,10 @@ package testutils import ( - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" "golang.org/x/exp/rand" ) diff --git a/go/parquet/internal/testutils/utils.go b/go/parquet/internal/testutils/utils.go index 3da76c17ddc32..5a8f6cb3eeb0c 100644 --- a/go/parquet/internal/testutils/utils.go +++ b/go/parquet/internal/testutils/utils.go @@ -19,7 +19,7 @@ package testutils import ( "reflect" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/parquet" ) var typeToParquetTypeMap = map[reflect.Type]parquet.Type{ diff --git a/go/parquet/internal/thrift/helpers.go b/go/parquet/internal/thrift/helpers.go index 3835830ac6c2d..81c7e38fe7df7 100644 --- a/go/parquet/internal/thrift/helpers.go +++ b/go/parquet/internal/thrift/helpers.go @@ -23,7 +23,7 @@ import ( "context" "io" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" "github.com/apache/thrift/lib/go/thrift" ) diff --git a/go/parquet/internal/utils/bit_benchmark_test.go b/go/parquet/internal/utils/bit_benchmark_test.go index 14353380a5694..3ef548ed2d631 100644 --- a/go/parquet/internal/utils/bit_benchmark_test.go +++ b/go/parquet/internal/utils/bit_benchmark_test.go @@ -20,9 +20,9 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/parquet/internal/testutils" ) type linearBitRunReader struct { diff --git a/go/parquet/internal/utils/bit_packing_arm64.go b/go/parquet/internal/utils/bit_packing_arm64.go index e9fb2ef1d810b..0025db75c8755 100644 --- a/go/parquet/internal/utils/bit_packing_arm64.go +++ b/go/parquet/internal/utils/bit_packing_arm64.go @@ -23,7 +23,7 @@ import ( "github.com/klauspost/cpuid/v2" // import for side effect of initializing feature flags // based on ARM_ENABLE_EXT env var - _ "github.com/apache/arrow/go/v15/parquet/internal/bmi" + _ "github.com/apache/arrow/go/v16/parquet/internal/bmi" ) func init() { diff --git a/go/parquet/internal/utils/bit_reader.go b/go/parquet/internal/utils/bit_reader.go index d327be5f5253e..5ddbe3bea1ca6 100644 --- a/go/parquet/internal/utils/bit_reader.go +++ b/go/parquet/internal/utils/bit_reader.go @@ -24,10 +24,10 @@ import ( "reflect" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" ) // masks for grabbing the trailing bits based on the number of trailing bits desired diff --git a/go/parquet/internal/utils/bit_reader_test.go b/go/parquet/internal/utils/bit_reader_test.go index 5ce1b799b463b..3e5d4ed724bc5 100644 --- a/go/parquet/internal/utils/bit_reader_test.go +++ b/go/parquet/internal/utils/bit_reader_test.go @@ -25,11 +25,11 @@ import ( "strconv" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet/internal/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "golang.org/x/exp/rand" diff --git a/go/parquet/internal/utils/bit_writer.go b/go/parquet/internal/utils/bit_writer.go index 6cb255f5b0473..106461d33e048 100644 --- a/go/parquet/internal/utils/bit_writer.go +++ b/go/parquet/internal/utils/bit_writer.go @@ -21,7 +21,7 @@ import ( "io" "log" - "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/bitutil" ) // WriterAtBuffer is a convenience struct for providing a WriteAt function diff --git a/go/parquet/internal/utils/bitmap_writer.go b/go/parquet/internal/utils/bitmap_writer.go index 3ef99291e3748..514275f8728b1 100644 --- a/go/parquet/internal/utils/bitmap_writer.go +++ b/go/parquet/internal/utils/bitmap_writer.go @@ -20,7 +20,7 @@ import ( "encoding/binary" "math/bits" - "github.com/apache/arrow/go/v15/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/bitutil" ) // BitmapWriter is an interface for bitmap writers so that we can use multiple diff --git a/go/parquet/internal/utils/bitmap_writer_test.go b/go/parquet/internal/utils/bitmap_writer_test.go index 3dddc7567903e..812402ec8d0d0 100644 --- a/go/parquet/internal/utils/bitmap_writer_test.go +++ b/go/parquet/internal/utils/bitmap_writer_test.go @@ -22,8 +22,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/parquet/internal/utils" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/parquet/internal/utils" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/internal/utils/rle.go b/go/parquet/internal/utils/rle.go index dffe55402b95a..5c668567b23f4 100644 --- a/go/parquet/internal/utils/rle.go +++ b/go/parquet/internal/utils/rle.go @@ -24,10 +24,10 @@ import ( "encoding/binary" "math" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/utils/typed_rle_dict.gen.go b/go/parquet/internal/utils/typed_rle_dict.gen.go index 37dc49a695806..ef67ee1fe2f36 100644 --- a/go/parquet/internal/utils/typed_rle_dict.gen.go +++ b/go/parquet/internal/utils/typed_rle_dict.gen.go @@ -19,9 +19,9 @@ package utils import ( - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl b/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl index 88c7dd979ebf1..f4fa5dedc0636 100644 --- a/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl +++ b/go/parquet/internal/utils/typed_rle_dict.gen.go.tmpl @@ -17,9 +17,9 @@ package utils import ( - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/internal/utils" ) {{range .In}} diff --git a/go/parquet/metadata/app_version.go b/go/parquet/metadata/app_version.go index 9966827026106..6c4f06a6c0e6d 100644 --- a/go/parquet/metadata/app_version.go +++ b/go/parquet/metadata/app_version.go @@ -21,8 +21,8 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" ) var ( diff --git a/go/parquet/metadata/column_chunk.go b/go/parquet/metadata/column_chunk.go index 729f741e1b4f9..8cda380d39e3d 100644 --- a/go/parquet/metadata/column_chunk.go +++ b/go/parquet/metadata/column_chunk.go @@ -22,13 +22,13 @@ import ( "io" "reflect" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/thrift" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/file.go b/go/parquet/metadata/file.go index fbbbae1eef892..f40081f172a75 100644 --- a/go/parquet/metadata/file.go +++ b/go/parquet/metadata/file.go @@ -24,12 +24,12 @@ import ( "reflect" "unicode/utf8" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/thrift" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/thrift" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/metadata_test.go b/go/parquet/metadata/metadata_test.go index 8caa319f83e63..872d92325f655 100644 --- a/go/parquet/metadata/metadata_test.go +++ b/go/parquet/metadata/metadata_test.go @@ -21,9 +21,9 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/metadata/row_group.go b/go/parquet/metadata/row_group.go index 2923720371abe..fec5d95540acc 100644 --- a/go/parquet/metadata/row_group.go +++ b/go/parquet/metadata/row_group.go @@ -20,10 +20,10 @@ import ( "fmt" "reflect" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encryption" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encryption" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" ) // RowGroupMetaData is a proxy around the thrift RowGroup meta data object diff --git a/go/parquet/metadata/stat_compare_test.go b/go/parquet/metadata/stat_compare_test.go index 041696d84d17a..ad403d15f0082 100644 --- a/go/parquet/metadata/stat_compare_test.go +++ b/go/parquet/metadata/stat_compare_test.go @@ -20,8 +20,8 @@ import ( "encoding/binary" "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/metadata/statistics.go b/go/parquet/metadata/statistics.go index a9bda405bb9b5..ba52e247d458b 100644 --- a/go/parquet/metadata/statistics.go +++ b/go/parquet/metadata/statistics.go @@ -22,15 +22,15 @@ import ( "math" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/debug" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" ) //go:generate go run ../../arrow/_tools/tmpl/main.go -i -data=statistics_types.tmpldata statistics_types.gen.go.tmpl diff --git a/go/parquet/metadata/statistics_test.go b/go/parquet/metadata/statistics_test.go index 19311dc8955d3..02c19c1039968 100644 --- a/go/parquet/metadata/statistics_test.go +++ b/go/parquet/metadata/statistics_test.go @@ -21,12 +21,12 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/metadata/statistics_types.gen.go b/go/parquet/metadata/statistics_types.gen.go index a0e2949251368..3854898354c85 100644 --- a/go/parquet/metadata/statistics_types.gen.go +++ b/go/parquet/metadata/statistics_types.gen.go @@ -22,15 +22,15 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/metadata/statistics_types.gen.go.tmpl b/go/parquet/metadata/statistics_types.gen.go.tmpl index 26fe7f1531999..7dbb9b2d8c4c0 100644 --- a/go/parquet/metadata/statistics_types.gen.go.tmpl +++ b/go/parquet/metadata/statistics_types.gen.go.tmpl @@ -19,13 +19,13 @@ package metadata import ( "fmt" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" - "github.com/apache/arrow/go/v15/parquet/internal/utils" - shared_utils "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/internal/bitutils" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" + "github.com/apache/arrow/go/v16/parquet/internal/utils" + shared_utils "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/internal/bitutils" ) {{range .In}} diff --git a/go/parquet/pqarrow/column_readers.go b/go/parquet/pqarrow/column_readers.go index a403b2196a80c..440753de24f65 100644 --- a/go/parquet/pqarrow/column_readers.go +++ b/go/parquet/pqarrow/column_readers.go @@ -26,16 +26,16 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/sync/errgroup" ) diff --git a/go/parquet/pqarrow/encode_arrow.go b/go/parquet/pqarrow/encode_arrow.go index 0836d135243da..cad9d14e5b319 100644 --- a/go/parquet/pqarrow/encode_arrow.go +++ b/go/parquet/pqarrow/encode_arrow.go @@ -25,16 +25,16 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/debug" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/debug" ) // get the count of the number of leaf arrays for the type diff --git a/go/parquet/pqarrow/encode_arrow_test.go b/go/parquet/pqarrow/encode_arrow_test.go index 25d31b54e1b31..fe82a89e88fba 100644 --- a/go/parquet/pqarrow/encode_arrow_test.go +++ b/go/parquet/pqarrow/encode_arrow_test.go @@ -25,22 +25,22 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/bitutil" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/decimal256" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/pqarrow" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/bitutil" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/decimal256" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/parquet/internal/testutils" + "github.com/apache/arrow/go/v16/parquet/pqarrow" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/pqarrow/encode_dict_compute.go b/go/parquet/pqarrow/encode_dict_compute.go index b43b4002ed0af..f2e257024cfcc 100644 --- a/go/parquet/pqarrow/encode_dict_compute.go +++ b/go/parquet/pqarrow/encode_dict_compute.go @@ -21,14 +21,14 @@ package pqarrow import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/debug" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" ) func isDictEncoding(enc parquet.Encoding) bool { diff --git a/go/parquet/pqarrow/encode_dict_nocompute.go b/go/parquet/pqarrow/encode_dict_nocompute.go index 73ec5cfc46682..db3d248aa80cf 100644 --- a/go/parquet/pqarrow/encode_dict_nocompute.go +++ b/go/parquet/pqarrow/encode_dict_nocompute.go @@ -21,8 +21,8 @@ package pqarrow import ( "errors" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/parquet/file" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/parquet/file" ) func writeDictionaryArrow(*arrowWriteContext, file.ColumnChunkWriter, arrow.Array, []int16, []int16, bool) (err error) { diff --git a/go/parquet/pqarrow/encode_dictionary_test.go b/go/parquet/pqarrow/encode_dictionary_test.go index 28ebee53e1b83..2f16fc99b5367 100644 --- a/go/parquet/pqarrow/encode_dictionary_test.go +++ b/go/parquet/pqarrow/encode_dictionary_test.go @@ -26,14 +26,14 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/compute" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/internal/testutils" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/internal/testutils" + "github.com/apache/arrow/go/v16/parquet/pqarrow" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/go/parquet/pqarrow/file_reader.go b/go/parquet/pqarrow/file_reader.go index a55dbe46e07fa..7cd922dcee87c 100755 --- a/go/parquet/pqarrow/file_reader.go +++ b/go/parquet/pqarrow/file_reader.go @@ -23,13 +23,13 @@ import ( "sync" "sync/atomic" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/arrio" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/arrio" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/file_reader_test.go b/go/parquet/pqarrow/file_reader_test.go index 0c52eec9e3459..1547004ebf076 100644 --- a/go/parquet/pqarrow/file_reader_test.go +++ b/go/parquet/pqarrow/file_reader_test.go @@ -26,14 +26,14 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/pqarrow" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/pqarrow/file_writer.go b/go/parquet/pqarrow/file_writer.go index 1164cd690c399..b989add03409a 100644 --- a/go/parquet/pqarrow/file_writer.go +++ b/go/parquet/pqarrow/file_writer.go @@ -22,12 +22,12 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/internal/utils" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/metadata" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/internal/utils" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/metadata" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/file_writer_test.go b/go/parquet/pqarrow/file_writer_test.go index 0b76733a62876..8e80df5179d01 100644 --- a/go/parquet/pqarrow/file_writer_test.go +++ b/go/parquet/pqarrow/file_writer_test.go @@ -21,11 +21,11 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/pqarrow" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/pqarrow/helpers.go b/go/parquet/pqarrow/helpers.go index 39d17cf80ae68..db141c377fb33 100644 --- a/go/parquet/pqarrow/helpers.go +++ b/go/parquet/pqarrow/helpers.go @@ -17,7 +17,7 @@ package pqarrow import ( - "github.com/apache/arrow/go/v15/arrow" + "github.com/apache/arrow/go/v16/arrow" ) func releaseArrays(arrays []arrow.Array) { diff --git a/go/parquet/pqarrow/path_builder.go b/go/parquet/pqarrow/path_builder.go index 6b94205f5dcc8..42352de69eed0 100644 --- a/go/parquet/pqarrow/path_builder.go +++ b/go/parquet/pqarrow/path_builder.go @@ -21,11 +21,11 @@ import ( "sync/atomic" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/bitutils" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/bitutils" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/path_builder_test.go b/go/parquet/pqarrow/path_builder_test.go index e0a60262d3f4a..54fccf6cd22d7 100644 --- a/go/parquet/pqarrow/path_builder_test.go +++ b/go/parquet/pqarrow/path_builder_test.go @@ -20,10 +20,10 @@ import ( "context" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/go/parquet/pqarrow/properties.go b/go/parquet/pqarrow/properties.go index cc100fa80d87b..d301e5bff9885 100755 --- a/go/parquet/pqarrow/properties.go +++ b/go/parquet/pqarrow/properties.go @@ -19,9 +19,9 @@ package pqarrow import ( "context" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet/internal/encoding" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet/internal/encoding" ) // ArrowWriterProperties are used to determine how to manipulate the arrow data diff --git a/go/parquet/pqarrow/reader_writer_test.go b/go/parquet/pqarrow/reader_writer_test.go index 9d09bcec15da6..b3bf8ab8b29fe 100644 --- a/go/parquet/pqarrow/reader_writer_test.go +++ b/go/parquet/pqarrow/reader_writer_test.go @@ -22,12 +22,12 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/array" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/pqarrow" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/pqarrow" "golang.org/x/exp/rand" "gonum.org/v1/gonum/stat/distuv" ) diff --git a/go/parquet/pqarrow/schema.go b/go/parquet/pqarrow/schema.go index f2aa4cdfe05ad..c3694482f425e 100644 --- a/go/parquet/pqarrow/schema.go +++ b/go/parquet/pqarrow/schema.go @@ -22,15 +22,15 @@ import ( "math" "strconv" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/decimal128" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/file" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/decimal128" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/schema" "golang.org/x/xerrors" ) diff --git a/go/parquet/pqarrow/schema_test.go b/go/parquet/pqarrow/schema_test.go index f320b903033db..5edebb369ff1d 100644 --- a/go/parquet/pqarrow/schema_test.go +++ b/go/parquet/pqarrow/schema_test.go @@ -20,15 +20,15 @@ import ( "encoding/base64" "testing" - "github.com/apache/arrow/go/v15/arrow" - "github.com/apache/arrow/go/v15/arrow/flight" - "github.com/apache/arrow/go/v15/arrow/ipc" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/types" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/metadata" - "github.com/apache/arrow/go/v15/parquet/pqarrow" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/flight" + "github.com/apache/arrow/go/v16/arrow/ipc" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/types" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/apache/arrow/go/v16/parquet/pqarrow" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go/parquet/reader_properties.go b/go/parquet/reader_properties.go index be2377527d782..d7d75d3a23c16 100644 --- a/go/parquet/reader_properties.go +++ b/go/parquet/reader_properties.go @@ -21,8 +21,8 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/internal/utils" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/internal/utils" ) // ReaderProperties are used to define how the file reader will handle buffering and allocating buffers diff --git a/go/parquet/reader_writer_properties_test.go b/go/parquet/reader_writer_properties_test.go index 698129471adda..538896c86bc59 100644 --- a/go/parquet/reader_writer_properties_test.go +++ b/go/parquet/reader_writer_properties_test.go @@ -20,9 +20,9 @@ import ( "bytes" "testing" - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/column.go b/go/parquet/schema/column.go index 02d0d83e8465e..d6747795d1c76 100644 --- a/go/parquet/schema/column.go +++ b/go/parquet/schema/column.go @@ -20,8 +20,8 @@ import ( "fmt" "strings" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" ) // Column encapsulates the information necessary to interpret primitive diff --git a/go/parquet/schema/converted_types.go b/go/parquet/schema/converted_types.go index 58051dba7f19b..9417881a59f14 100644 --- a/go/parquet/schema/converted_types.go +++ b/go/parquet/schema/converted_types.go @@ -17,7 +17,7 @@ package schema import ( - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" ) // ConvertedType corresponds to the ConvertedType in the parquet.Thrift, diff --git a/go/parquet/schema/converted_types_test.go b/go/parquet/schema/converted_types_test.go index 8b8e061466474..3148a6e4b1f78 100644 --- a/go/parquet/schema/converted_types_test.go +++ b/go/parquet/schema/converted_types_test.go @@ -19,7 +19,7 @@ package schema_test import ( "testing" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/helpers.go b/go/parquet/schema/helpers.go index 13075a065f073..a4ce5e4ea8bad 100644 --- a/go/parquet/schema/helpers.go +++ b/go/parquet/schema/helpers.go @@ -17,7 +17,7 @@ package schema import ( - "github.com/apache/arrow/go/v15/parquet" + "github.com/apache/arrow/go/v16/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/helpers_test.go b/go/parquet/schema/helpers_test.go index 98f3cab36d433..5f7c0f7c0e5d0 100644 --- a/go/parquet/schema/helpers_test.go +++ b/go/parquet/schema/helpers_test.go @@ -21,8 +21,8 @@ import ( "strings" "testing" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/logical_types.go b/go/parquet/schema/logical_types.go index 69e69363887cd..90d0bdfcd1e6a 100644 --- a/go/parquet/schema/logical_types.go +++ b/go/parquet/schema/logical_types.go @@ -20,10 +20,10 @@ import ( "fmt" "math" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/internal/debug" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/internal/json" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/internal/debug" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" ) // DecimalMetadata is a struct for managing scale and precision information between diff --git a/go/parquet/schema/logical_types_test.go b/go/parquet/schema/logical_types_test.go index 0fd91daf8d668..18247118b752e 100644 --- a/go/parquet/schema/logical_types_test.go +++ b/go/parquet/schema/logical_types_test.go @@ -19,9 +19,9 @@ package schema_test import ( "testing" - "github.com/apache/arrow/go/v15/internal/json" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/internal/json" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/node.go b/go/parquet/schema/node.go index c1b325eb90183..611f52c9e1215 100644 --- a/go/parquet/schema/node.go +++ b/go/parquet/schema/node.go @@ -19,8 +19,8 @@ package schema import ( "fmt" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" "github.com/apache/thrift/lib/go/thrift" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/reflection.go b/go/parquet/schema/reflection.go index 7f8b337795592..4eee3c86c35f0 100644 --- a/go/parquet/schema/reflection.go +++ b/go/parquet/schema/reflection.go @@ -22,9 +22,9 @@ import ( "strconv" "strings" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/reflection_test.go b/go/parquet/schema/reflection_test.go index e3a880cacc1e8..9fa289388ec65 100644 --- a/go/parquet/schema/reflection_test.go +++ b/go/parquet/schema/reflection_test.go @@ -22,9 +22,9 @@ import ( "reflect" "testing" - "github.com/apache/arrow/go/v15/arrow/float16" - "github.com/apache/arrow/go/v15/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/arrow/float16" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/stretchr/testify/assert" ) diff --git a/go/parquet/schema/schema.go b/go/parquet/schema/schema.go index ace2775763a06..e4864e9330622 100644 --- a/go/parquet/schema/schema.go +++ b/go/parquet/schema/schema.go @@ -35,8 +35,8 @@ import ( "io" "strings" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" "golang.org/x/xerrors" ) diff --git a/go/parquet/schema/schema_element_test.go b/go/parquet/schema/schema_element_test.go index d190ffe5a253a..af281146980af 100644 --- a/go/parquet/schema/schema_element_test.go +++ b/go/parquet/schema/schema_element_test.go @@ -19,8 +19,8 @@ package schema import ( "testing" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/schema/schema_flatten_test.go b/go/parquet/schema/schema_flatten_test.go index 34f4ac8d3c450..6b7de6b9b718c 100644 --- a/go/parquet/schema/schema_flatten_test.go +++ b/go/parquet/schema/schema_flatten_test.go @@ -19,8 +19,8 @@ package schema import ( "testing" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/go/parquet/schema/schema_test.go b/go/parquet/schema/schema_test.go index 232a386a25470..3f0bdab79aa9e 100644 --- a/go/parquet/schema/schema_test.go +++ b/go/parquet/schema/schema_test.go @@ -20,9 +20,9 @@ import ( "os" "testing" - "github.com/apache/arrow/go/v15/parquet" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" - "github.com/apache/arrow/go/v15/parquet/schema" + "github.com/apache/arrow/go/v16/parquet" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/parquet/schema" "github.com/apache/thrift/lib/go/thrift" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" diff --git a/go/parquet/types.go b/go/parquet/types.go index 0020a079d94a9..8742c3ba8bfba 100644 --- a/go/parquet/types.go +++ b/go/parquet/types.go @@ -24,8 +24,8 @@ import ( "time" "unsafe" - "github.com/apache/arrow/go/v15/arrow" - format "github.com/apache/arrow/go/v15/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/v16/arrow" + format "github.com/apache/arrow/go/v16/parquet/internal/gen-go/parquet" ) const ( diff --git a/go/parquet/writer_properties.go b/go/parquet/writer_properties.go index 9e33bddf7faa7..7fd3ed9f4be8d 100644 --- a/go/parquet/writer_properties.go +++ b/go/parquet/writer_properties.go @@ -17,8 +17,8 @@ package parquet import ( - "github.com/apache/arrow/go/v15/arrow/memory" - "github.com/apache/arrow/go/v15/parquet/compress" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet/compress" ) // Constants for default property values used for the default reader, writer and column props. @@ -46,7 +46,7 @@ const ( DefaultStatsEnabled = true // If the stats are larger than 4K the writer will skip writing them out anyways. DefaultMaxStatsSize int64 = 4096 - DefaultCreatedBy = "parquet-go version 15.0.0-SNAPSHOT" + DefaultCreatedBy = "parquet-go version 16.0.0-SNAPSHOT" DefaultRootName = "schema" ) diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index 90864eab006a2..6644748b5e597 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -16,7 +16,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT ../../pom.xml diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index e964aa1871a0e..dfcd4cfe8f61b 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -16,7 +16,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT ../../pom.xml diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index 605b9871639ea..265a9a71b80e2 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -115,7 +115,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT ../../pom.xml diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index 99740f2002847..25669010d2d42 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT arrow-algorithm Arrow Algorithms diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 7ffb833e7f6d7..025632c45a56d 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-bom - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT pom Arrow Bill of Materials Arrow Bill of Materials diff --git a/java/c/pom.xml b/java/c/pom.xml index a999292979d56..ffd41b62dd674 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -13,7 +13,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT 4.0.0 diff --git a/java/compression/pom.xml b/java/compression/pom.xml index e8008c9754374..dea8c778735a8 100644 --- a/java/compression/pom.xml +++ b/java/compression/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT arrow-compression Arrow Compression diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index a18f443b7e15a..8723fafa8dadd 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT 4.0.0 diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index ec3034d14e271..b7624d7748e7f 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -14,7 +14,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT ../pom.xml diff --git a/java/flight/flight-integration-tests/pom.xml b/java/flight/flight-integration-tests/pom.xml index bb4f6a6b18733..944c624d630a2 100644 --- a/java/flight/flight-integration-tests/pom.xml +++ b/java/flight/flight-integration-tests/pom.xml @@ -15,7 +15,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT ../pom.xml diff --git a/java/flight/flight-sql-jdbc-core/pom.xml b/java/flight/flight-sql-jdbc-core/pom.xml index 1f20912b9974f..ce1f52e39676e 100644 --- a/java/flight/flight-sql-jdbc-core/pom.xml +++ b/java/flight/flight-sql-jdbc-core/pom.xml @@ -16,7 +16,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml index 653ee5c192756..9ab9e1820cd27 100644 --- a/java/flight/flight-sql-jdbc-driver/pom.xml +++ b/java/flight/flight-sql-jdbc-driver/pom.xml @@ -16,7 +16,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index 5ae1a0a23f3c8..a0598f70b9545 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -14,7 +14,7 @@ arrow-flight org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT ../pom.xml diff --git a/java/flight/pom.xml b/java/flight/pom.xml index 9ef01d07a7388..3ce1b1149ef20 100644 --- a/java/flight/pom.xml +++ b/java/flight/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT 4.0.0 diff --git a/java/format/pom.xml b/java/format/pom.xml index 3f581311e20ea..a98edefbeb217 100644 --- a/java/format/pom.xml +++ b/java/format/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT arrow-format diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index 330c156a0346b..d0290b6814ed5 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT org.apache.arrow.gandiva diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index 70d1993b33c6e..e6bf8a63cb74a 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -16,7 +16,7 @@ org.apache.arrow.maven.plugins arrow-maven-plugins - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT module-info-compiler-maven-plugin maven-plugin diff --git a/java/maven/pom.xml b/java/maven/pom.xml index 6e8a4cb0102f6..3a88ec762e19c 100644 --- a/java/maven/pom.xml +++ b/java/maven/pom.xml @@ -17,7 +17,7 @@ --> org.apache.arrow.maven.plugins arrow-maven-plugins - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT Arrow Maven Plugins pom diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml index 6e411c0cd5440..2a92d032942c9 100644 --- a/java/memory/memory-core/pom.xml +++ b/java/memory/memory-core/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT 4.0.0 diff --git a/java/memory/memory-netty-buffer-patch/pom.xml b/java/memory/memory-netty-buffer-patch/pom.xml index 1d4407c638d8a..97b224e9ccc5c 100644 --- a/java/memory/memory-netty-buffer-patch/pom.xml +++ b/java/memory/memory-netty-buffer-patch/pom.xml @@ -15,7 +15,7 @@ arrow-memory org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT 4.0.0 diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml index 159ab5160c983..9b20e1bde2ae7 100644 --- a/java/memory/memory-netty/pom.xml +++ b/java/memory/memory-netty/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT 4.0.0 diff --git a/java/memory/memory-unsafe/pom.xml b/java/memory/memory-unsafe/pom.xml index 5ef4e8a9149a5..07a140e594522 100644 --- a/java/memory/memory-unsafe/pom.xml +++ b/java/memory/memory-unsafe/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT 4.0.0 diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 55fbb90353f34..9e2d612765738 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT arrow-memory Arrow Memory diff --git a/java/performance/pom.xml b/java/performance/pom.xml index eff3240890beb..a302a216c53d3 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -14,7 +14,7 @@ arrow-java-root org.apache.arrow - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT arrow-performance jar diff --git a/java/pom.xml b/java/pom.xml index 6fc4df67af3e7..3951f1c1bc8ed 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT pom Apache Arrow Java Root POM diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 0d7eacfe2ddce..0688fae1ab78c 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT arrow-tools Arrow Tools diff --git a/java/vector/pom.xml b/java/vector/pom.xml index da26fc2982765..dc453963b62f6 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0-SNAPSHOT + 16.0.0-SNAPSHOT arrow-vector Arrow Vectors diff --git a/js/package.json b/js/package.json index d1346eb37c9ed..57f9267afa3a8 100644 --- a/js/package.json +++ b/js/package.json @@ -121,5 +121,5 @@ "engines": { "node": ">=12.0" }, - "version": "15.0.0-SNAPSHOT" + "version": "16.0.0-SNAPSHOT" } diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index 47d2acd613f8b..206ecb318b3cc 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -94,7 +94,7 @@ endfunction() set(CMAKE_CXX_STANDARD 17) -set(MLARROW_VERSION "15.0.0-SNAPSHOT") +set(MLARROW_VERSION "16.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" MLARROW_BASE_VERSION "${MLARROW_VERSION}") project(mlarrow VERSION "${MLARROW_BASE_VERSION}") diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 2df1e67b9f4c7..54a5b99e058a5 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -21,7 +21,7 @@ cmake_minimum_required(VERSION 3.16) project(pyarrow) -set(PYARROW_VERSION "15.0.0-SNAPSHOT") +set(PYARROW_VERSION "16.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" PYARROW_BASE_VERSION "${PYARROW_VERSION}") # Running from a Python sdist tarball diff --git a/python/setup.py b/python/setup.py index 51eb40af088e5..d7a2da2077cdd 100755 --- a/python/setup.py +++ b/python/setup.py @@ -407,7 +407,7 @@ def get_outputs(self): # If the event of not running from a git clone (e.g. from a git archive # or a Python sdist), see if we can set the version number ourselves -default_version = '15.0.0-SNAPSHOT' +default_version = '16.0.0-SNAPSHOT' if (not os.path.exists('../.git') and not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')): os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'] = \ diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 4acd21269cc49..21cc4dec902d2 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -1,6 +1,6 @@ Package: arrow Title: Integration to 'Apache' 'Arrow' -Version: 14.0.2.9000 +Version: 15.0.0.9000 Authors@R: c( person("Neal", "Richardson", email = "neal.p.richardson@gmail.com", role = c("aut")), person("Ian", "Cook", email = "ianmcook@gmail.com", role = c("aut")), diff --git a/r/NEWS.md b/r/NEWS.md index 22eb5b34ceb0f..58c82c5128b82 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -17,7 +17,9 @@ under the License. --> -# arrow 14.0.2.9000 +# arrow 15.0.0.9000 + +# arrow 15.0.0 ## New features diff --git a/r/pkgdown/assets/versions.json b/r/pkgdown/assets/versions.json index 35a1ef3b5ecb3..0b7f9884f9b6f 100644 --- a/r/pkgdown/assets/versions.json +++ b/r/pkgdown/assets/versions.json @@ -1,12 +1,16 @@ [ { - "name": "14.0.2.9000 (dev)", + "name": "15.0.0.9000 (dev)", "version": "dev/" }, { - "name": "14.0.2 (release)", + "name": "15.0.0 (release)", "version": "" }, + { + "name": "14.0.2", + "version": "14.0/" + }, { "name": "13.0.0.1", "version": "13.0/" diff --git a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb index 8551b647cb86f..816751fcba8ff 100644 --- a/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb +++ b/ruby/red-arrow-cuda/lib/arrow-cuda/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowCUDA - VERSION = "15.0.0-SNAPSHOT" + VERSION = "16.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb index acfdd675687be..e391493e15974 100644 --- a/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb +++ b/ruby/red-arrow-dataset/lib/arrow-dataset/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowDataset - VERSION = "15.0.0-SNAPSHOT" + VERSION = "16.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb index 3354678e30032..d90751be80cb0 100644 --- a/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb +++ b/ruby/red-arrow-flight-sql/lib/arrow-flight-sql/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlightSQL - VERSION = "15.0.0-SNAPSHOT" + VERSION = "16.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow-flight/lib/arrow-flight/version.rb b/ruby/red-arrow-flight/lib/arrow-flight/version.rb index f2141a68432e5..6c2d676809f8f 100644 --- a/ruby/red-arrow-flight/lib/arrow-flight/version.rb +++ b/ruby/red-arrow-flight/lib/arrow-flight/version.rb @@ -16,7 +16,7 @@ # under the License. module ArrowFlight - VERSION = "15.0.0-SNAPSHOT" + VERSION = "16.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-arrow/lib/arrow/version.rb b/ruby/red-arrow/lib/arrow/version.rb index 235a7df75d672..2b1c14e389116 100644 --- a/ruby/red-arrow/lib/arrow/version.rb +++ b/ruby/red-arrow/lib/arrow/version.rb @@ -16,7 +16,7 @@ # under the License. module Arrow - VERSION = "15.0.0-SNAPSHOT" + VERSION = "16.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-gandiva/lib/gandiva/version.rb b/ruby/red-gandiva/lib/gandiva/version.rb index 6a1835f0e50e8..0a20a520194b0 100644 --- a/ruby/red-gandiva/lib/gandiva/version.rb +++ b/ruby/red-gandiva/lib/gandiva/version.rb @@ -16,7 +16,7 @@ # under the License. module Gandiva - VERSION = "15.0.0-SNAPSHOT" + VERSION = "16.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") diff --git a/ruby/red-parquet/lib/parquet/version.rb b/ruby/red-parquet/lib/parquet/version.rb index c5a945c4e4297..cd61c772a3285 100644 --- a/ruby/red-parquet/lib/parquet/version.rb +++ b/ruby/red-parquet/lib/parquet/version.rb @@ -16,7 +16,7 @@ # under the License. module Parquet - VERSION = "15.0.0-SNAPSHOT" + VERSION = "16.0.0-SNAPSHOT" module Version numbers, TAG = VERSION.split("-") From 48d47525b838c2455f198a0dba984a79f8ca8d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sun, 21 Jan 2024 16:06:39 +0100 Subject: [PATCH 54/92] MINOR: [Release] Update .deb package names for 16.0.0 --- .../apache-arrow/debian/control.in | 104 +++++++++--------- ...500.install => libarrow-acero1600.install} | 0 ...install => libarrow-cuda-glib1600.install} | 0 ...1500.install => libarrow-cuda1600.install} | 0 ...tall => libarrow-dataset-glib1600.install} | 0 ...0.install => libarrow-dataset1600.install} | 0 ...stall => libarrow-flight-glib1600.install} | 0 ...l => libarrow-flight-sql-glib1600.install} | 0 ...nstall => libarrow-flight-sql1600.install} | 0 ...00.install => libarrow-flight1600.install} | 0 ...1500.install => libarrow-glib1600.install} | 0 ...arrow1500.install => libarrow1600.install} | 0 ...00.install => libgandiva-glib1600.install} | 0 ...iva1500.install => libgandiva1600.install} | 0 ...00.install => libparquet-glib1600.install} | 0 ...uet1500.install => libparquet1600.install} | 0 dev/tasks/tasks.yml | 60 +++++----- 17 files changed, 82 insertions(+), 82 deletions(-) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-acero1500.install => libarrow-acero1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-cuda-glib1500.install => libarrow-cuda-glib1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-cuda1500.install => libarrow-cuda1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-dataset-glib1500.install => libarrow-dataset-glib1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-dataset1500.install => libarrow-dataset1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-flight-glib1500.install => libarrow-flight-glib1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-flight-sql-glib1500.install => libarrow-flight-sql-glib1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-flight-sql1500.install => libarrow-flight-sql1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-flight1500.install => libarrow-flight1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow-glib1500.install => libarrow-glib1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libarrow1500.install => libarrow1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libgandiva-glib1500.install => libgandiva-glib1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libgandiva1500.install => libgandiva1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libparquet-glib1500.install => libparquet-glib1600.install} (100%) rename dev/tasks/linux-packages/apache-arrow/debian/{libparquet1500.install => libparquet1600.install} (100%) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in index 6ea7e56e88365..5b8de89dcd67e 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/control.in +++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in @@ -41,7 +41,7 @@ Build-Depends-Indep: libglib2.0-doc Standards-Version: 3.9.8 Homepage: https://arrow.apache.org/ -Package: libarrow1500 +Package: libarrow1600 Section: libs Architecture: any Multi-Arch: same @@ -61,12 +61,12 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow1500 (= ${binary:Version}) + libarrow1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides tools. -Package: libarrow-cuda1500 +Package: libarrow-cuda1600 Section: libs Architecture: @CUDA_ARCHITECTURE@ Multi-Arch: same @@ -74,12 +74,12 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow1500 (= ${binary:Version}) + libarrow1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for CUDA support. -Package: libarrow-acero1500 +Package: libarrow-acero1600 Section: libs Architecture: any Multi-Arch: same @@ -87,12 +87,12 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow1500 (= ${binary:Version}) + libarrow1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for Acero module. -Package: libarrow-dataset1500 +Package: libarrow-dataset1600 Section: libs Architecture: any Multi-Arch: same @@ -100,13 +100,13 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow-acero1500 (= ${binary:Version}), - libparquet1500 (= ${binary:Version}) + libarrow-acero1600 (= ${binary:Version}), + libparquet1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for Dataset module. -Package: libarrow-flight1500 +Package: libarrow-flight1600 Section: libs Architecture: any Multi-Arch: same @@ -114,12 +114,12 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow1500 (= ${binary:Version}) + libarrow1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for Flight RPC system. -Package: libarrow-flight-sql1500 +Package: libarrow-flight-sql1600 Section: libs Architecture: any Multi-Arch: same @@ -127,7 +127,7 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow-flight1500 (= ${binary:Version}) + libarrow-flight1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for Flight SQL system. @@ -138,7 +138,7 @@ Architecture: any Multi-Arch: same Depends: ${misc:Depends}, - libarrow1500 (= ${binary:Version}), + libarrow1600 (= ${binary:Version}), libbrotli-dev, libbz2-dev, libcurl4-openssl-dev, @@ -163,7 +163,7 @@ Multi-Arch: same Depends: ${misc:Depends}, libarrow-dev (= ${binary:Version}), - libarrow-cuda1500 (= ${binary:Version}) + libarrow-cuda1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides C++ header files for CUDA support. @@ -174,7 +174,7 @@ Architecture: any Multi-Arch: same Depends: ${misc:Depends}, - libarrow-acero1500 (= ${binary:Version}), + libarrow-acero1600 (= ${binary:Version}), libparquet-dev (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . @@ -187,7 +187,7 @@ Multi-Arch: same Depends: ${misc:Depends}, libarrow-acero-dev (= ${binary:Version}), - libarrow-dataset1500 (= ${binary:Version}), + libarrow-dataset1600 (= ${binary:Version}), libparquet-dev (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . @@ -200,7 +200,7 @@ Multi-Arch: same Depends: ${misc:Depends}, libarrow-dev (= ${binary:Version}), - libarrow-flight1500 (= ${binary:Version}), + libarrow-flight1600 (= ${binary:Version}), libc-ares-dev, @USE_SYSTEM_GRPC@ libgrpc++-dev, @USE_SYSTEM_PROTOBUF@ libprotobuf-dev, @@ -216,12 +216,12 @@ Multi-Arch: same Depends: ${misc:Depends}, libarrow-flight-dev (= ${binary:Version}), - libarrow-flight-sql1500 (= ${binary:Version}) + libarrow-flight-sql1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides C++ header files for Flight SQL system. -Package: libgandiva1500 +Package: libgandiva1600 Section: libs Architecture: any Multi-Arch: same @@ -229,7 +229,7 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow1500 (= ${binary:Version}) + libarrow1600 (= ${binary:Version}) Description: Gandiva is a toolset for compiling and evaluating expressions on Arrow Data. . @@ -242,13 +242,13 @@ Multi-Arch: same Depends: ${misc:Depends}, libarrow-dev (= ${binary:Version}), - libgandiva1500 (= ${binary:Version}) + libgandiva1600 (= ${binary:Version}) Description: Gandiva is a toolset for compiling and evaluating expressions on Arrow Data. . This package provides C++ header files. -Package: libparquet1500 +Package: libparquet1600 Section: libs Architecture: any Multi-Arch: same @@ -268,7 +268,7 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libparquet1500 (= ${binary:Version}) + libparquet1600 (= ${binary:Version}) Description: Apache Parquet is a columnar storage format . This package provides tools. @@ -280,13 +280,13 @@ Multi-Arch: same Depends: ${misc:Depends}, libarrow-dev (= ${binary:Version}), - libparquet1500 (= ${binary:Version}), + libparquet1600 (= ${binary:Version}), libthrift-dev Description: Apache Parquet is a columnar storage format . This package provides C++ header files. -Package: libarrow-glib1500 +Package: libarrow-glib1600 Section: libs Architecture: any Multi-Arch: same @@ -294,7 +294,7 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow1500 (= ${binary:Version}) + libarrow1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides GLib based library files. @@ -318,7 +318,7 @@ Depends: ${misc:Depends}, libglib2.0-dev, libarrow-acero-dev (= ${binary:Version}), - libarrow-glib1500 (= ${binary:Version}), + libarrow-glib1600 (= ${binary:Version}), gir1.2-arrow-1.0 (= ${binary:Version}) Suggests: libarrow-glib-doc Description: Apache Arrow is a data processing library for analysis @@ -336,7 +336,7 @@ Description: Apache Arrow is a data processing library for analysis . This package provides documentations. -Package: libarrow-cuda-glib1500 +Package: libarrow-cuda-glib1600 Section: libs Architecture: @CUDA_ARCHITECTURE@ Multi-Arch: same @@ -344,8 +344,8 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow-glib1500 (= ${binary:Version}), - libarrow-cuda1500 (= ${binary:Version}) + libarrow-glib1600 (= ${binary:Version}), + libarrow-cuda1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides GLib based library files for CUDA support. @@ -370,13 +370,13 @@ Depends: ${misc:Depends}, libarrow-cuda-dev (= ${binary:Version}), libarrow-glib-dev (= ${binary:Version}), - libarrow-cuda-glib1500 (= ${binary:Version}), + libarrow-cuda-glib1600 (= ${binary:Version}), gir1.2-arrow-cuda-1.0 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides GLib based header files for CUDA support. -Package: libarrow-dataset-glib1500 +Package: libarrow-dataset-glib1600 Section: libs Architecture: any Multi-Arch: same @@ -384,8 +384,8 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow-glib1500 (= ${binary:Version}), - libarrow-dataset1500 (= ${binary:Version}) + libarrow-glib1600 (= ${binary:Version}), + libarrow-dataset1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides GLib based library files for dataset module. @@ -410,7 +410,7 @@ Depends: ${misc:Depends}, libarrow-dataset-dev (= ${binary:Version}), libarrow-glib-dev (= ${binary:Version}), - libarrow-dataset-glib1500 (= ${binary:Version}), + libarrow-dataset-glib1600 (= ${binary:Version}), gir1.2-arrow-dataset-1.0 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . @@ -427,7 +427,7 @@ Description: Apache Arrow is a data processing library for analysis . This package provides documentations for dataset module. -Package: libarrow-flight-glib1500 +Package: libarrow-flight-glib1600 Section: libs Architecture: any Multi-Arch: same @@ -435,8 +435,8 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow-glib1500 (= ${binary:Version}), - libarrow-flight1500 (= ${binary:Version}) + libarrow-glib1600 (= ${binary:Version}), + libarrow-flight1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides GLib based library files for Apache Arrow Flight. @@ -462,7 +462,7 @@ Depends: ${misc:Depends}, libarrow-flight-dev (= ${binary:Version}), libarrow-glib-dev (= ${binary:Version}), - libarrow-flight-glib1500 (= ${binary:Version}), + libarrow-flight-glib1600 (= ${binary:Version}), gir1.2-arrow-flight-1.0 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . @@ -479,7 +479,7 @@ Description: Apache Arrow is a data processing library for analysis . This package provides documentations for Apache Arrow Flight. -Package: libarrow-flight-sql-glib1500 +Package: libarrow-flight-sql-glib1600 Section: libs Architecture: any Multi-Arch: same @@ -487,8 +487,8 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow-flight-glib1500 (= ${binary:Version}), - libarrow-flight-sql1500 (= ${binary:Version}) + libarrow-flight-glib1600 (= ${binary:Version}), + libarrow-flight-sql1600 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . This package provides GLib based library files for Apache Arrow Flight SQL. @@ -514,7 +514,7 @@ Depends: ${misc:Depends}, libarrow-flight-sql-dev (= ${binary:Version}), libarrow-flight-glib-dev (= ${binary:Version}), - libarrow-flight-sql-glib1500 (= ${binary:Version}), + libarrow-flight-sql-glib1600 (= ${binary:Version}), gir1.2-arrow-flight-sql-1.0 (= ${binary:Version}) Description: Apache Arrow is a data processing library for analysis . @@ -531,7 +531,7 @@ Description: Apache Arrow is a data processing library for analysis . This package provides documentations for Apache Arrow Flight SQL. -Package: libgandiva-glib1500 +Package: libgandiva-glib1600 Section: libs Architecture: any Multi-Arch: same @@ -539,8 +539,8 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow-glib1500 (= ${binary:Version}), - libgandiva1500 (= ${binary:Version}) + libarrow-glib1600 (= ${binary:Version}), + libgandiva1600 (= ${binary:Version}) Description: Gandiva is a toolset for compiling and evaluating expressions on Arrow Data. . @@ -567,7 +567,7 @@ Depends: ${misc:Depends}, libgandiva-dev (= ${binary:Version}), libarrow-glib-dev (= ${binary:Version}), - libgandiva-glib1500 (= ${binary:Version}), + libgandiva-glib1600 (= ${binary:Version}), gir1.2-gandiva-1.0 (= ${binary:Version}) Description: Gandiva is a toolset for compiling and evaluating expressions on Arrow Data. @@ -586,7 +586,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions . This package provides documentations. -Package: libparquet-glib1500 +Package: libparquet-glib1600 Section: libs Architecture: any Multi-Arch: same @@ -594,8 +594,8 @@ Pre-Depends: ${misc:Pre-Depends} Depends: ${misc:Depends}, ${shlibs:Depends}, - libarrow-glib1500 (= ${binary:Version}), - libparquet1500 (= ${binary:Version}) + libarrow-glib1600 (= ${binary:Version}), + libparquet1600 (= ${binary:Version}) Description: Apache Parquet is a columnar storage format . This package provides GLib based library files. @@ -620,7 +620,7 @@ Depends: ${misc:Depends}, libarrow-glib-dev (= ${binary:Version}), libparquet-dev (= ${binary:Version}), - libparquet-glib1500 (= ${binary:Version}), + libparquet-glib1600 (= ${binary:Version}), gir1.2-parquet-1.0 (= ${binary:Version}) Suggests: libparquet-glib-doc Description: Apache Parquet is a columnar storage format diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-acero1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-acero1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-acero1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-acero1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libarrow1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libarrow1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libgandiva1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libgandiva1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libgandiva1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libgandiva1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib1600.install diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libparquet1500.install b/dev/tasks/linux-packages/apache-arrow/debian/libparquet1600.install similarity index 100% rename from dev/tasks/linux-packages/apache-arrow/debian/libparquet1500.install rename to dev/tasks/linux-packages/apache-arrow/debian/libparquet1600.install diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 64620d41993d5..6c59364d51a50 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -502,59 +502,59 @@ tasks: - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-acero-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-acero1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-acero1500_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-acero1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-acero1600_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset-glib1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset-glib1500_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-dataset1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-dataset1500_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-dataset-glib1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-dataset-glib1600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-dataset1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-dataset1600_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-flight-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-flight-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-glib1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-flight-glib1500_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight-glib1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-flight-glib1600_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-flight-sql-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-flight-sql-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-flight-sql-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-sql-glib1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-flight-sql-glib1500_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight-sql1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-flight-sql1500_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-flight1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-flight1500_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight-sql-glib1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-flight-sql-glib1600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight-sql1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-flight-sql1600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-flight1600_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-glib1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-glib1500_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow1500_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-glib1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-glib1600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow1600_{no_rc_version}-1_[a-z0-9]+.deb - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva-glib1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva-glib1500_{no_rc_version}-1_[a-z0-9]+.deb - - libgandiva1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libgandiva1500_{no_rc_version}-1_[a-z0-9]+.deb + - libgandiva-glib1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libgandiva-glib1600_{no_rc_version}-1_[a-z0-9]+.deb + - libgandiva1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libgandiva1600_{no_rc_version}-1_[a-z0-9]+.deb - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet-glib1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet-glib1500_{no_rc_version}-1_[a-z0-9]+.deb - - libparquet1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libparquet1500_{no_rc_version}-1_[a-z0-9]+.deb + - libparquet-glib1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libparquet-glib1600_{no_rc_version}-1_[a-z0-9]+.deb + - libparquet1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libparquet1600_{no_rc_version}-1_[a-z0-9]+.deb - parquet-tools_{no_rc_version}-1_[a-z0-9]+.deb {% if architecture == "amd64" %} - gir1.2-arrow-cuda-1.0_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda-glib1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-cuda-glib1500_{no_rc_version}-1_[a-z0-9]+.deb - - libarrow-cuda1500-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb - - libarrow-cuda1500_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-cuda-glib1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-cuda-glib1600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-cuda1600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-cuda1600_{no_rc_version}-1_[a-z0-9]+.deb {% endif %} {% endfor %} {% endfor %} From bb7f584bf7a0b67e90337e1abc5f0ae74d88b1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Sun, 21 Jan 2024 16:06:40 +0100 Subject: [PATCH 55/92] MINOR: [Release] Update .deb/.rpm changelogs for 15.0.0 --- .../linux-packages/apache-arrow-apt-source/debian/changelog | 6 ++++++ .../apache-arrow-release/yum/apache-arrow-release.spec.in | 3 +++ dev/tasks/linux-packages/apache-arrow/debian/changelog | 6 ++++++ dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 3 +++ 4 files changed, 18 insertions(+) diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog index 32a5a38afebf3..bc83f0ed7c4b0 100644 --- a/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog @@ -1,3 +1,9 @@ +apache-arrow-apt-source (15.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Raúl Cumplido Tue, 16 Jan 2024 14:38:51 -0000 + apache-arrow-apt-source (14.0.2-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in index 348f8064ecc5f..9b6c963593fc3 100644 --- a/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in +++ b/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in @@ -102,6 +102,9 @@ else fi %changelog +* Tue Jan 16 2024 Raúl Cumplido - 15.0.0-1 +- New upstream release. + * Tue Dec 12 2023 Raúl Cumplido - 14.0.2-1 - New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/debian/changelog b/dev/tasks/linux-packages/apache-arrow/debian/changelog index b14bb0985893e..edff045a48111 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/changelog +++ b/dev/tasks/linux-packages/apache-arrow/debian/changelog @@ -1,3 +1,9 @@ +apache-arrow (15.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Raúl Cumplido Tue, 16 Jan 2024 14:38:51 -0000 + apache-arrow (14.0.2-1) unstable; urgency=low * New upstream release. diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 44421ce0ea1e4..79b4eadd92265 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -864,6 +864,9 @@ Documentation for Apache Parquet GLib. %{_datadir}/gtk-doc/html/parquet-glib/ %changelog +* Tue Jan 16 2024 Raúl Cumplido - 15.0.0-1 +- New upstream release. + * Tue Dec 12 2023 Raúl Cumplido - 14.0.2-1 - New upstream release. From 26f515a667884f1b9f1da102adbe96e75b52e162 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sun, 21 Jan 2024 09:22:54 -0800 Subject: [PATCH 56/92] MINOR: [C#] Update copyright year on built assembly (#39726) ### Rationale for this change The copyright is currently shown as "2016-2019". A lot has changed since 2019, both in the world in general and in Arrow. Authored-by: Curt Hagenlocher Signed-off-by: Curt Hagenlocher --- csharp/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/Directory.Build.props b/csharp/Directory.Build.props index 7cc4b35f2a614..c759c49b395d8 100644 --- a/csharp/Directory.Build.props +++ b/csharp/Directory.Build.props @@ -27,7 +27,7 @@ Apache Arrow library - Copyright 2016-2019 The Apache Software Foundation + Copyright 2016-2024 The Apache Software Foundation The Apache Software Foundation 16.0.0-SNAPSHOT From c33ffb033a49c17619f2547d873569980ee91b9c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 22 Jan 2024 11:19:09 +0900 Subject: [PATCH 57/92] GH-39702: [GLib] Add support for time zone in GArrowTimestampDataType (#39717) ### Rationale for this change Timestamp data type in Apache Arrow supports time zone but Apache Arrow C GLib didn't support it. Timestamp data type has "timezone-aware" mode and "timezone-naive" mode. If a timestamp data type has a valid time zone information, it uses "timezone-aware" mode. If a timestamp data type doesn't have a valid time zone information, it uses "timezone-naive" mode. Apache Arrow C GLib should support both of them. ### What changes are included in this PR? This change adds a new `GTimeZone *time_zone` argument to `garrow_timestamp_data_type_new()` instead of adding a new `garrow_timestamp_data_type_new_time_zone()` function. This breaks backward compatibility for Apache Arrow C GLib users. But this still keeps backward compatibility for users of bindings such as Ruby and Vala. Because the new `GTimeZone *time_zone` is nullable. I tried to use the "adding a new `garrow_timestamp_data_type_new_time_zone()` function" approach but Vala didn't like it. Both of `garrow_timestamp_data_type_new_time_zone()` (constructor) and `garrow_timestamp_data_type_get_time_zone()` (instance method or property reader) were mapped to `GArrow.TimestampDataType.time_zone()`. So I chose the "adding a new argument to `garrow_timestamp_data_type_new()`" approach. ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. **This PR includes breaking changes to public APIs.** * Closes: #39702 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- c_glib/arrow-glib/basic-data-type.cpp | 109 ++++++++++++++++++++-- c_glib/arrow-glib/basic-data-type.h | 6 +- c_glib/arrow-glib/version.h.in | 23 +++++ c_glib/doc/arrow-glib/arrow-glib-docs.xml | 4 + c_glib/test/test-timestamp-data-type.rb | 17 ++++ 5 files changed, 147 insertions(+), 12 deletions(-) diff --git a/c_glib/arrow-glib/basic-data-type.cpp b/c_glib/arrow-glib/basic-data-type.cpp index 0697646e5806d..0de9466eee456 100644 --- a/c_glib/arrow-glib/basic-data-type.cpp +++ b/c_glib/arrow-glib/basic-data-type.cpp @@ -125,9 +125,9 @@ G_BEGIN_DECLS * data types. */ -typedef struct GArrowDataTypePrivate_ { +struct GArrowDataTypePrivate { std::shared_ptr data_type; -} GArrowDataTypePrivate; +}; enum { PROP_DATA_TYPE = 1 @@ -1113,9 +1113,71 @@ garrow_date64_data_type_new(void) } -G_DEFINE_TYPE(GArrowTimestampDataType, - garrow_timestamp_data_type, - GARROW_TYPE_TEMPORAL_DATA_TYPE) +struct GArrowTimestampDataTypePrivate { + GTimeZone *time_zone; +}; + +enum { + PROP_TIME_ZONE = 1 +}; + +G_DEFINE_TYPE_WITH_PRIVATE(GArrowTimestampDataType, + garrow_timestamp_data_type, + GARROW_TYPE_TEMPORAL_DATA_TYPE) + +#define GARROW_TIMESTAMP_DATA_TYPE_GET_PRIVATE(object) \ + static_cast( \ + garrow_timestamp_data_type_get_instance_private( \ + GARROW_TIMESTAMP_DATA_TYPE(object))) + +static void +garrow_timestamp_data_type_dispose(GObject *object) +{ + auto priv = GARROW_TIMESTAMP_DATA_TYPE_GET_PRIVATE(object); + + if (priv->time_zone) { + g_time_zone_unref(priv->time_zone); + priv->time_zone = nullptr; + } + + G_OBJECT_CLASS(garrow_timestamp_data_type_parent_class)->dispose(object); +} + +static void +garrow_timestamp_data_type_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_TIMESTAMP_DATA_TYPE_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_TIME_ZONE: + priv->time_zone = static_cast(g_value_dup_boxed(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_timestamp_data_type_get_property(GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_TIMESTAMP_DATA_TYPE_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_TIME_ZONE: + g_value_set_boxed(value, priv->time_zone); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} static void garrow_timestamp_data_type_init(GArrowTimestampDataType *object) @@ -1125,11 +1187,32 @@ garrow_timestamp_data_type_init(GArrowTimestampDataType *object) static void garrow_timestamp_data_type_class_init(GArrowTimestampDataTypeClass *klass) { + auto gobject_class = G_OBJECT_CLASS(klass); + gobject_class->dispose = garrow_timestamp_data_type_dispose; + gobject_class->set_property = garrow_timestamp_data_type_set_property; + gobject_class->get_property = garrow_timestamp_data_type_get_property; + + GParamSpec *spec; + /** + * GArrowTimestampDataType:time-zone: + * + * The time zone of this data type. + * + * Since: 16.0.0 + */ + spec = g_param_spec_boxed("time-zone", + "Time zone", + "The time zone of this data type", + G_TYPE_TIME_ZONE, + static_cast(G_PARAM_READWRITE | + G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_TIME_ZONE, spec); } /** * garrow_timestamp_data_type_new: * @unit: The unit of the timestamp data. + * @time_zone: (nullable): The time zone of the timestamp data. * * Returns: A newly created the number of * seconds/milliseconds/microseconds/nanoseconds since UNIX epoch in @@ -1138,30 +1221,36 @@ garrow_timestamp_data_type_class_init(GArrowTimestampDataTypeClass *klass) * Since: 0.7.0 */ GArrowTimestampDataType * -garrow_timestamp_data_type_new(GArrowTimeUnit unit) +garrow_timestamp_data_type_new(GArrowTimeUnit unit, + GTimeZone *time_zone) { auto arrow_unit = garrow_time_unit_to_raw(unit); - auto arrow_data_type = arrow::timestamp(arrow_unit); + std::string arrow_timezone; + if (time_zone) { + arrow_timezone = g_time_zone_get_identifier(time_zone); + } + auto arrow_data_type = arrow::timestamp(arrow_unit, arrow_timezone); auto data_type = GARROW_TIMESTAMP_DATA_TYPE(g_object_new(GARROW_TYPE_TIMESTAMP_DATA_TYPE, "data-type", &arrow_data_type, + "time-zone", time_zone, NULL)); return data_type; } /** * garrow_timestamp_data_type_get_unit: - * @timestamp_data_type: The #GArrowTimestampDataType. + * @data_type: The #GArrowTimestampDataType. * * Returns: The unit of the timestamp data type. * * Since: 0.8.0 */ GArrowTimeUnit -garrow_timestamp_data_type_get_unit(GArrowTimestampDataType *timestamp_data_type) +garrow_timestamp_data_type_get_unit(GArrowTimestampDataType *data_type) { const auto arrow_data_type = - garrow_data_type_get_raw(GARROW_DATA_TYPE(timestamp_data_type)); + garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); const auto arrow_timestamp_data_type = std::static_pointer_cast(arrow_data_type); return garrow_time_unit_from_raw(arrow_timestamp_data_type->unit()); diff --git a/c_glib/arrow-glib/basic-data-type.h b/c_glib/arrow-glib/basic-data-type.h index affbfcf13c283..f1c5af409c9da 100644 --- a/c_glib/arrow-glib/basic-data-type.h +++ b/c_glib/arrow-glib/basic-data-type.h @@ -425,9 +425,11 @@ struct _GArrowTimestampDataTypeClass GArrowTemporalDataTypeClass parent_class; }; -GArrowTimestampDataType *garrow_timestamp_data_type_new (GArrowTimeUnit unit); +GArrowTimestampDataType * +garrow_timestamp_data_type_new(GArrowTimeUnit unit, + GTimeZone *time_zone); GArrowTimeUnit -garrow_timestamp_data_type_get_unit (GArrowTimestampDataType *timestamp_data_type); +garrow_timestamp_data_type_get_unit(GArrowTimestampDataType *data_type); #define GARROW_TYPE_TIME_DATA_TYPE (garrow_time_data_type_get_type()) diff --git a/c_glib/arrow-glib/version.h.in b/c_glib/arrow-glib/version.h.in index abb8ba08708de..01760fbfed1ff 100644 --- a/c_glib/arrow-glib/version.h.in +++ b/c_glib/arrow-glib/version.h.in @@ -110,6 +110,15 @@ # define GARROW_UNAVAILABLE(major, minor) G_UNAVAILABLE(major, minor) #endif +/** + * GARROW_VERSION_16_0: + * + * You can use this macro value for compile time API version check. + * + * Since: 16.0.0 + */ +#define GARROW_VERSION_16_0 G_ENCODE_VERSION(16, 0) + /** * GARROW_VERSION_15_0: * @@ -355,6 +364,20 @@ #define GARROW_AVAILABLE_IN_ALL +#if GARROW_VERSION_MIN_REQUIRED >= GARROW_VERSION_16_0 +# define GARROW_DEPRECATED_IN_16_0 GARROW_DEPRECATED +# define GARROW_DEPRECATED_IN_16_0_FOR(function) GARROW_DEPRECATED_FOR(function) +#else +# define GARROW_DEPRECATED_IN_16_0 +# define GARROW_DEPRECATED_IN_16_0_FOR(function) +#endif + +#if GARROW_VERSION_MAX_ALLOWED < GARROW_VERSION_16_0 +# define GARROW_AVAILABLE_IN_16_0 GARROW_UNAVAILABLE(16, 0) +#else +# define GARROW_AVAILABLE_IN_16_0 +#endif + #if GARROW_VERSION_MIN_REQUIRED >= GARROW_VERSION_15_0 # define GARROW_DEPRECATED_IN_15_0 GARROW_DEPRECATED # define GARROW_DEPRECATED_IN_15_0_FOR(function) GARROW_DEPRECATED_FOR(function) diff --git a/c_glib/doc/arrow-glib/arrow-glib-docs.xml b/c_glib/doc/arrow-glib/arrow-glib-docs.xml index 57b4b98701686..e92eb955675ed 100644 --- a/c_glib/doc/arrow-glib/arrow-glib-docs.xml +++ b/c_glib/doc/arrow-glib/arrow-glib-docs.xml @@ -193,6 +193,10 @@ Index of deprecated API + + Index of new symbols in 16.0.0 + + Index of new symbols in 13.0.0 diff --git a/c_glib/test/test-timestamp-data-type.rb b/c_glib/test/test-timestamp-data-type.rb index dac3a9bc631d0..69437609feebf 100644 --- a/c_glib/test/test-timestamp-data-type.rb +++ b/c_glib/test/test-timestamp-data-type.rb @@ -26,6 +26,23 @@ def test_name assert_equal("timestamp", data_type.name) end + sub_test_case("time_zone") do + def test_nil + data_type = Arrow::TimestampDataType.new(:micro) + assert_nil(data_type.time_zone) + end + + def test_time_zone + data_type = Arrow::TimestampDataType.new(:micro, GLib::TimeZone.new("UTC")) + time_zone = data_type.time_zone + assert_not_nil(time_zone) + # glib2 gem 4.2.1 or later is required + if time_zone.respond_to?(:identifier) + assert_equal("UTC", time_zone.identifier) + end + end + end + sub_test_case("second") do def setup @data_type = Arrow::TimestampDataType.new(:second) From a7f81404c656707e3f7f2dc2cc8b36b0dec2dadf Mon Sep 17 00:00:00 2001 From: 0x0000ffff Date: Mon, 22 Jan 2024 22:27:05 +0800 Subject: [PATCH 58/92] GH-39579: [Python] fix raising ValueError on _ensure_partitioning (#39593) ### Rationale for this change The `_ensure_partitioning` method in dataset.py is missing a "raise" which currently ignores bad scheme silently. ### What changes are included in this PR? Fixed the typo. ### Are these changes tested? Tried with new code that the exception is properly raised. ### Are there any user-facing changes? No. * Closes: #39579 Lead-authored-by: idailylife Co-authored-by: 0x0000ffff Co-authored-by: Alenka Frim Signed-off-by: AlenkaF --- python/pyarrow/dataset.py | 4 ++-- python/pyarrow/tests/test_dataset.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/dataset.py b/python/pyarrow/dataset.py index 9301a5fee5ade..f83753ac57d47 100644 --- a/python/pyarrow/dataset.py +++ b/python/pyarrow/dataset.py @@ -292,8 +292,8 @@ def _ensure_partitioning(scheme): elif isinstance(scheme, (Partitioning, PartitioningFactory)): pass else: - ValueError("Expected Partitioning or PartitioningFactory, got {}" - .format(type(scheme))) + raise ValueError("Expected Partitioning or PartitioningFactory, got {}" + .format(type(scheme))) return scheme diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index d473299f20320..a4838d63a6b0b 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -701,6 +701,17 @@ def test_partitioning(): load_back_table = load_back.to_table() assert load_back_table.equals(table) + # test invalid partitioning input + with tempfile.TemporaryDirectory() as tempdir: + partitioning = ds.DirectoryPartitioning(partitioning_schema) + ds.write_dataset(table, tempdir, + format='ipc', partitioning=partitioning) + load_back = None + with pytest.raises(ValueError, + match="Expected Partitioning or PartitioningFactory"): + load_back = ds.dataset(tempdir, format='ipc', partitioning=int(0)) + assert load_back is None + def test_partitioning_pickling(pickle_module): schema = pa.schema([ From 3b73f438bf0a1abc76009f017c1df9c244854aca Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 22 Jan 2024 09:38:57 -0500 Subject: [PATCH 59/92] GH-39690: [C++][FlightRPC] Fix nullptr dereference in PollInfo (#39711) ### Rationale for this change The current implementation is a bit painful to use due to the lack of a move constructor. ### What changes are included in this PR? - Fix a crash in PollInfo with a nullptr FlightInfo. - Declare all necessary constructors (https://en.cppreference.com/w/cpp/language/rule_of_three) ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes, this adds new copy constructors. * Closes: #39673. * Closes: #39690 Authored-by: David Li Signed-off-by: David Li --- cpp/cmake_modules/FindClangTools.cmake | 3 ++- cpp/src/arrow/flight/flight_internals_test.cc | 2 ++ cpp/src/arrow/flight/serialization_internal.cc | 10 +++++++--- cpp/src/arrow/flight/types.cc | 7 ++++++- cpp/src/arrow/flight/types.h | 13 ++++++++++++- 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/FindClangTools.cmake b/cpp/cmake_modules/FindClangTools.cmake index 90df60bf541d4..1364ccbed8162 100644 --- a/cpp/cmake_modules/FindClangTools.cmake +++ b/cpp/cmake_modules/FindClangTools.cmake @@ -40,7 +40,8 @@ set(CLANG_TOOLS_SEARCH_PATHS /usr/local/bin /usr/bin "C:/Program Files/LLVM/bin" # Windows, non-conda - "$ENV{CONDA_PREFIX}/Library/bin") # Windows, conda + "$ENV{CONDA_PREFIX}/Library/bin" # Windows, conda + "$ENV{CONDA_PREFIX}/bin") # Unix, conda if(APPLE) find_program(BREW brew) if(BREW) diff --git a/cpp/src/arrow/flight/flight_internals_test.cc b/cpp/src/arrow/flight/flight_internals_test.cc index 522973bec7231..a1c5250ba66fa 100644 --- a/cpp/src/arrow/flight/flight_internals_test.cc +++ b/cpp/src/arrow/flight/flight_internals_test.cc @@ -282,6 +282,7 @@ TEST(FlightTypes, PollInfo) { std::nullopt}, PollInfo{std::make_unique(info), FlightDescriptor::Command("poll"), 0.1, expiration_time}, + PollInfo{}, }; std::vector reprs = { " " "progress=0.1 expiration_time=2023-06-19 03:14:06.004339000>", + "", }; ASSERT_NO_FATAL_FAILURE(TestRoundtrip(values, reprs)); diff --git a/cpp/src/arrow/flight/serialization_internal.cc b/cpp/src/arrow/flight/serialization_internal.cc index 64a40564afd72..e5a7503a6386b 100644 --- a/cpp/src/arrow/flight/serialization_internal.cc +++ b/cpp/src/arrow/flight/serialization_internal.cc @@ -306,8 +306,10 @@ Status ToProto(const FlightInfo& info, pb::FlightInfo* pb_info) { // PollInfo Status FromProto(const pb::PollInfo& pb_info, PollInfo* info) { - ARROW_ASSIGN_OR_RAISE(auto flight_info, FromProto(pb_info.info())); - info->info = std::make_unique(std::move(flight_info)); + if (pb_info.has_info()) { + ARROW_ASSIGN_OR_RAISE(auto flight_info, FromProto(pb_info.info())); + info->info = std::make_unique(std::move(flight_info)); + } if (pb_info.has_flight_descriptor()) { FlightDescriptor descriptor; RETURN_NOT_OK(FromProto(pb_info.flight_descriptor(), &descriptor)); @@ -331,7 +333,9 @@ Status FromProto(const pb::PollInfo& pb_info, PollInfo* info) { } Status ToProto(const PollInfo& info, pb::PollInfo* pb_info) { - RETURN_NOT_OK(ToProto(*info.info, pb_info->mutable_info())); + if (info.info) { + RETURN_NOT_OK(ToProto(*info.info, pb_info->mutable_info())); + } if (info.descriptor) { RETURN_NOT_OK(ToProto(*info.descriptor, pb_info->mutable_flight_descriptor())); } diff --git a/cpp/src/arrow/flight/types.cc b/cpp/src/arrow/flight/types.cc index 9da83fa8a11f2..1d43c41b69d9f 100644 --- a/cpp/src/arrow/flight/types.cc +++ b/cpp/src/arrow/flight/types.cc @@ -373,7 +373,12 @@ arrow::Result> PollInfo::Deserialize( std::string PollInfo::ToString() const { std::stringstream ss; - ss << "(*other.info) : NULLPTR), descriptor(other.descriptor), progress(other.progress), expiration_time(other.expiration_time) {} + PollInfo(PollInfo&& other) noexcept = default; // NOLINT(runtime/explicit) + ~PollInfo() = default; + PollInfo& operator=(const PollInfo& other) { + info = other.info ? std::make_unique(*other.info) : NULLPTR; + descriptor = other.descriptor; + progress = other.progress; + expiration_time = other.expiration_time; + return *this; + } + PollInfo& operator=(PollInfo&& other) = default; /// \brief Get the wire-format representation of this type. /// From 92c5afe69abeeeb43503488b14681a321170da1d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 22 Jan 2024 18:28:15 +0100 Subject: [PATCH 60/92] GH-39706: [Archery] Fix `benchmark diff` subcommand (#39733) ### What changes are included in this PR? This fixes a regression introduced in GH-39303. ### Are these changes tested? Only manually. ### Are there any user-facing changes? No. * Closes: #39706 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- dev/archery/archery/benchmark/runner.py | 1 + dev/archery/archery/cli.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dev/archery/archery/benchmark/runner.py b/dev/archery/archery/benchmark/runner.py index c12c74135e96e..a91989fb95257 100644 --- a/dev/archery/archery/benchmark/runner.py +++ b/dev/archery/archery/benchmark/runner.py @@ -210,6 +210,7 @@ def from_rev_or_path(src, root, rev_or_path, cmake_conf, **kwargs): """ build = None if StaticBenchmarkRunner.is_json_result(rev_or_path): + kwargs.pop('benchmark_extras', None) return StaticBenchmarkRunner.from_json(rev_or_path, **kwargs) elif CMakeBuild.is_build_dir(rev_or_path): build = CMakeBuild.from_path(rev_or_path) diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 052fe23bfc969..0ad3eee14d1f3 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -407,7 +407,7 @@ def benchmark_filter_options(cmd): @click.pass_context def benchmark_list(ctx, rev_or_path, src, preserve, output, cmake_extras, java_home, java_options, build_extras, benchmark_extras, - language, **kwargs): + cpp_benchmark_extras, language, **kwargs): """ List benchmark suite. """ with tmpdir(preserve=preserve) as root: @@ -418,7 +418,8 @@ def benchmark_list(ctx, rev_or_path, src, preserve, output, cmake_extras, cmake_extras=cmake_extras, **kwargs) runner_base = CppBenchmarkRunner.from_rev_or_path( - src, root, rev_or_path, conf) + src, root, rev_or_path, conf, + benchmark_extras=cpp_benchmark_extras) elif language == "java": for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: @@ -546,7 +547,8 @@ def benchmark_run(ctx, rev_or_path, src, preserve, output, cmake_extras, def benchmark_diff(ctx, src, preserve, output, language, cmake_extras, suite_filter, benchmark_filter, repetitions, no_counters, java_home, java_options, build_extras, benchmark_extras, - threshold, contender, baseline, **kwargs): + cpp_benchmark_extras, threshold, contender, baseline, + **kwargs): """Compare (diff) benchmark runs. This command acts like git-diff but for benchmark results. @@ -633,12 +635,14 @@ def benchmark_diff(ctx, src, preserve, output, language, cmake_extras, src, root, contender, conf, repetitions=repetitions, suite_filter=suite_filter, - benchmark_filter=benchmark_filter) + benchmark_filter=benchmark_filter, + benchmark_extras=cpp_benchmark_extras) runner_base = CppBenchmarkRunner.from_rev_or_path( src, root, baseline, conf, repetitions=repetitions, suite_filter=suite_filter, - benchmark_filter=benchmark_filter) + benchmark_filter=benchmark_filter, + benchmark_extras=cpp_benchmark_extras) elif language == "java": for key in {'cpp_package_prefix', 'cxx_flags', 'cxx', 'cc'}: From 315e06ae5b5733727696dd5a1298dcbb0e9ae6f0 Mon Sep 17 00:00:00 2001 From: Divyansh200102 <146909065+Divyansh200102@users.noreply.github.com> Date: Tue, 23 Jan 2024 00:18:57 +0530 Subject: [PATCH 61/92] GH-35369: [Docs] Add a missing space after ref:`IPC format ` (#38276) * Closes: #35369 Lead-authored-by: Divyansh200102 Co-authored-by: Divyansh200102 <146909065+Divyansh200102@users.noreply.github.com> Co-authored-by: Alenka Frim Signed-off-by: AlenkaF --- docs/source/format/Glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/format/Glossary.rst b/docs/source/format/Glossary.rst index 65d6e0afa4557..3f2f118a95d6d 100644 --- a/docs/source/format/Glossary.rst +++ b/docs/source/format/Glossary.rst @@ -174,7 +174,7 @@ Glossary .. seealso:: :term:`data type` record batch - **In the :ref:`IPC format `**: the primitive unit + **In the** :ref:`IPC format `: the primitive unit of data. A record batch consists of an ordered list of :term:`buffers ` corresponding to a :term:`schema`. From 4fc2a708bac92614a42c07ea1f23b1a4a3af88cb Mon Sep 17 00:00:00 2001 From: Felipe Oliveira Carvalho Date: Mon, 22 Jan 2024 17:23:54 -0300 Subject: [PATCH 62/92] GH-39718: [C++][FS][Azure] Remove StatusFromErrorResponse as it's not necessary (#39719) ### Rationale for this change Only the "*IfExists" functions from the Azure SDK ever set `response.Value.Deleted` to `false` to indicate that a resource wasn't found and the request succeeded without deleting anything. It's better that we use the `Delete()` versions of these functions instead of `DeleteIfExists` and check the `ErrorCode` ourselves to return an appropriate `Status` instead of something generic. ### What changes are included in this PR? - Removing `StatusFromErrorResponse` - Comments explaining the error handling decisions - Addition of a boolean parameter to `DeleteDirOnFileSystem` that controls how it fails when the directory being deleted doesn't exist ### Are these changes tested? - Yes, by the existing tests in `azurefs_test.cc`. * Closes: #39718 Authored-by: Felipe Oliveira Carvalho Signed-off-by: Felipe Oliveira Carvalho --- cpp/src/arrow/filesystem/azurefs.cc | 62 ++++++++++------------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/cpp/src/arrow/filesystem/azurefs.cc b/cpp/src/arrow/filesystem/azurefs.cc index 730adabd48bec..a5179c22190e1 100644 --- a/cpp/src/arrow/filesystem/azurefs.cc +++ b/cpp/src/arrow/filesystem/azurefs.cc @@ -305,31 +305,9 @@ Status ValidateFileLocation(const AzureLocation& location) { return internal::AssertNoTrailingSlash(location.path); } -std::string_view BodyTextView(const Http::RawResponse& raw_response) { - const auto& body = raw_response.GetBody(); -#ifndef NDEBUG - auto& headers = raw_response.GetHeaders(); - auto content_type = headers.find("Content-Type"); - if (content_type != headers.end()) { - DCHECK_EQ(std::string_view{content_type->second}.substr(5), "text/"); - } -#endif - return std::string_view{reinterpret_cast(body.data()), body.size()}; -} - -Status StatusFromErrorResponse(const std::string& url, - const Http::RawResponse& raw_response, - const std::string& context) { - // There isn't an Azure specification that response body on error - // doesn't contain any binary data but we assume it. We hope that - // error response body has useful information for the error. - auto body_text = BodyTextView(raw_response); - return Status::IOError(context, ": ", url, ": ", raw_response.GetReasonPhrase(), " (", - static_cast(raw_response.GetStatusCode()), - "): ", body_text); -} - bool IsContainerNotFound(const Storage::StorageException& e) { + // In some situations, only the ReasonPhrase is set and the + // ErrorCode is empty, so we check both. if (e.ErrorCode == "ContainerNotFound" || e.ReasonPhrase == "The specified container does not exist." || e.ReasonPhrase == "The specified filesystem does not exist.") { @@ -1515,13 +1493,9 @@ class AzureFileSystem::Impl { DCHECK(location.path.empty()); try { auto response = container_client.Delete(); - if (response.Value.Deleted) { - return Status::OK(); - } else { - return StatusFromErrorResponse( - container_client.GetUrl(), *response.RawResponse, - "Failed to delete a container: " + location.container); - } + // Only the "*IfExists" functions ever set Deleted to false. + // All the others either succeed or throw an exception. + DCHECK(response.Value.Deleted); } catch (const Storage::StorageException& exception) { if (IsContainerNotFound(exception)) { return PathNotFound(location); @@ -1530,6 +1504,7 @@ class AzureFileSystem::Impl { "Failed to delete a container: ", location.container, ": ", container_client.GetUrl()); } + return Status::OK(); } /// Deletes contents of a directory and possibly the directory itself @@ -1649,23 +1624,29 @@ class AzureFileSystem::Impl { /// \pre location.container is not empty. /// \pre location.path is not empty. Status DeleteDirOnFileSystem(const DataLake::DataLakeFileSystemClient& adlfs_client, - const AzureLocation& location) { + const AzureLocation& location, bool recursive, + bool require_dir_to_exist) { DCHECK(!location.container.empty()); DCHECK(!location.path.empty()); auto directory_client = adlfs_client.GetDirectoryClient(location.path); - // XXX: should "directory not found" be considered an error? try { - auto response = directory_client.DeleteRecursive(); - if (response.Value.Deleted) { + auto response = + recursive ? directory_client.DeleteRecursive() : directory_client.DeleteEmpty(); + // Only the "*IfExists" functions ever set Deleted to false. + // All the others either succeed or throw an exception. + DCHECK(response.Value.Deleted); + } catch (const Storage::StorageException& exception) { + if (exception.ErrorCode == "FilesystemNotFound" || + exception.ErrorCode == "PathNotFound") { + if (require_dir_to_exist) { + return PathNotFound(location); + } return Status::OK(); - } else { - return StatusFromErrorResponse(directory_client.GetUrl(), *response.RawResponse, - "Failed to delete a directory: " + location.path); } - } catch (const Storage::StorageException& exception) { return ExceptionToStatus(exception, "Failed to delete a directory: ", location.path, ": ", directory_client.GetUrl()); } + return Status::OK(); } /// \pre location.container is not empty. @@ -1855,7 +1836,8 @@ Status AzureFileSystem::DeleteDir(const std::string& path) { return PathNotFound(location); } if (hns_support == HNSSupport::kEnabled) { - return impl_->DeleteDirOnFileSystem(adlfs_client, location); + return impl_->DeleteDirOnFileSystem(adlfs_client, location, /*recursive=*/true, + /*require_dir_to_exist=*/true); } DCHECK_EQ(hns_support, HNSSupport::kDisabled); auto container_client = impl_->GetBlobContainerClient(location.container); From f8f6eb9d1694abee26af5f011ff81aeb05164758 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 06:34:49 +0900 Subject: [PATCH 63/92] MINOR: [CI] Bump actions/cache from 3 to 4 (#39741) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4.
Release notes

Sourced from actions/cache's releases.

v4.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/cache/compare/v3...v4.0.0

v3.3.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/cache/compare/v3...v3.3.3

v3.3.2

What's Changed

New Contributors

Full Changelog: https://github.com/actions/cache/compare/v3...v3.3.2

v3.3.1

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v3...v3.3.1

v3.3.0

What's Changed

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- .github/workflows/cpp.yml | 8 ++++---- .github/workflows/docs.yml | 2 +- .github/workflows/docs_light.yml | 2 +- .github/workflows/integration.yml | 2 +- .github/workflows/java.yml | 2 +- .github/workflows/java_jni.yml | 4 ++-- .github/workflows/js.yml | 4 ++-- .github/workflows/matlab.yml | 6 +++--- .github/workflows/python.yml | 2 +- .github/workflows/r.yml | 4 ++-- .github/workflows/r_nightly.yml | 2 +- .github/workflows/ruby.yml | 8 ++++---- 12 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 3d4fb10b10c39..bd14f1b895bf6 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -101,7 +101,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }} @@ -214,7 +214,7 @@ jobs: run: | echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.ccache-info.outputs.cache-dir }} key: cpp-ccache-macos-${{ hashFiles('cpp/**') }} @@ -310,7 +310,7 @@ jobs: run: | echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.ccache-info.outputs.cache-dir }} key: cpp-ccache-windows-${{ env.CACHE_VERSION }}-${{ hashFiles('cpp/**') }} @@ -402,7 +402,7 @@ jobs: shell: msys2 {0} run: ci/scripts/msys2_setup.sh cpp - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ccache key: cpp-ccache-${{ matrix.msystem_lower}}-${{ hashFiles('cpp/**') }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 098b5ff29df5a..e394347e95261 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,7 +45,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: ubuntu-docs-${{ hashFiles('cpp/**') }} diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml index 8d10060c9d8a0..5303531f34350 100644 --- a/.github/workflows/docs_light.yml +++ b/.github/workflows/docs_light.yml @@ -51,7 +51,7 @@ jobs: with: fetch-depth: 0 - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: conda-docs-${{ hashFiles('cpp/**') }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4960b4dbd61e8..adb6fb2b57c75 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -75,7 +75,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: conda-${{ hashFiles('cpp/**') }} diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index ee4c1b21c37d4..1f1fc1b47a3c8 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -69,7 +69,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: maven-${{ hashFiles('java/**') }} diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 9f05a357a11d3..45de57f360a42 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -63,7 +63,7 @@ jobs: run: | ci/scripts/util_free_space.sh - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }} @@ -103,7 +103,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: maven-${{ hashFiles('java/**') }} diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index e2c76a3d1cb24..0d09e30d6eab5 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -91,7 +91,7 @@ jobs: with: fetch-depth: 0 - name: Jest Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: js/.jest-cache key: js-jest-cache-${{ runner.os }}-${{ hashFiles('js/src/**/*.ts', 'js/test/**/*.ts', 'js/yarn.lock') }} @@ -121,7 +121,7 @@ jobs: with: fetch-depth: 0 - name: Jest Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: js/.jest-cache key: js-jest-cache-${{ runner.os }}-${{ hashFiles('js/src/**/*.ts', 'js/test/**/*.ts', 'js/yarn.lock') }} diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml index 6921e12213b5b..512ff2bb929b3 100644 --- a/.github/workflows/matlab.yml +++ b/.github/workflows/matlab.yml @@ -65,7 +65,7 @@ jobs: shell: bash run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.ccache-info.outputs.cache-dir }} key: matlab-ccache-ubuntu-${{ hashFiles('cpp/**', 'matlab/**') }} @@ -113,7 +113,7 @@ jobs: shell: bash run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.ccache-info.outputs.cache-dir }} key: matlab-ccache-macos-${{ hashFiles('cpp/**', 'matlab/**') }} @@ -155,7 +155,7 @@ jobs: shell: bash run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ${{ steps.ccache-info.outputs.cache-dir }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d9979da0ee12a..6e3797b29c21e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -94,7 +94,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }} diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 4fc308a28d4d6..2a801b6040ec8 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -73,7 +73,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker # As this key is identical on both matrix builds only one will be able to successfully cache, @@ -206,7 +206,7 @@ jobs: ci/scripts/ccache_setup.sh echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ccache key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} diff --git a/.github/workflows/r_nightly.yml b/.github/workflows/r_nightly.yml index 27a32d22f90c0..a57a8cddea3c0 100644 --- a/.github/workflows/r_nightly.yml +++ b/.github/workflows/r_nightly.yml @@ -86,7 +86,7 @@ jobs: exit 1 fi - name: Cache Repo - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: repo key: r-nightly-${{ github.run_id }} diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index be30865ac7ac6..74d56895f4c34 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -76,7 +76,7 @@ jobs: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: .docker key: ubuntu-${{ matrix.ubuntu }}-ruby-${{ hashFiles('cpp/**') }} @@ -167,7 +167,7 @@ jobs: run: | echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.ccache-info.outputs.cache-dir }} key: ruby-ccache-macos-${{ hashFiles('cpp/**') }} @@ -252,7 +252,7 @@ jobs: run: | ridk exec bash ci\scripts\msys2_setup.sh ruby - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ccache key: ruby-ccache-ucrt${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} @@ -277,7 +277,7 @@ jobs: Write-Output "gem-dir=$(ridk exec gem env gemdir)" | ` Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - name: Cache RubyGems - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.rubygems-info.outputs.gem-dir }} key: ruby-rubygems-ucrt${{ matrix.mingw-n-bits }}-${{ hashFiles('**/Gemfile', 'ruby/*/*.gemspec') }} From 6eb62892a4cd6ff37df13a4a8be8f14d468c962c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 06:35:12 +0900 Subject: [PATCH 64/92] MINOR: [Java] Bump joda-time:joda-time from 2.12.5 to 2.12.6 in /java (#39742) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [joda-time:joda-time](https://github.com/JodaOrg/joda-time) from 2.12.5 to 2.12.6.
Release notes

Sourced from joda-time:joda-time's releases.

Release v2.12.6

See the change notes for more information.

What's Changed

New Contributors

Full Changelog: https://github.com/JodaOrg/joda-time/compare/v2.12.5...v2.12.6

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=joda-time:joda-time&package-manager=maven&previous-version=2.12.5&new-version=2.12.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/flight/flight-sql-jdbc-driver/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml index 9ab9e1820cd27..28534a9b0badd 100644 --- a/java/flight/flight-sql-jdbc-driver/pom.xml +++ b/java/flight/flight-sql-jdbc-driver/pom.xml @@ -107,7 +107,7 @@ joda-time joda-time - 2.12.5 + 2.12.6 runtime From 067d177c803f3fe785b24cfc9167cd4843dafe59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 06:35:35 +0900 Subject: [PATCH 65/92] MINOR: [Java] Bump org.apache.maven.plugins:maven-plugin-plugin from 3.10.2 to 3.11.0 in /java (#39743) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.maven.plugins:maven-plugin-plugin](https://github.com/apache/maven-plugin-tools) from 3.10.2 to 3.11.0.
Release notes

Sourced from org.apache.maven.plugins:maven-plugin-plugin's releases.

3.11.0

Release Notes - Maven Plugin Tools - Version 3.11.0

Bug

  • [MPLUGIN-496] - Translation for keys report.plugin.goal.yes,no are missing
  • [MPLUGIN-499] - Deprecate descriptions are missing in description table

Improvement

  • [MPLUGIN-450] - Make goal prefix mandatory by default
  • [MPLUGIN-474] - Improve descriptor docs for requiredJavaVersion
  • [MPLUGIN-492] - Documentation for plugins in general: Goals comprises more than that
  • [MPLUGIN-495] - WARNINGs based on usage of @ Component for MavenSession/MavenProject instead of @ Parameter

Task

  • [MPLUGIN-493] - Consistently evaluate skip parameter in MavenReport#canGenerateReport()
  • [MPLUGIN-498] - Move section rendering to separate methods

Dependency upgrade

Commits
  • 4178d33 [maven-release-plugin] prepare release maven-plugin-tools-3.11.0
  • 25d920f [MNG-5695] document Maven 3.2.5+ scoped components usage
  • 6418490 [MPLUGIN-495] WARNINGs based on usage of @​Component for MavenSession/MavenPro...
  • 8b93d12 Bump org.jsoup:jsoup from 1.17.1 to 1.17.2
  • f4973ac Bump org.assertj:assertj-core from 3.24.2 to 3.25.1
  • 7dd3a25 [MPLUGIN-499] Add deprecate description in parameters table (#250)
  • 9bb13f0 [MPLUGIN-492] Documentation for plugins in general: Goals comprises more than...
  • fc41218 [MPLUGIN-498] Move section rendering to separate methods
  • ed4774b [MPLUGIN-450] Require goalPrefix to be valid (#240)
  • 331cf42 [MPLUGIN-497] Upgrade components
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.maven.plugins:maven-plugin-plugin&package-manager=maven&previous-version=3.10.2&new-version=3.11.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/maven/module-info-compiler-maven-plugin/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index e6bf8a63cb74a..5231c4c1f0430 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -80,7 +80,7 @@ maven-plugin-plugin - 3.10.2 + 3.11.0 maven-jar-plugin @@ -104,7 +104,7 @@ org.apache.maven.plugins maven-plugin-plugin - 3.10.2 + 3.11.0 true From c27cf66905b98f3cd260630e98cd9cc90a9c6741 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 06:35:55 +0900 Subject: [PATCH 66/92] MINOR: [Java] Bump org.apache.maven.plugins:maven-install-plugin from 2.5.1 to 3.1.1 in /java (#39744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.maven.plugins:maven-install-plugin](https://github.com/apache/maven-install-plugin) from 2.5.1 to 3.1.1.
Release notes

Sourced from org.apache.maven.plugins:maven-install-plugin's releases.

3.1.1

What's Changed

New Contributors

Full Changelog: https://github.com/apache/maven-install-plugin/compare/maven-install-plugin-3.1.0...maven-install-plugin-3.1.1

3.1.0

Bug

  • [MINSTALL-179] - installAtEnd when module does not use m-install-p

Improvement

  • [MINSTALL-183] - Don't use metadata from main artifact to fetch pom.xml
  • [MINSTALL-185] - Install all artifacts in one request

Task

Dependency upgrade

3.0.1

What's Changed

Full Changelog: https://github.com/apache/maven-install-plugin/compare/maven-install-plugin-3.0.0...maven-install-plugin-3.0.1

3.0.0

What's Changed

... (truncated)

Commits
  • 2a366ed [maven-release-plugin] prepare release maven-install-plugin-3.1.1
  • 0d0df35 [MINSTALL-189] Add parameter to lax project validation (#57)
  • ba4cdf2 [MINSTALL-188] Upgrade Parent to 39 - code reformat
  • 1f9c6df [MINSTALL-188] Upgrade Parent to 39
  • 508a3cd Disable merge button and add jira autolink
  • 4cbbc08 [MINSTALL-186] Use proper repositorySystemSession
  • bb53609 [maven-release-plugin] prepare for next development iteration
  • 4a985dc [maven-release-plugin] prepare release maven-install-plugin-3.1.0
  • b8635ab Remove broken link to Release Info example
  • ce0666e [MINSTALL-184] Cleanup IT tests
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.maven.plugins:maven-install-plugin&package-manager=maven&previous-version=2.5.1&new-version=3.1.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/maven/module-info-compiler-maven-plugin/pom.xml | 2 +- java/performance/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index 5231c4c1f0430..37d14ad412d88 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -88,7 +88,7 @@ maven-install-plugin - 2.5.2 + 3.1.1 maven-deploy-plugin diff --git a/java/performance/pom.xml b/java/performance/pom.xml index a302a216c53d3..a1d53171f549b 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -187,7 +187,7 @@ maven-install-plugin - 2.5.1 + 3.1.1 maven-jar-plugin From c9ca62ad781f1dd999570a51cae6d7c464af6e5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 06:36:22 +0900 Subject: [PATCH 67/92] MINOR: [Java] Bump org.mockito:mockito-inline from 4.11.0 to 5.2.0 in /java (#39746) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.mockito:mockito-inline](https://github.com/mockito/mockito) from 4.11.0 to 5.2.0.
Release notes

Sourced from org.mockito:mockito-inline's releases.

v5.2.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.2.0

v5.1.1

Changelog generated by Shipkit Changelog Gradle Plugin

5.1.1

v5.1.0

Changelog generated by Shipkit Changelog Gradle Plugin

5.1.0

... (truncated)

Commits
  • 74c811a Make InjectMocks aware of generic types (#2923)
  • fc136e4 Explicitly use UTF-8 in IOUtils (#2935)
  • 9473470 Bump io.github.gradle-nexus:publish-plugin from 1.2.0 to 1.3.0 (#2932)
  • dddeb72 Bump versions.bytebuddy from 1.14.0 to 1.14.1 (#2931)
  • 92ed602 Bump com.google.googlejavaformat:google-java-format from 1.15.0 to 1.16.0 (#2...
  • 19ef24a Bump com.diffplug.spotless from 6.15.0 to 6.16.0 (#2930)
  • 8b96cc1 Bump io.github.gradle-nexus:publish-plugin from 1.1.0 to 1.2.0 (#2924)
  • 4eb275c Make project relocatable by using relative paths in the OSGi test task (#2920)
  • d937334 Bump com.github.ben-manes.versions from 0.45.0 to 0.46.0 (#2922)
  • 60b0e71 Bump versions.bytebuddy from 1.13.0 to 1.14.0 (#2918)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.mockito:mockito-inline&package-manager=maven&previous-version=4.11.0&new-version=5.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/flight/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/flight/pom.xml b/java/flight/pom.xml index 3ce1b1149ef20..2f777ab42b756 100644 --- a/java/flight/pom.xml +++ b/java/flight/pom.xml @@ -60,7 +60,7 @@ 4.11.0 - 4.11.0 + 5.2.0 From 78ec4dc9e823f4283d52f62f9092330e3a9c717b Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 23 Jan 2024 10:36:40 +0100 Subject: [PATCH 68/92] GH-39747: [C++][Parquet] Make BYTE_STREAM_SPLIT routines type-agnostic (#39748) ### Rationale for this change The low-level BYTE_STREAM_SPLIT routines currently reference the logical type they are operating on (float or double). However, the BYTE_STREAM_SPLIT encoding is type-agnostic and only cares about the type width. Removing references to logical types makes these routines easier to reuse. ### Are these changes tested? Yes, including more exhaustive SIMD tests. ### Are there any user-facing changes? No. These routines are internal. * Closes: #39747 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- .../arrow/util/byte_stream_split_internal.h | 305 +++++++++--------- cpp/src/arrow/util/byte_stream_split_test.cc | 30 +- cpp/src/parquet/encoding.cc | 22 +- cpp/src/parquet/encoding_benchmark.cc | 35 +- 4 files changed, 199 insertions(+), 193 deletions(-) diff --git a/cpp/src/arrow/util/byte_stream_split_internal.h b/cpp/src/arrow/util/byte_stream_split_internal.h index 4bc732ec24313..f70b3991473fa 100644 --- a/cpp/src/arrow/util/byte_stream_split_internal.h +++ b/cpp/src/arrow/util/byte_stream_split_internal.h @@ -26,7 +26,6 @@ #include #ifdef ARROW_HAVE_SSE4_2 -// Enable the SIMD for ByteStreamSplit Encoder/Decoder #define ARROW_HAVE_SIMD_SPLIT #endif // ARROW_HAVE_SSE4_2 @@ -37,17 +36,15 @@ namespace arrow::util::internal { // #if defined(ARROW_HAVE_SSE4_2) -template +template void ByteStreamSplitDecodeSse2(const uint8_t* data, int64_t num_values, int64_t stride, - T* out) { - constexpr size_t kNumStreams = sizeof(T); - static_assert(kNumStreams == 4U || kNumStreams == 8U, "Invalid number of streams."); - constexpr size_t kNumStreamsLog2 = (kNumStreams == 8U ? 3U : 2U); + uint8_t* out) { + static_assert(kNumStreams == 4 || kNumStreams == 8, "Invalid number of streams."); + constexpr int kNumStreamsLog2 = (kNumStreams == 8 ? 3 : 2); constexpr int64_t kBlockSize = sizeof(__m128i) * kNumStreams; - const int64_t size = num_values * sizeof(T); + const int64_t size = num_values * kNumStreams; const int64_t num_blocks = size / kBlockSize; - uint8_t* output_data = reinterpret_cast(out); // First handle suffix. // This helps catch if the simd-based processing overflows into the suffix @@ -55,11 +52,11 @@ void ByteStreamSplitDecodeSse2(const uint8_t* data, int64_t num_values, int64_t const int64_t num_processed_elements = (num_blocks * kBlockSize) / kNumStreams; for (int64_t i = num_processed_elements; i < num_values; ++i) { uint8_t gathered_byte_data[kNumStreams]; - for (size_t b = 0; b < kNumStreams; ++b) { - const size_t byte_index = b * stride + i; + for (int b = 0; b < kNumStreams; ++b) { + const int64_t byte_index = b * stride + i; gathered_byte_data[b] = data[byte_index]; } - out[i] = arrow::util::SafeLoadAs(&gathered_byte_data[0]); + memcpy(out + i * kNumStreams, gathered_byte_data, kNumStreams); } // The blocks get processed hierarchically using the unpack intrinsics. @@ -67,53 +64,52 @@ void ByteStreamSplitDecodeSse2(const uint8_t* data, int64_t num_values, int64_t // Stage 1: AAAA BBBB CCCC DDDD // Stage 2: ACAC ACAC BDBD BDBD // Stage 3: ABCD ABCD ABCD ABCD - __m128i stage[kNumStreamsLog2 + 1U][kNumStreams]; - constexpr size_t kNumStreamsHalf = kNumStreams / 2U; + __m128i stage[kNumStreamsLog2 + 1][kNumStreams]; + constexpr int kNumStreamsHalf = kNumStreams / 2U; for (int64_t i = 0; i < num_blocks; ++i) { - for (size_t j = 0; j < kNumStreams; ++j) { + for (int j = 0; j < kNumStreams; ++j) { stage[0][j] = _mm_loadu_si128( reinterpret_cast(&data[i * sizeof(__m128i) + j * stride])); } - for (size_t step = 0; step < kNumStreamsLog2; ++step) { - for (size_t j = 0; j < kNumStreamsHalf; ++j) { + for (int step = 0; step < kNumStreamsLog2; ++step) { + for (int j = 0; j < kNumStreamsHalf; ++j) { stage[step + 1U][j * 2] = _mm_unpacklo_epi8(stage[step][j], stage[step][kNumStreamsHalf + j]); stage[step + 1U][j * 2 + 1U] = _mm_unpackhi_epi8(stage[step][j], stage[step][kNumStreamsHalf + j]); } } - for (size_t j = 0; j < kNumStreams; ++j) { - _mm_storeu_si128(reinterpret_cast<__m128i*>( - &output_data[(i * kNumStreams + j) * sizeof(__m128i)]), - stage[kNumStreamsLog2][j]); + for (int j = 0; j < kNumStreams; ++j) { + _mm_storeu_si128( + reinterpret_cast<__m128i*>(out + (i * kNumStreams + j) * sizeof(__m128i)), + stage[kNumStreamsLog2][j]); } } } -template -void ByteStreamSplitEncodeSse2(const uint8_t* raw_values, const size_t num_values, +template +void ByteStreamSplitEncodeSse2(const uint8_t* raw_values, const int64_t num_values, uint8_t* output_buffer_raw) { - constexpr size_t kNumStreams = sizeof(T); - static_assert(kNumStreams == 4U || kNumStreams == 8U, "Invalid number of streams."); - constexpr size_t kBlockSize = sizeof(__m128i) * kNumStreams; + static_assert(kNumStreams == 4 || kNumStreams == 8, "Invalid number of streams."); + constexpr int kBlockSize = sizeof(__m128i) * kNumStreams; __m128i stage[3][kNumStreams]; __m128i final_result[kNumStreams]; - const size_t size = num_values * sizeof(T); - const size_t num_blocks = size / kBlockSize; + const int64_t size = num_values * kNumStreams; + const int64_t num_blocks = size / kBlockSize; const __m128i* raw_values_sse = reinterpret_cast(raw_values); __m128i* output_buffer_streams[kNumStreams]; - for (size_t i = 0; i < kNumStreams; ++i) { + for (int i = 0; i < kNumStreams; ++i) { output_buffer_streams[i] = reinterpret_cast<__m128i*>(&output_buffer_raw[num_values * i]); } // First handle suffix. - const size_t num_processed_elements = (num_blocks * kBlockSize) / sizeof(T); - for (size_t i = num_processed_elements; i < num_values; ++i) { - for (size_t j = 0U; j < kNumStreams; ++j) { + const int64_t num_processed_elements = (num_blocks * kBlockSize) / kNumStreams; + for (int64_t i = num_processed_elements; i < num_values; ++i) { + for (int j = 0; j < kNumStreams; ++j) { const uint8_t byte_in_value = raw_values[i * kNumStreams + j]; output_buffer_raw[j * num_values + i] = byte_in_value; } @@ -131,48 +127,47 @@ void ByteStreamSplitEncodeSse2(const uint8_t* raw_values, const size_t num_value // 0: AAAA AAAA BBBB BBBB 1: CCCC CCCC DDDD DDDD ... // Step 4: __mm_unpacklo_epi64 and _mm_unpackhi_epi64: // 0: AAAA AAAA AAAA AAAA 1: BBBB BBBB BBBB BBBB ... - for (size_t block_index = 0; block_index < num_blocks; ++block_index) { + for (int64_t block_index = 0; block_index < num_blocks; ++block_index) { // First copy the data to stage 0. - for (size_t i = 0; i < kNumStreams; ++i) { + for (int i = 0; i < kNumStreams; ++i) { stage[0][i] = _mm_loadu_si128(&raw_values_sse[block_index * kNumStreams + i]); } // The shuffling of bytes is performed through the unpack intrinsics. // In my measurements this gives better performance then an implementation // which uses the shuffle intrinsics. - for (size_t stage_lvl = 0; stage_lvl < 2U; ++stage_lvl) { - for (size_t i = 0; i < kNumStreams / 2U; ++i) { + for (int stage_lvl = 0; stage_lvl < 2; ++stage_lvl) { + for (int i = 0; i < kNumStreams / 2; ++i) { stage[stage_lvl + 1][i * 2] = _mm_unpacklo_epi8(stage[stage_lvl][i * 2], stage[stage_lvl][i * 2 + 1]); stage[stage_lvl + 1][i * 2 + 1] = _mm_unpackhi_epi8(stage[stage_lvl][i * 2], stage[stage_lvl][i * 2 + 1]); } } - if constexpr (kNumStreams == 8U) { + if constexpr (kNumStreams == 8) { // This is the path for double. __m128i tmp[8]; - for (size_t i = 0; i < 4; ++i) { + for (int i = 0; i < 4; ++i) { tmp[i * 2] = _mm_unpacklo_epi32(stage[2][i], stage[2][i + 4]); tmp[i * 2 + 1] = _mm_unpackhi_epi32(stage[2][i], stage[2][i + 4]); } - - for (size_t i = 0; i < 4; ++i) { + for (int i = 0; i < 4; ++i) { final_result[i * 2] = _mm_unpacklo_epi32(tmp[i], tmp[i + 4]); final_result[i * 2 + 1] = _mm_unpackhi_epi32(tmp[i], tmp[i + 4]); } } else { // this is the path for float. __m128i tmp[4]; - for (size_t i = 0; i < 2; ++i) { + for (int i = 0; i < 2; ++i) { tmp[i * 2] = _mm_unpacklo_epi8(stage[2][i * 2], stage[2][i * 2 + 1]); tmp[i * 2 + 1] = _mm_unpackhi_epi8(stage[2][i * 2], stage[2][i * 2 + 1]); } - for (size_t i = 0; i < 2; ++i) { + for (int i = 0; i < 2; ++i) { final_result[i * 2] = _mm_unpacklo_epi64(tmp[i], tmp[i + 2]); final_result[i * 2 + 1] = _mm_unpackhi_epi64(tmp[i], tmp[i + 2]); } } - for (size_t i = 0; i < kNumStreams; ++i) { + for (int i = 0; i < kNumStreams; ++i) { _mm_storeu_si128(&output_buffer_streams[i][block_index], final_result[i]); } } @@ -180,52 +175,50 @@ void ByteStreamSplitEncodeSse2(const uint8_t* raw_values, const size_t num_value #endif // ARROW_HAVE_SSE4_2 #if defined(ARROW_HAVE_AVX2) -template +template void ByteStreamSplitDecodeAvx2(const uint8_t* data, int64_t num_values, int64_t stride, - T* out) { - constexpr size_t kNumStreams = sizeof(T); - static_assert(kNumStreams == 4U || kNumStreams == 8U, "Invalid number of streams."); - constexpr size_t kNumStreamsLog2 = (kNumStreams == 8U ? 3U : 2U); + uint8_t* out) { + static_assert(kNumStreams == 4 || kNumStreams == 8, "Invalid number of streams."); + constexpr int kNumStreamsLog2 = (kNumStreams == 8 ? 3 : 2); constexpr int64_t kBlockSize = sizeof(__m256i) * kNumStreams; - const int64_t size = num_values * sizeof(T); + const int64_t size = num_values * kNumStreams; if (size < kBlockSize) // Back to SSE for small size - return ByteStreamSplitDecodeSse2(data, num_values, stride, out); + return ByteStreamSplitDecodeSse2(data, num_values, stride, out); const int64_t num_blocks = size / kBlockSize; - uint8_t* output_data = reinterpret_cast(out); // First handle suffix. const int64_t num_processed_elements = (num_blocks * kBlockSize) / kNumStreams; for (int64_t i = num_processed_elements; i < num_values; ++i) { uint8_t gathered_byte_data[kNumStreams]; - for (size_t b = 0; b < kNumStreams; ++b) { - const size_t byte_index = b * stride + i; + for (int b = 0; b < kNumStreams; ++b) { + const int64_t byte_index = b * stride + i; gathered_byte_data[b] = data[byte_index]; } - out[i] = arrow::util::SafeLoadAs(&gathered_byte_data[0]); + memcpy(out + i * kNumStreams, gathered_byte_data, kNumStreams); } // Processed hierarchically using unpack intrinsics, then permute intrinsics. - __m256i stage[kNumStreamsLog2 + 1U][kNumStreams]; + __m256i stage[kNumStreamsLog2 + 1][kNumStreams]; __m256i final_result[kNumStreams]; - constexpr size_t kNumStreamsHalf = kNumStreams / 2U; + constexpr int kNumStreamsHalf = kNumStreams / 2; for (int64_t i = 0; i < num_blocks; ++i) { - for (size_t j = 0; j < kNumStreams; ++j) { + for (int j = 0; j < kNumStreams; ++j) { stage[0][j] = _mm256_loadu_si256( reinterpret_cast(&data[i * sizeof(__m256i) + j * stride])); } - for (size_t step = 0; step < kNumStreamsLog2; ++step) { - for (size_t j = 0; j < kNumStreamsHalf; ++j) { - stage[step + 1U][j * 2] = + for (int step = 0; step < kNumStreamsLog2; ++step) { + for (int j = 0; j < kNumStreamsHalf; ++j) { + stage[step + 1][j * 2] = _mm256_unpacklo_epi8(stage[step][j], stage[step][kNumStreamsHalf + j]); - stage[step + 1U][j * 2 + 1U] = + stage[step + 1][j * 2 + 1] = _mm256_unpackhi_epi8(stage[step][j], stage[step][kNumStreamsHalf + j]); } } - if constexpr (kNumStreams == 8U) { + if constexpr (kNumStreams == 8) { // path for double, 128i index: // {0x00, 0x08}, {0x01, 0x09}, {0x02, 0x0A}, {0x03, 0x0B}, // {0x04, 0x0C}, {0x05, 0x0D}, {0x06, 0x0E}, {0x07, 0x0F}, @@ -258,40 +251,41 @@ void ByteStreamSplitDecodeAvx2(const uint8_t* data, int64_t num_values, int64_t stage[kNumStreamsLog2][3], 0b00110001); } - for (size_t j = 0; j < kNumStreams; ++j) { - _mm256_storeu_si256(reinterpret_cast<__m256i*>( - &output_data[(i * kNumStreams + j) * sizeof(__m256i)]), - final_result[j]); + for (int j = 0; j < kNumStreams; ++j) { + _mm256_storeu_si256( + reinterpret_cast<__m256i*>(out + (i * kNumStreams + j) * sizeof(__m256i)), + final_result[j]); } } } -template -void ByteStreamSplitEncodeAvx2(const uint8_t* raw_values, const size_t num_values, +template +void ByteStreamSplitEncodeAvx2(const uint8_t* raw_values, const int64_t num_values, uint8_t* output_buffer_raw) { - constexpr size_t kNumStreams = sizeof(T); - static_assert(kNumStreams == 4U || kNumStreams == 8U, "Invalid number of streams."); - constexpr size_t kBlockSize = sizeof(__m256i) * kNumStreams; + static_assert(kNumStreams == 4 || kNumStreams == 8, "Invalid number of streams."); + constexpr int kBlockSize = sizeof(__m256i) * kNumStreams; - if constexpr (kNumStreams == 8U) // Back to SSE, currently no path for double. - return ByteStreamSplitEncodeSse2(raw_values, num_values, output_buffer_raw); + if constexpr (kNumStreams == 8) // Back to SSE, currently no path for double. + return ByteStreamSplitEncodeSse2(raw_values, num_values, + output_buffer_raw); - const size_t size = num_values * sizeof(T); + const int64_t size = num_values * kNumStreams; if (size < kBlockSize) // Back to SSE for small size - return ByteStreamSplitEncodeSse2(raw_values, num_values, output_buffer_raw); - const size_t num_blocks = size / kBlockSize; + return ByteStreamSplitEncodeSse2(raw_values, num_values, + output_buffer_raw); + const int64_t num_blocks = size / kBlockSize; const __m256i* raw_values_simd = reinterpret_cast(raw_values); __m256i* output_buffer_streams[kNumStreams]; - for (size_t i = 0; i < kNumStreams; ++i) { + for (int i = 0; i < kNumStreams; ++i) { output_buffer_streams[i] = reinterpret_cast<__m256i*>(&output_buffer_raw[num_values * i]); } // First handle suffix. - const size_t num_processed_elements = (num_blocks * kBlockSize) / sizeof(T); - for (size_t i = num_processed_elements; i < num_values; ++i) { - for (size_t j = 0U; j < kNumStreams; ++j) { + const int64_t num_processed_elements = (num_blocks * kBlockSize) / kNumStreams; + for (int64_t i = num_processed_elements; i < num_values; ++i) { + for (int j = 0; j < kNumStreams; ++j) { const uint8_t byte_in_value = raw_values[i * kNumStreams + j]; output_buffer_raw[j * num_values + i] = byte_in_value; } @@ -301,20 +295,20 @@ void ByteStreamSplitEncodeAvx2(const uint8_t* raw_values, const size_t num_value // 1. Processed hierarchically to 32i block using the unpack intrinsics. // 2. Pack 128i block using _mm256_permutevar8x32_epi32. // 3. Pack final 256i block with _mm256_permute2x128_si256. - constexpr size_t kNumUnpack = 3U; + constexpr int kNumUnpack = 3; __m256i stage[kNumUnpack + 1][kNumStreams]; static const __m256i kPermuteMask = _mm256_set_epi32(0x07, 0x03, 0x06, 0x02, 0x05, 0x01, 0x04, 0x00); __m256i permute[kNumStreams]; __m256i final_result[kNumStreams]; - for (size_t block_index = 0; block_index < num_blocks; ++block_index) { - for (size_t i = 0; i < kNumStreams; ++i) { + for (int64_t block_index = 0; block_index < num_blocks; ++block_index) { + for (int i = 0; i < kNumStreams; ++i) { stage[0][i] = _mm256_loadu_si256(&raw_values_simd[block_index * kNumStreams + i]); } - for (size_t stage_lvl = 0; stage_lvl < kNumUnpack; ++stage_lvl) { - for (size_t i = 0; i < kNumStreams / 2U; ++i) { + for (int stage_lvl = 0; stage_lvl < kNumUnpack; ++stage_lvl) { + for (int i = 0; i < kNumStreams / 2; ++i) { stage[stage_lvl + 1][i * 2] = _mm256_unpacklo_epi8(stage[stage_lvl][i * 2], stage[stage_lvl][i * 2 + 1]); stage[stage_lvl + 1][i * 2 + 1] = @@ -322,7 +316,7 @@ void ByteStreamSplitEncodeAvx2(const uint8_t* raw_values, const size_t num_value } } - for (size_t i = 0; i < kNumStreams; ++i) { + for (int i = 0; i < kNumStreams; ++i) { permute[i] = _mm256_permutevar8x32_epi32(stage[kNumUnpack][i], kPermuteMask); } @@ -331,7 +325,7 @@ void ByteStreamSplitEncodeAvx2(const uint8_t* raw_values, const size_t num_value final_result[2] = _mm256_permute2x128_si256(permute[1], permute[3], 0b00100000); final_result[3] = _mm256_permute2x128_si256(permute[1], permute[3], 0b00110001); - for (size_t i = 0; i < kNumStreams; ++i) { + for (int i = 0; i < kNumStreams; ++i) { _mm256_storeu_si256(&output_buffer_streams[i][block_index], final_result[i]); } } @@ -339,53 +333,51 @@ void ByteStreamSplitEncodeAvx2(const uint8_t* raw_values, const size_t num_value #endif // ARROW_HAVE_AVX2 #if defined(ARROW_HAVE_AVX512) -template +template void ByteStreamSplitDecodeAvx512(const uint8_t* data, int64_t num_values, int64_t stride, - T* out) { - constexpr size_t kNumStreams = sizeof(T); - static_assert(kNumStreams == 4U || kNumStreams == 8U, "Invalid number of streams."); - constexpr size_t kNumStreamsLog2 = (kNumStreams == 8U ? 3U : 2U); + uint8_t* out) { + static_assert(kNumStreams == 4 || kNumStreams == 8, "Invalid number of streams."); + constexpr int kNumStreamsLog2 = (kNumStreams == 8 ? 3 : 2); constexpr int64_t kBlockSize = sizeof(__m512i) * kNumStreams; - const int64_t size = num_values * sizeof(T); + const int64_t size = num_values * kNumStreams; if (size < kBlockSize) // Back to AVX2 for small size - return ByteStreamSplitDecodeAvx2(data, num_values, stride, out); + return ByteStreamSplitDecodeAvx2(data, num_values, stride, out); const int64_t num_blocks = size / kBlockSize; - uint8_t* output_data = reinterpret_cast(out); // First handle suffix. const int64_t num_processed_elements = (num_blocks * kBlockSize) / kNumStreams; for (int64_t i = num_processed_elements; i < num_values; ++i) { uint8_t gathered_byte_data[kNumStreams]; - for (size_t b = 0; b < kNumStreams; ++b) { - const size_t byte_index = b * stride + i; + for (int b = 0; b < kNumStreams; ++b) { + const int64_t byte_index = b * stride + i; gathered_byte_data[b] = data[byte_index]; } - out[i] = arrow::util::SafeLoadAs(&gathered_byte_data[0]); + memcpy(out + i * kNumStreams, gathered_byte_data, kNumStreams); } // Processed hierarchically using the unpack, then two shuffles. - __m512i stage[kNumStreamsLog2 + 1U][kNumStreams]; + __m512i stage[kNumStreamsLog2 + 1][kNumStreams]; __m512i shuffle[kNumStreams]; __m512i final_result[kNumStreams]; - constexpr size_t kNumStreamsHalf = kNumStreams / 2U; + constexpr int kNumStreamsHalf = kNumStreams / 2U; for (int64_t i = 0; i < num_blocks; ++i) { - for (size_t j = 0; j < kNumStreams; ++j) { + for (int j = 0; j < kNumStreams; ++j) { stage[0][j] = _mm512_loadu_si512( reinterpret_cast(&data[i * sizeof(__m512i) + j * stride])); } - for (size_t step = 0; step < kNumStreamsLog2; ++step) { - for (size_t j = 0; j < kNumStreamsHalf; ++j) { - stage[step + 1U][j * 2] = + for (int step = 0; step < kNumStreamsLog2; ++step) { + for (int j = 0; j < kNumStreamsHalf; ++j) { + stage[step + 1][j * 2] = _mm512_unpacklo_epi8(stage[step][j], stage[step][kNumStreamsHalf + j]); - stage[step + 1U][j * 2 + 1U] = + stage[step + 1][j * 2 + 1] = _mm512_unpackhi_epi8(stage[step][j], stage[step][kNumStreamsHalf + j]); } } - if constexpr (kNumStreams == 8U) { + if constexpr (kNumStreams == 8) { // path for double, 128i index: // {0x00, 0x04, 0x08, 0x0C}, {0x10, 0x14, 0x18, 0x1C}, // {0x01, 0x05, 0x09, 0x0D}, {0x11, 0x15, 0x19, 0x1D}, @@ -435,49 +427,49 @@ void ByteStreamSplitDecodeAvx512(const uint8_t* data, int64_t num_values, int64_ final_result[3] = _mm512_shuffle_i32x4(shuffle[2], shuffle[3], 0b11011101); } - for (size_t j = 0; j < kNumStreams; ++j) { - _mm512_storeu_si512(reinterpret_cast<__m512i*>( - &output_data[(i * kNumStreams + j) * sizeof(__m512i)]), - final_result[j]); + for (int j = 0; j < kNumStreams; ++j) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>(out + (i * kNumStreams + j) * sizeof(__m512i)), + final_result[j]); } } } -template -void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const size_t num_values, +template +void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const int64_t num_values, uint8_t* output_buffer_raw) { - constexpr size_t kNumStreams = sizeof(T); - static_assert(kNumStreams == 4U || kNumStreams == 8U, "Invalid number of streams."); - constexpr size_t kBlockSize = sizeof(__m512i) * kNumStreams; + static_assert(kNumStreams == 4 || kNumStreams == 8, "Invalid number of streams."); + constexpr int kBlockSize = sizeof(__m512i) * kNumStreams; - const size_t size = num_values * sizeof(T); + const int64_t size = num_values * kNumStreams; if (size < kBlockSize) // Back to AVX2 for small size - return ByteStreamSplitEncodeAvx2(raw_values, num_values, output_buffer_raw); + return ByteStreamSplitEncodeAvx2(raw_values, num_values, + output_buffer_raw); - const size_t num_blocks = size / kBlockSize; + const int64_t num_blocks = size / kBlockSize; const __m512i* raw_values_simd = reinterpret_cast(raw_values); __m512i* output_buffer_streams[kNumStreams]; - for (size_t i = 0; i < kNumStreams; ++i) { + for (int i = 0; i < kNumStreams; ++i) { output_buffer_streams[i] = reinterpret_cast<__m512i*>(&output_buffer_raw[num_values * i]); } // First handle suffix. - const size_t num_processed_elements = (num_blocks * kBlockSize) / sizeof(T); - for (size_t i = num_processed_elements; i < num_values; ++i) { - for (size_t j = 0U; j < kNumStreams; ++j) { + const int64_t num_processed_elements = (num_blocks * kBlockSize) / kNumStreams; + for (int64_t i = num_processed_elements; i < num_values; ++i) { + for (int j = 0; j < kNumStreams; ++j) { const uint8_t byte_in_value = raw_values[i * kNumStreams + j]; output_buffer_raw[j * num_values + i] = byte_in_value; } } - constexpr size_t KNumUnpack = (kNumStreams == 8U) ? 2U : 3U; + constexpr int KNumUnpack = (kNumStreams == 8) ? 2 : 3; __m512i final_result[kNumStreams]; __m512i unpack[KNumUnpack + 1][kNumStreams]; __m512i permutex[kNumStreams]; __m512i permutex_mask; - if constexpr (kNumStreams == 8U) { + if constexpr (kNumStreams == 8) { // use _mm512_set_epi32, no _mm512_set_epi16 for some old gcc version. permutex_mask = _mm512_set_epi32(0x001F0017, 0x000F0007, 0x001E0016, 0x000E0006, 0x001D0015, 0x000D0005, 0x001C0014, 0x000C0004, @@ -488,13 +480,13 @@ void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const size_t num_val 0x09, 0x05, 0x01, 0x0C, 0x08, 0x04, 0x00); } - for (size_t block_index = 0; block_index < num_blocks; ++block_index) { - for (size_t i = 0; i < kNumStreams; ++i) { + for (int64_t block_index = 0; block_index < num_blocks; ++block_index) { + for (int i = 0; i < kNumStreams; ++i) { unpack[0][i] = _mm512_loadu_si512(&raw_values_simd[block_index * kNumStreams + i]); } - for (size_t unpack_lvl = 0; unpack_lvl < KNumUnpack; ++unpack_lvl) { - for (size_t i = 0; i < kNumStreams / 2U; ++i) { + for (int unpack_lvl = 0; unpack_lvl < KNumUnpack; ++unpack_lvl) { + for (int i = 0; i < kNumStreams / 2; ++i) { unpack[unpack_lvl + 1][i * 2] = _mm512_unpacklo_epi8( unpack[unpack_lvl][i * 2], unpack[unpack_lvl][i * 2 + 1]); unpack[unpack_lvl + 1][i * 2 + 1] = _mm512_unpackhi_epi8( @@ -502,7 +494,7 @@ void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const size_t num_val } } - if constexpr (kNumStreams == 8U) { + if constexpr (kNumStreams == 8) { // path for double // 1. unpack to epi16 block // 2. permutexvar_epi16 to 128i block @@ -511,7 +503,7 @@ void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const size_t num_val // {0x01, 0x05, 0x09, 0x0D}, {0x11, 0x15, 0x19, 0x1D}, // {0x02, 0x06, 0x0A, 0x0E}, {0x12, 0x16, 0x1A, 0x1E}, // {0x03, 0x07, 0x0B, 0x0F}, {0x13, 0x17, 0x1B, 0x1F}, - for (size_t i = 0; i < kNumStreams; ++i) + for (int i = 0; i < kNumStreams; ++i) permutex[i] = _mm512_permutexvar_epi16(permutex_mask, unpack[KNumUnpack][i]); __m512i shuffle[kNumStreams]; @@ -537,7 +529,7 @@ void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const size_t num_val // 1. Processed hierarchically to 32i block using the unpack intrinsics. // 2. Pack 128i block using _mm256_permutevar8x32_epi32. // 3. Pack final 256i block with _mm256_permute2x128_si256. - for (size_t i = 0; i < kNumStreams; ++i) + for (int i = 0; i < kNumStreams; ++i) permutex[i] = _mm512_permutexvar_epi32(permutex_mask, unpack[KNumUnpack][i]); final_result[0] = _mm512_shuffle_i32x4(permutex[0], permutex[2], 0b01000100); @@ -546,7 +538,7 @@ void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const size_t num_val final_result[3] = _mm512_shuffle_i32x4(permutex[1], permutex[3], 0b11101110); } - for (size_t i = 0; i < kNumStreams; ++i) { + for (int i = 0; i < kNumStreams; ++i) { _mm512_storeu_si512(&output_buffer_streams[i][block_index], final_result[i]); } } @@ -554,32 +546,32 @@ void ByteStreamSplitEncodeAvx512(const uint8_t* raw_values, const size_t num_val #endif // ARROW_HAVE_AVX512 #if defined(ARROW_HAVE_SIMD_SPLIT) -template +template void inline ByteStreamSplitDecodeSimd(const uint8_t* data, int64_t num_values, - int64_t stride, T* out) { + int64_t stride, uint8_t* out) { #if defined(ARROW_HAVE_AVX512) - return ByteStreamSplitDecodeAvx512(data, num_values, stride, out); + return ByteStreamSplitDecodeAvx512(data, num_values, stride, out); #elif defined(ARROW_HAVE_AVX2) - return ByteStreamSplitDecodeAvx2(data, num_values, stride, out); + return ByteStreamSplitDecodeAvx2(data, num_values, stride, out); #elif defined(ARROW_HAVE_SSE4_2) - return ByteStreamSplitDecodeSse2(data, num_values, stride, out); + return ByteStreamSplitDecodeSse2(data, num_values, stride, out); #else #error "ByteStreamSplitDecodeSimd not implemented" #endif } -template +template void inline ByteStreamSplitEncodeSimd(const uint8_t* raw_values, const int64_t num_values, uint8_t* output_buffer_raw) { #if defined(ARROW_HAVE_AVX512) - return ByteStreamSplitEncodeAvx512(raw_values, static_cast(num_values), - output_buffer_raw); + return ByteStreamSplitEncodeAvx512(raw_values, num_values, + output_buffer_raw); #elif defined(ARROW_HAVE_AVX2) - return ByteStreamSplitEncodeAvx2(raw_values, static_cast(num_values), - output_buffer_raw); + return ByteStreamSplitEncodeAvx2(raw_values, num_values, + output_buffer_raw); #elif defined(ARROW_HAVE_SSE4_2) - return ByteStreamSplitEncodeSse2(raw_values, static_cast(num_values), - output_buffer_raw); + return ByteStreamSplitEncodeSse2(raw_values, num_values, + output_buffer_raw); #else #error "ByteStreamSplitEncodeSimd not implemented" #endif @@ -678,10 +670,9 @@ inline void DoMergeStreams(const uint8_t** src_streams, int width, int64_t nvalu } } -template +template void ByteStreamSplitEncodeScalar(const uint8_t* raw_values, const int64_t num_values, uint8_t* output_buffer_raw) { - constexpr int kNumStreams = static_cast(sizeof(T)); std::array dest_streams; for (int stream = 0; stream < kNumStreams; ++stream) { dest_streams[stream] = &output_buffer_raw[stream * num_values]; @@ -689,35 +680,35 @@ void ByteStreamSplitEncodeScalar(const uint8_t* raw_values, const int64_t num_va DoSplitStreams(raw_values, kNumStreams, num_values, dest_streams.data()); } -template +template void ByteStreamSplitDecodeScalar(const uint8_t* data, int64_t num_values, int64_t stride, - T* out) { - constexpr int kNumStreams = static_cast(sizeof(T)); + uint8_t* out) { std::array src_streams; for (int stream = 0; stream < kNumStreams; ++stream) { src_streams[stream] = &data[stream * stride]; } - DoMergeStreams(src_streams.data(), kNumStreams, num_values, - reinterpret_cast(out)); + DoMergeStreams(src_streams.data(), kNumStreams, num_values, out); } -template +template void inline ByteStreamSplitEncode(const uint8_t* raw_values, const int64_t num_values, uint8_t* output_buffer_raw) { #if defined(ARROW_HAVE_SIMD_SPLIT) - return ByteStreamSplitEncodeSimd(raw_values, num_values, output_buffer_raw); + return ByteStreamSplitEncodeSimd(raw_values, num_values, + output_buffer_raw); #else - return ByteStreamSplitEncodeScalar(raw_values, num_values, output_buffer_raw); + return ByteStreamSplitEncodeScalar(raw_values, num_values, + output_buffer_raw); #endif } -template +template void inline ByteStreamSplitDecode(const uint8_t* data, int64_t num_values, int64_t stride, - T* out) { + uint8_t* out) { #if defined(ARROW_HAVE_SIMD_SPLIT) - return ByteStreamSplitDecodeSimd(data, num_values, stride, out); + return ByteStreamSplitDecodeSimd(data, num_values, stride, out); #else - return ByteStreamSplitDecodeScalar(data, num_values, stride, out); + return ByteStreamSplitDecodeScalar(data, num_values, stride, out); #endif } diff --git a/cpp/src/arrow/util/byte_stream_split_test.cc b/cpp/src/arrow/util/byte_stream_split_test.cc index c98f0a086738b..71c6063179ea6 100644 --- a/cpp/src/arrow/util/byte_stream_split_test.cc +++ b/cpp/src/arrow/util/byte_stream_split_test.cc @@ -61,18 +61,30 @@ void ReferenceByteStreamSplitEncode(const uint8_t* src, int width, template class TestByteStreamSplitSpecialized : public ::testing::Test { public: - using EncodeFunc = NamedFunc)>>; - using DecodeFunc = NamedFunc)>>; - static constexpr int kWidth = static_cast(sizeof(T)); + using EncodeFunc = NamedFunc)>>; + using DecodeFunc = NamedFunc)>>; + void SetUp() override { encode_funcs_.push_back({"reference", &ReferenceEncode}); - encode_funcs_.push_back({"scalar", &ByteStreamSplitEncodeScalar}); - decode_funcs_.push_back({"scalar", &ByteStreamSplitDecodeScalar}); + encode_funcs_.push_back({"scalar", &ByteStreamSplitEncodeScalar}); + decode_funcs_.push_back({"scalar", &ByteStreamSplitDecodeScalar}); #if defined(ARROW_HAVE_SIMD_SPLIT) - encode_funcs_.push_back({"simd", &ByteStreamSplitEncodeSimd}); - decode_funcs_.push_back({"simd", &ByteStreamSplitDecodeSimd}); + encode_funcs_.push_back({"simd", &ByteStreamSplitEncodeSimd}); + decode_funcs_.push_back({"simd", &ByteStreamSplitDecodeSimd}); +#endif +#if defined(ARROW_HAVE_SSE4_2) + encode_funcs_.push_back({"sse2", &ByteStreamSplitEncodeSse2}); + decode_funcs_.push_back({"sse2", &ByteStreamSplitDecodeSse2}); +#endif +#if defined(ARROW_HAVE_AVX2) + encode_funcs_.push_back({"avx2", &ByteStreamSplitEncodeAvx2}); + decode_funcs_.push_back({"avx2", &ByteStreamSplitDecodeAvx2}); +#endif +#if defined(ARROW_HAVE_AVX512) + encode_funcs_.push_back({"avx512", &ByteStreamSplitEncodeAvx512}); + decode_funcs_.push_back({"avx512", &ByteStreamSplitDecodeAvx512}); #endif } @@ -92,7 +104,7 @@ class TestByteStreamSplitSpecialized : public ::testing::Test { ARROW_SCOPED_TRACE("decode_func = ", decode_func); decoded.assign(decoded.size(), T{}); decode_func.func(encoded.data(), num_values, /*stride=*/num_values, - decoded.data()); + reinterpret_cast(decoded.data())); ASSERT_EQ(decoded, input); } } @@ -118,7 +130,7 @@ class TestByteStreamSplitSpecialized : public ::testing::Test { while (offset < num_values) { auto chunk_size = std::min(num_values - offset, chunk_size_dist(gen)); decode_func.func(encoded.data() + offset, chunk_size, /*stride=*/num_values, - decoded.data() + offset); + reinterpret_cast(decoded.data() + offset)); offset += chunk_size; } ASSERT_EQ(offset, num_values); diff --git a/cpp/src/parquet/encoding.cc b/cpp/src/parquet/encoding.cc index b07ad6c9fb062..b801b5ab11bb9 100644 --- a/cpp/src/parquet/encoding.cc +++ b/cpp/src/parquet/encoding.cc @@ -850,8 +850,8 @@ std::shared_ptr ByteStreamSplitEncoder::FlushValues() { AllocateBuffer(this->memory_pool(), EstimatedDataEncodedSize()); uint8_t* output_buffer_raw = output_buffer->mutable_data(); const uint8_t* raw_values = sink_.data(); - ::arrow::util::internal::ByteStreamSplitEncode(raw_values, num_values_in_buffer_, - output_buffer_raw); + ::arrow::util::internal::ByteStreamSplitEncode( + raw_values, num_values_in_buffer_, output_buffer_raw); sink_.Reset(); num_values_in_buffer_ = 0; return std::move(output_buffer); @@ -3577,7 +3577,7 @@ class ByteStreamSplitDecoder : public DecoderImpl, virtual public TypedDecoder decode_buffer_; - static constexpr size_t kNumStreams = sizeof(T); + static constexpr int kNumStreams = sizeof(T); }; template @@ -3607,8 +3607,8 @@ int ByteStreamSplitDecoder::Decode(T* buffer, int max_values) { const int num_decoded_previously = num_values_in_buffer_ - num_values_; const uint8_t* data = data_ + num_decoded_previously; - ::arrow::util::internal::ByteStreamSplitDecode(data, values_to_decode, - num_values_in_buffer_, buffer); + ::arrow::util::internal::ByteStreamSplitDecode( + data, values_to_decode, num_values_in_buffer_, reinterpret_cast(buffer)); num_values_ -= values_to_decode; len_ -= sizeof(T) * values_to_decode; return values_to_decode; @@ -3618,7 +3618,7 @@ template int ByteStreamSplitDecoder::DecodeArrow( int num_values, int null_count, const uint8_t* valid_bits, int64_t valid_bits_offset, typename EncodingTraits::Accumulator* builder) { - constexpr int value_size = static_cast(kNumStreams); + constexpr int value_size = kNumStreams; int values_decoded = num_values - null_count; if (ARROW_PREDICT_FALSE(len_ < value_size * values_decoded)) { ParquetException::EofException(); @@ -3634,8 +3634,9 @@ int ByteStreamSplitDecoder::DecodeArrow( // Use fast decoding into intermediate buffer. This will also decode // some null values, but it's fast enough that we don't care. T* decode_out = EnsureDecodeBuffer(values_decoded); - ::arrow::util::internal::ByteStreamSplitDecode(data, values_decoded, - num_values_in_buffer_, decode_out); + ::arrow::util::internal::ByteStreamSplitDecode( + data, values_decoded, num_values_in_buffer_, + reinterpret_cast(decode_out)); // XXX If null_count is 0, we could even append in bulk or decode directly into // builder @@ -3648,12 +3649,13 @@ int ByteStreamSplitDecoder::DecodeArrow( [&]() { builder->UnsafeAppendNull(); }); #else + // XXX should operate over runs of 0s / 1s VisitNullBitmapInline( valid_bits, valid_bits_offset, num_values, null_count, [&]() { uint8_t gathered_byte_data[kNumStreams]; - for (size_t b = 0; b < kNumStreams; ++b) { - const size_t byte_index = b * num_values_in_buffer_ + offset; + for (int b = 0; b < kNumStreams; ++b) { + const int64_t byte_index = b * num_values_in_buffer_ + offset; gathered_byte_data[b] = data[byte_index]; } builder->UnsafeAppend(SafeLoadAs(&gathered_byte_data[0])); diff --git a/cpp/src/parquet/encoding_benchmark.cc b/cpp/src/parquet/encoding_benchmark.cc index b5b6cc8d93e03..76c411244b22d 100644 --- a/cpp/src/parquet/encoding_benchmark.cc +++ b/cpp/src/parquet/encoding_benchmark.cc @@ -369,7 +369,8 @@ static void BM_ByteStreamSplitDecode(benchmark::State& state, DecodeFunc&& decod for (auto _ : state) { decode_func(values_raw, static_cast(values.size()), - static_cast(values.size()), output.data()); + static_cast(values.size()), + reinterpret_cast(output.data())); benchmark::ClobberMemory(); } state.SetBytesProcessed(state.iterations() * values.size() * sizeof(T)); @@ -390,22 +391,22 @@ static void BM_ByteStreamSplitEncode(benchmark::State& state, EncodeFunc&& encod static void BM_ByteStreamSplitDecode_Float_Scalar(benchmark::State& state) { BM_ByteStreamSplitDecode( - state, ::arrow::util::internal::ByteStreamSplitDecodeScalar); + state, ::arrow::util::internal::ByteStreamSplitDecodeScalar); } static void BM_ByteStreamSplitDecode_Double_Scalar(benchmark::State& state) { BM_ByteStreamSplitDecode( - state, ::arrow::util::internal::ByteStreamSplitDecodeScalar); + state, ::arrow::util::internal::ByteStreamSplitDecodeScalar); } static void BM_ByteStreamSplitEncode_Float_Scalar(benchmark::State& state) { BM_ByteStreamSplitEncode( - state, ::arrow::util::internal::ByteStreamSplitEncodeScalar); + state, ::arrow::util::internal::ByteStreamSplitEncodeScalar); } static void BM_ByteStreamSplitEncode_Double_Scalar(benchmark::State& state) { BM_ByteStreamSplitEncode( - state, ::arrow::util::internal::ByteStreamSplitEncodeScalar); + state, ::arrow::util::internal::ByteStreamSplitEncodeScalar); } BENCHMARK(BM_ByteStreamSplitDecode_Float_Scalar)->Range(MIN_RANGE, MAX_RANGE); @@ -416,22 +417,22 @@ BENCHMARK(BM_ByteStreamSplitEncode_Double_Scalar)->Range(MIN_RANGE, MAX_RANGE); #if defined(ARROW_HAVE_SSE4_2) static void BM_ByteStreamSplitDecode_Float_Sse2(benchmark::State& state) { BM_ByteStreamSplitDecode( - state, ::arrow::util::internal::ByteStreamSplitDecodeSse2); + state, ::arrow::util::internal::ByteStreamSplitDecodeSse2); } static void BM_ByteStreamSplitDecode_Double_Sse2(benchmark::State& state) { BM_ByteStreamSplitDecode( - state, ::arrow::util::internal::ByteStreamSplitDecodeSse2); + state, ::arrow::util::internal::ByteStreamSplitDecodeSse2); } static void BM_ByteStreamSplitEncode_Float_Sse2(benchmark::State& state) { BM_ByteStreamSplitEncode( - state, ::arrow::util::internal::ByteStreamSplitEncodeSse2); + state, ::arrow::util::internal::ByteStreamSplitEncodeSse2); } static void BM_ByteStreamSplitEncode_Double_Sse2(benchmark::State& state) { BM_ByteStreamSplitEncode( - state, ::arrow::util::internal::ByteStreamSplitEncodeSse2); + state, ::arrow::util::internal::ByteStreamSplitEncodeSse2); } BENCHMARK(BM_ByteStreamSplitDecode_Float_Sse2)->Range(MIN_RANGE, MAX_RANGE); @@ -443,22 +444,22 @@ BENCHMARK(BM_ByteStreamSplitEncode_Double_Sse2)->Range(MIN_RANGE, MAX_RANGE); #if defined(ARROW_HAVE_AVX2) static void BM_ByteStreamSplitDecode_Float_Avx2(benchmark::State& state) { BM_ByteStreamSplitDecode( - state, ::arrow::util::internal::ByteStreamSplitDecodeAvx2); + state, ::arrow::util::internal::ByteStreamSplitDecodeAvx2); } static void BM_ByteStreamSplitDecode_Double_Avx2(benchmark::State& state) { BM_ByteStreamSplitDecode( - state, ::arrow::util::internal::ByteStreamSplitDecodeAvx2); + state, ::arrow::util::internal::ByteStreamSplitDecodeAvx2); } static void BM_ByteStreamSplitEncode_Float_Avx2(benchmark::State& state) { BM_ByteStreamSplitEncode( - state, ::arrow::util::internal::ByteStreamSplitEncodeAvx2); + state, ::arrow::util::internal::ByteStreamSplitEncodeAvx2); } static void BM_ByteStreamSplitEncode_Double_Avx2(benchmark::State& state) { BM_ByteStreamSplitEncode( - state, ::arrow::util::internal::ByteStreamSplitEncodeAvx2); + state, ::arrow::util::internal::ByteStreamSplitEncodeAvx2); } BENCHMARK(BM_ByteStreamSplitDecode_Float_Avx2)->Range(MIN_RANGE, MAX_RANGE); @@ -470,22 +471,22 @@ BENCHMARK(BM_ByteStreamSplitEncode_Double_Avx2)->Range(MIN_RANGE, MAX_RANGE); #if defined(ARROW_HAVE_AVX512) static void BM_ByteStreamSplitDecode_Float_Avx512(benchmark::State& state) { BM_ByteStreamSplitDecode( - state, ::arrow::util::internal::ByteStreamSplitDecodeAvx512); + state, ::arrow::util::internal::ByteStreamSplitDecodeAvx512); } static void BM_ByteStreamSplitDecode_Double_Avx512(benchmark::State& state) { BM_ByteStreamSplitDecode( - state, ::arrow::util::internal::ByteStreamSplitDecodeAvx512); + state, ::arrow::util::internal::ByteStreamSplitDecodeAvx512); } static void BM_ByteStreamSplitEncode_Float_Avx512(benchmark::State& state) { BM_ByteStreamSplitEncode( - state, ::arrow::util::internal::ByteStreamSplitEncodeAvx512); + state, ::arrow::util::internal::ByteStreamSplitEncodeAvx512); } static void BM_ByteStreamSplitEncode_Double_Avx512(benchmark::State& state) { BM_ByteStreamSplitEncode( - state, ::arrow::util::internal::ByteStreamSplitEncodeAvx512); + state, ::arrow::util::internal::ByteStreamSplitEncodeAvx512); } BENCHMARK(BM_ByteStreamSplitDecode_Float_Avx512)->Range(MIN_RANGE, MAX_RANGE); From 015c2d676f62477aca29fdd244988ea13b331814 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 23 Jan 2024 11:20:57 +0100 Subject: [PATCH 69/92] MINOR: Revert "GH-39628: [C++] Use -j1 for cmake >= 3.28" (#39736) Revert apache/arrow#39629: it makes all builds using CMake >= 3.28 much slower, while only addressing a very specific build failure. Hopefully we can find a more targeted workaround. Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 1f5cd3a2b4d5d..6bb9c0f6af2ca 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1005,13 +1005,8 @@ if("${MAKE}" STREQUAL "") endif() endif() -# Args for external projects using make -if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.28") - # Prevent 'bad file descriptor' error see #39517 #39628 - set(MAKE_BUILD_ARGS "-j1") -else() - set(MAKE_BUILD_ARGS "-j${NPROC}") -endif() +# Args for external projects using make. +set(MAKE_BUILD_ARGS "-j${NPROC}") include(FetchContent) set(FC_DECLARE_COMMON_OPTIONS) @@ -2639,7 +2634,7 @@ macro(build_bzip2) BUILD_IN_SOURCE 1 BUILD_COMMAND ${MAKE} libbz2.a ${MAKE_BUILD_ARGS} ${BZIP2_EXTRA_ARGS} - INSTALL_COMMAND ${MAKE} install -j1 PREFIX=${BZIP2_PREFIX} + INSTALL_COMMAND ${MAKE} install PREFIX=${BZIP2_PREFIX} ${BZIP2_EXTRA_ARGS} INSTALL_DIR ${BZIP2_PREFIX} URL ${ARROW_BZIP2_SOURCE_URL} From eed53bbd59957a80c8f55fe4d265cd2371fbea11 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 23 Jan 2024 12:32:57 +0100 Subject: [PATCH 70/92] MINOR: [Docs] Fix formatting of note on Device data interface docs (#39757) Authored-by: Joris Van den Bossche Signed-off-by: Joris Van den Bossche --- docs/source/format/CDeviceDataInterface.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/format/CDeviceDataInterface.rst b/docs/source/format/CDeviceDataInterface.rst index 76b7132681b02..b5b7229a679e1 100644 --- a/docs/source/format/CDeviceDataInterface.rst +++ b/docs/source/format/CDeviceDataInterface.rst @@ -341,8 +341,8 @@ Notes: * \(1) Currently unknown if framework has an event type to support. * \(2) Extension Device has producer defined semantics and thus if - synchronization is needed for an extension device, the producer - should document the type. + synchronization is needed for an extension device, the producer + should document the type. Semantics From 7e9f2658786b966685ddedf6b90415968f207b75 Mon Sep 17 00:00:00 2001 From: Rok Mihevc Date: Tue, 23 Jan 2024 12:43:05 +0100 Subject: [PATCH 71/92] GH-38655: [C++] "iso_calendar" kernel returns incorrect results for array length > 32 (#39360) ### Rationale for this change When defining `StructArray`'s field builders for `ISOCalendar` we don't pre-allocate memory and then use unsafe append. This causes the resulting array to be at most 32 rows long. ### What changes are included in this PR? This introduces required memory pre-allocation in the `ISOCalendar` c++ kernel. ### Are these changes tested? This adds a test for the Python wrapper. ### Are there any user-facing changes? Fixes the behavior of `iso_calendar` kernel. * Closes: #38655 Lead-authored-by: Rok Mihevc Co-authored-by: Joris Van den Bossche Signed-off-by: Joris Van den Bossche --- .../arrow/compute/kernels/scalar_temporal_unary.cc | 2 +- python/pyarrow/tests/test_compute.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc b/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc index a88ce389360f5..f49e201492c9b 100644 --- a/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc +++ b/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc @@ -1510,7 +1510,7 @@ struct ISOCalendar { for (int i = 0; i < 3; i++) { field_builders.push_back( checked_cast(struct_builder->field_builder(i))); - RETURN_NOT_OK(field_builders[i]->Reserve(1)); + RETURN_NOT_OK(field_builders[i]->Reserve(in.length)); } auto visit_null = [&]() { return struct_builder->AppendNull(); }; std::function visit_value; diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 34d4da580f526..4b58dc65bae9b 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -2263,6 +2263,19 @@ def test_extract_datetime_components(): _check_datetime_components(timestamps, timezone) +@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"]) +def test_iso_calendar_longer_array(unit): + # https://github.com/apache/arrow/issues/38655 + # ensure correct result for array length > 32 + arr = pa.array([datetime.datetime(2022, 1, 2, 9)]*50, pa.timestamp(unit)) + result = pc.iso_calendar(arr) + expected = pa.StructArray.from_arrays( + [[2021]*50, [52]*50, [7]*50], + names=['iso_year', 'iso_week', 'iso_day_of_week'] + ) + assert result.equals(expected) + + @pytest.mark.pandas @pytest.mark.skipif(sys.platform == "win32" and not util.windows_has_tzdata(), reason="Timezone database is not installed on Windows") From ae9c0a96a0a7dbf4710bb16cf043269a1bc33aa7 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Tue, 23 Jan 2024 20:25:39 +0530 Subject: [PATCH 72/92] GH-39330: [Java][CI] Fix or suppress spurious errorprone warnings (#39529) ### Rationale for this change This PR fixes the warnings generated by the errorprone library. ### What changes are included in this PR? Updating the code to remove warnings. Covered modules - [x] algorithm - [x] compression - [x] flight - [x] tools - [x] vector ### Are these changes tested? Tested by existing test cases. ### Are there any user-facing changes? No * Closes: #39330 Lead-authored-by: Vibhatha Lakmal Abeykoon Co-authored-by: vibhatha Signed-off-by: David Li --- .../algorithm/search/ParallelSearcher.java | 5 +- .../FixedWidthOutOfPlaceVectorSorter.java | 6 +- .../VariableWidthOutOfPlaceVectorSorter.java | 11 +- .../deduplicate/TestDeduplicationUtils.java | 6 +- .../TestVectorRunDeduplicator.java | 8 +- .../TestHashTableBasedDictionaryBuilder.java | 59 ++++---- .../TestHashTableDictionaryEncoder.java | 20 +-- .../TestLinearDictionaryEncoder.java | 20 +-- .../TestSearchDictionaryEncoder.java | 20 +-- .../TestSearchTreeBasedDictionaryBuilder.java | 59 ++++---- .../algorithm/misc/TestPartialSumUtils.java | 2 +- .../arrow/algorithm/rank/TestVectorRank.java | 22 +-- .../search/TestParallelSearcher.java | 5 +- .../search/TestVectorRangeSearcher.java | 2 +- .../algorithm/search/TestVectorSearcher.java | 6 +- .../sort/TestCompositeVectorComparator.java | 8 +- .../sort/TestDefaultVectorComparator.java | 8 +- .../algorithm/sort/TestFixedWidthSorting.java | 12 +- .../arrow/algorithm/sort/TestSortingUtil.java | 3 +- .../sort/TestStableVectorComparator.java | 56 ++++---- ...stVariableWidthOutOfPlaceVectorSorter.java | 35 ++--- .../sort/TestVariableWidthSorting.java | 11 +- .../compression/TestCompressionCodec.java | 4 +- java/flight/flight-core/pom.xml | 4 + .../org/apache/arrow/flight/ArrowMessage.java | 15 +- .../arrow/flight/CancelFlightInfoResult.java | 2 +- .../arrow/flight/ErrorFlightMetadata.java | 2 +- .../arrow/flight/FlightCallHeaders.java | 12 +- .../org/apache/arrow/flight/FlightClient.java | 21 ++- .../apache/arrow/flight/FlightDescriptor.java | 2 +- .../apache/arrow/flight/FlightEndpoint.java | 7 +- .../org/apache/arrow/flight/FlightInfo.java | 2 +- .../org/apache/arrow/flight/FlightServer.java | 28 ++-- .../apache/arrow/flight/FlightService.java | 30 ++-- .../org/apache/arrow/flight/FlightStream.java | 24 +++- .../org/apache/arrow/flight/Location.java | 4 +- .../org/apache/arrow/flight/PollInfo.java | 6 +- .../java/org/apache/arrow/flight/Ticket.java | 2 +- .../arrow/flight/auth/AuthConstants.java | 4 +- .../auth2/BearerTokenAuthenticator.java | 1 - .../flight/grpc/ClientInterceptorAdapter.java | 7 +- .../arrow/flight/grpc/MetadataAdapter.java | 20 +-- .../flight/grpc/ServerInterceptorAdapter.java | 2 +- .../apache/arrow/flight/grpc/StatusUtils.java | 3 +- .../apache/arrow/flight/FlightTestUtil.java | 3 - .../arrow/flight/TestApplicationMetadata.java | 3 +- .../apache/arrow/flight/TestBackPressure.java | 6 +- .../arrow/flight/TestBasicOperation.java | 10 +- .../apache/arrow/flight/TestCallOptions.java | 7 +- .../arrow/flight/TestDictionaryUtils.java | 7 +- .../apache/arrow/flight/TestDoExchange.java | 14 +- .../arrow/flight/TestErrorMetadata.java | 4 +- .../arrow/flight/TestFlightGrpcUtils.java | 2 +- .../arrow/flight/TestFlightService.java | 5 +- .../apache/arrow/flight/TestLargeMessage.java | 3 +- .../arrow/flight/TestServerMiddleware.java | 32 ++--- .../java/org/apache/arrow/flight/TestTls.java | 6 +- .../flight/client/TestCookieHandling.java | 1 + .../flight/perf/PerformanceTestServer.java | 35 ++--- .../apache/arrow/flight/perf/TestPerf.java | 16 +-- .../tests/IntegrationTestClient.java | 4 +- .../driver/jdbc/ArrowFlightJdbcDriver.java | 2 + .../driver/jdbc/ArrowFlightMetaImpl.java | 5 - .../ArrowFlightJdbcBaseIntVectorAccessor.java | 22 ++- .../ArrowFlightJdbcBitVectorAccessor.java | 1 - .../ArrowFlightConnectionConfigImpl.java | 1 + .../driver/jdbc/utils/ConnectionWrapper.java | 8 +- .../arrow/driver/jdbc/utils/ConvertUtils.java | 7 +- .../arrow/driver/jdbc/utils/UrlParser.java | 3 +- .../driver/jdbc/ArrowFlightJdbcArrayTest.java | 2 +- .../jdbc/ArrowFlightJdbcDriverTest.java | 4 +- .../driver/jdbc/ConnectionMutualTlsTest.java | 4 - .../arrow/driver/jdbc/ResultSetTest.java | 12 +- .../accessor/ArrowFlightJdbcAccessorTest.java | 8 +- ...FlightJdbcTimeStampVectorAccessorTest.java | 6 +- ...stractArrowFlightJdbcListAccessorTest.java | 6 +- ...rowFlightJdbcStructVectorAccessorTest.java | 4 +- .../ArrowFlightJdbcBitVectorAccessorTest.java | 2 +- .../jdbc/utils/MockFlightSqlProducer.java | 4 +- .../jdbc/utils/RootAllocatorTestRule.java | 19 +-- .../jdbc/utils/ThrowableAssertionUtils.java | 2 +- .../arrow/flight/sql/FlightSqlProducer.java | 2 +- .../arrow/flight/sql/SqlInfoBuilder.java | 5 +- .../flight/sql/example/FlightSqlExample.java | 8 +- .../org/apache/arrow/tools/FileRoundtrip.java | 6 +- .../arrow/tools/ArrowFileTestFixtures.java | 2 - .../apache/arrow/tools/EchoServerTest.java | 1 - .../apache/arrow/tools/TestFileRoundtrip.java | 2 +- .../arrow/vector/BaseFixedWidthVector.java | 9 +- .../vector/BaseLargeVariableWidthVector.java | 10 ++ .../apache/arrow/vector/BaseValueVector.java | 6 +- .../arrow/vector/BaseVariableWidthVector.java | 21 ++- .../org/apache/arrow/vector/BigIntVector.java | 1 + .../org/apache/arrow/vector/BitVector.java | 13 +- .../org/apache/arrow/vector/BufferLayout.java | 2 +- .../apache/arrow/vector/DateDayVector.java | 1 + .../apache/arrow/vector/DateMilliVector.java | 1 + .../apache/arrow/vector/Decimal256Vector.java | 1 + .../apache/arrow/vector/DecimalVector.java | 1 + .../apache/arrow/vector/DurationVector.java | 1 + .../org/apache/arrow/vector/Float4Vector.java | 1 + .../org/apache/arrow/vector/Float8Vector.java | 1 + .../arrow/vector/GenerateSampleData.java | 4 +- .../org/apache/arrow/vector/IntVector.java | 1 + .../arrow/vector/IntervalDayVector.java | 17 +-- .../vector/IntervalMonthDayNanoVector.java | 1 + .../arrow/vector/IntervalYearVector.java | 5 +- .../arrow/vector/LargeVarBinaryVector.java | 1 + .../arrow/vector/LargeVarCharVector.java | 1 + .../apache/arrow/vector/SmallIntVector.java | 1 + .../apache/arrow/vector/TimeMicroVector.java | 1 + .../apache/arrow/vector/TimeMilliVector.java | 1 + .../apache/arrow/vector/TimeNanoVector.java | 1 + .../apache/arrow/vector/TimeSecVector.java | 1 + .../arrow/vector/TimeStampMicroTZVector.java | 1 + .../arrow/vector/TimeStampMicroVector.java | 1 + .../arrow/vector/TimeStampMilliTZVector.java | 1 + .../arrow/vector/TimeStampMilliVector.java | 1 + .../arrow/vector/TimeStampNanoTZVector.java | 1 + .../arrow/vector/TimeStampNanoVector.java | 1 + .../arrow/vector/TimeStampSecVector.java | 1 + .../apache/arrow/vector/TinyIntVector.java | 1 + .../org/apache/arrow/vector/TypeLayout.java | 5 +- .../org/apache/arrow/vector/UInt1Vector.java | 1 + .../org/apache/arrow/vector/UInt2Vector.java | 1 + .../org/apache/arrow/vector/UInt4Vector.java | 1 + .../org/apache/arrow/vector/UInt8Vector.java | 1 + .../apache/arrow/vector/VarBinaryVector.java | 11 +- .../apache/arrow/vector/VarCharVector.java | 11 +- .../complex/AbstractContainerVector.java | 1 + .../vector/complex/AbstractStructVector.java | 6 +- .../complex/BaseRepeatedValueVector.java | 20 +-- .../vector/complex/FixedSizeListVector.java | 12 +- .../arrow/vector/complex/LargeListVector.java | 24 ++-- .../arrow/vector/complex/ListVector.java | 33 +++-- .../arrow/vector/complex/StructVector.java | 29 ++-- .../complex/impl/AbstractBaseReader.java | 1 + .../vector/complex/impl/PromotableWriter.java | 5 +- .../complex/impl/StructOrListWriterImpl.java | 3 +- .../impl/UnionFixedSizeListReader.java | 1 + .../complex/impl/UnionLargeListReader.java | 1 - .../vector/complex/impl/UnionListReader.java | 5 +- .../arrow/vector/dictionary/Dictionary.java | 2 +- .../arrow/vector/table/package-info.java | 2 +- .../vector/types/FloatingPointPrecision.java | 2 +- .../arrow/vector/types/IntervalUnit.java | 2 +- .../vector/types/pojo/DictionaryEncoding.java | 2 +- .../apache/arrow/vector/types/pojo/Field.java | 16 ++- .../arrow/vector/types/pojo/FieldType.java | 2 +- .../arrow/vector/types/pojo/Schema.java | 9 +- .../arrow/vector/util/MapWithOrdinalImpl.java | 9 +- .../org/apache/arrow/vector/util/Text.java | 2 +- .../arrow/vector/ITTestLargeVector.java | 11 +- .../arrow/vector/TestBitVectorHelper.java | 6 +- .../vector/TestBufferOwnershipTransfer.java | 4 +- .../org/apache/arrow/vector/TestCopyFrom.java | 17 +-- .../arrow/vector/TestDecimal256Vector.java | 4 +- .../arrow/vector/TestDecimalVector.java | 4 +- .../arrow/vector/TestDenseUnionVector.java | 14 +- .../arrow/vector/TestDictionaryVector.java | 41 +++--- .../arrow/vector/TestFixedSizeListVector.java | 71 +++++----- .../arrow/vector/TestLargeListVector.java | 123 ++++++++--------- .../vector/TestLargeVarBinaryVector.java | 20 +-- .../arrow/vector/TestLargeVarCharVector.java | 49 +++---- .../apache/arrow/vector/TestListVector.java | 130 +++++++++--------- .../apache/arrow/vector/TestMapVector.java | 34 ++--- .../arrow/vector/TestSplitAndTransfer.java | 7 +- .../apache/arrow/vector/TestUnionVector.java | 10 +- .../apache/arrow/vector/TestValueVector.java | 61 ++++---- .../arrow/vector/TestVarCharListVector.java | 10 +- .../apache/arrow/vector/TestVectorAlloc.java | 4 +- .../arrow/vector/TestVectorReAlloc.java | 4 +- .../arrow/vector/TestVectorSchemaRoot.java | 2 +- .../vector/compare/TestTypeEqualsVisitor.java | 6 - .../complex/impl/TestPromotableWriter.java | 90 ++++++------ .../complex/writer/TestComplexWriter.java | 122 ++++++++-------- .../apache/arrow/vector/ipc/BaseFileTest.java | 14 +- .../arrow/vector/ipc/TestArrowStream.java | 4 +- .../apache/arrow/vector/ipc/TestJSONFile.java | 4 +- .../ipc/TestUIntDictionaryRoundTrip.java | 3 +- .../message/TestMessageMetadataResult.java | 2 +- .../arrow/vector/table/BaseTableTest.java | 14 +- .../apache/arrow/vector/table/RowTest.java | 14 +- .../apache/arrow/vector/table/TestUtils.java | 21 +-- .../testing/TestValueVectorPopulator.java | 18 ++- .../vector/types/pojo/TestExtensionType.java | 4 +- .../TestElementAddressableVectorIterator.java | 6 +- .../vector/util/TestReusableByteArray.java | 46 ++++--- .../arrow/vector/util/TestVectorAppender.java | 2 - 189 files changed, 1189 insertions(+), 1031 deletions(-) diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/ParallelSearcher.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/ParallelSearcher.java index e62ebdecb1bac..6226921b22ed6 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/ParallelSearcher.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/ParallelSearcher.java @@ -20,6 +20,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.vector.ValueVector; @@ -95,7 +96,7 @@ public int search(V keyVector, int keyIndex) throws ExecutionException, Interrup final int valueCount = vector.getValueCount(); for (int i = 0; i < numThreads; i++) { final int tid = i; - threadPool.submit(() -> { + Future unused = threadPool.submit(() -> { // convert to long to avoid overflow int start = (int) (((long) valueCount) * tid / numThreads); int end = (int) ((long) valueCount) * (tid + 1) / numThreads; @@ -153,7 +154,7 @@ public int search( final int valueCount = vector.getValueCount(); for (int i = 0; i < numThreads; i++) { final int tid = i; - threadPool.submit(() -> { + Future unused = threadPool.submit(() -> { // convert to long to avoid overflow int start = (int) (((long) valueCount) * tid / numThreads); int end = (int) ((long) valueCount) * (tid + 1) / numThreads; diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java index c3b68facfda97..05a4585792dc2 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java @@ -54,7 +54,7 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co "Expected capacity %s, actual capacity %s", (srcVector.getValueCount() + 7) / 8, dstValidityBuffer.capacity()); Preconditions.checkArgument( - dstValueBuffer.capacity() >= srcVector.getValueCount() * srcVector.getTypeWidth(), + dstValueBuffer.capacity() >= srcVector.getValueCount() * ((long) srcVector.getTypeWidth()), "Not enough capacity for the data buffer of the dst vector. " + "Expected capacity %s, actual capacity %s", srcVector.getValueCount() * srcVector.getTypeWidth(), dstValueBuffer.capacity()); @@ -73,8 +73,8 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co } else { BitVectorHelper.setBit(dstValidityBuffer, dstIndex); MemoryUtil.UNSAFE.copyMemory( - srcValueBuffer.memoryAddress() + srcIndex * valueWidth, - dstValueBuffer.memoryAddress() + dstIndex * valueWidth, + srcValueBuffer.memoryAddress() + srcIndex * ((long) valueWidth), + dstValueBuffer.memoryAddress() + dstIndex * ((long) valueWidth), valueWidth); } } diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java index c60e273e9e851..863b07c348ef2 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java @@ -51,12 +51,12 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co "Expected capacity %s, actual capacity %s", (srcVector.getValueCount() + 7) / 8, dstValidityBuffer.capacity()); Preconditions.checkArgument( - dstOffsetBuffer.capacity() >= (srcVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, + dstOffsetBuffer.capacity() >= (srcVector.getValueCount() + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH), "Not enough capacity for the offset buffer of the dst vector. " + "Expected capacity %s, actual capacity %s", (srcVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, dstOffsetBuffer.capacity()); long dataSize = srcVector.getOffsetBuffer().getInt( - srcVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); + srcVector.getValueCount() * ((long) BaseVariableWidthVector.OFFSET_WIDTH)); Preconditions.checkArgument( dstValueBuffer.capacity() >= dataSize, "No enough capacity for the data buffer of the dst vector. " + "Expected capacity %s, actual capacity %s", dataSize, dstValueBuffer.capacity()); @@ -77,15 +77,16 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co BitVectorHelper.unsetBit(dstValidityBuffer, dstIndex); } else { BitVectorHelper.setBit(dstValidityBuffer, dstIndex); - int srcOffset = srcOffsetBuffer.getInt(srcIndex * BaseVariableWidthVector.OFFSET_WIDTH); - int valueLength = srcOffsetBuffer.getInt((srcIndex + 1) * BaseVariableWidthVector.OFFSET_WIDTH) - srcOffset; + int srcOffset = srcOffsetBuffer.getInt(srcIndex * ((long) BaseVariableWidthVector.OFFSET_WIDTH)); + int valueLength = + srcOffsetBuffer.getInt((srcIndex + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH)) - srcOffset; MemoryUtil.UNSAFE.copyMemory( srcValueBuffer.memoryAddress() + srcOffset, dstValueBuffer.memoryAddress() + dstOffset, valueLength); dstOffset += valueLength; } - dstOffsetBuffer.setInt((dstIndex + 1) * BaseVariableWidthVector.OFFSET_WIDTH, dstOffset); + dstOffsetBuffer.setInt((dstIndex + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH), dstOffset); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java index def83fba7b74a..ac083b84f1611 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -107,7 +109,7 @@ public void testDeduplicateVariableWidth() { for (int i = 0; i < VECTOR_LENGTH; i++) { String str = String.valueOf(i * i); for (int j = 0; j < REPETITION_COUNT; j++) { - origVec.set(i * REPETITION_COUNT + j, str.getBytes()); + origVec.set(i * REPETITION_COUNT + j, str.getBytes(StandardCharsets.UTF_8)); } } @@ -120,7 +122,7 @@ public void testDeduplicateVariableWidth() { assertEquals(VECTOR_LENGTH, dedupVec.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(String.valueOf(i * i).getBytes(), dedupVec.get(i)); + assertArrayEquals(String.valueOf(i * i).getBytes(StandardCharsets.UTF_8), dedupVec.get(i)); } DeduplicationUtils.populateRunLengths( diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestVectorRunDeduplicator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestVectorRunDeduplicator.java index 4bfa6e2555176..788213b162870 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestVectorRunDeduplicator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestVectorRunDeduplicator.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -104,20 +106,20 @@ public void testDeduplicateVariableWidth() { for (int i = 0; i < VECTOR_LENGTH; i++) { String str = String.valueOf(i * i); for (int j = 0; j < REPETITION_COUNT; j++) { - origVec.set(i * REPETITION_COUNT + j, str.getBytes()); + origVec.set(i * REPETITION_COUNT + j, str.getBytes(StandardCharsets.UTF_8)); } } int distinctCount = deduplicator.getRunCount(); assertEquals(VECTOR_LENGTH, distinctCount); - dedupVec.allocateNew(distinctCount * 10, distinctCount); + dedupVec.allocateNew(distinctCount * 10L, distinctCount); deduplicator.populateDeduplicatedValues(dedupVec); assertEquals(VECTOR_LENGTH, dedupVec.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(String.valueOf(i * i).getBytes(), dedupVec.get(i)); + assertArrayEquals(String.valueOf(i * i).getBytes(StandardCharsets.UTF_8), dedupVec.get(i)); } deduplicator.populateRunLengths(lengthVec); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableBasedDictionaryBuilder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableBasedDictionaryBuilder.java index 0a3314535f234..45c47626b720e 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableBasedDictionaryBuilder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableBasedDictionaryBuilder.java @@ -21,6 +21,9 @@ import static org.junit.Assert.assertNull; import static org.junit.jupiter.api.Assertions.assertEquals; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -57,16 +60,16 @@ public void testBuildVariableWidthDictionaryWithNull() { dictionary.allocateNew(); // fill data - vec.set(0, "hello".getBytes()); - vec.set(1, "abc".getBytes()); + vec.set(0, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(1, "abc".getBytes(StandardCharsets.UTF_8)); vec.setNull(2); - vec.set(3, "world".getBytes()); - vec.set(4, "12".getBytes()); - vec.set(5, "dictionary".getBytes()); + vec.set(3, "world".getBytes(StandardCharsets.UTF_8)); + vec.set(4, "12".getBytes(StandardCharsets.UTF_8)); + vec.set(5, "dictionary".getBytes(StandardCharsets.UTF_8)); vec.setNull(6); - vec.set(7, "hello".getBytes()); - vec.set(8, "good".getBytes()); - vec.set(9, "abc".getBytes()); + vec.set(7, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(8, "good".getBytes(StandardCharsets.UTF_8)); + vec.set(9, "abc".getBytes(StandardCharsets.UTF_8)); HashTableBasedDictionaryBuilder dictionaryBuilder = new HashTableBasedDictionaryBuilder<>(dictionary, true); @@ -76,13 +79,13 @@ public void testBuildVariableWidthDictionaryWithNull() { assertEquals(7, result); assertEquals(7, dictionary.getValueCount()); - assertEquals("hello", new String(dictionary.get(0))); - assertEquals("abc", new String(dictionary.get(1))); + assertEquals("hello", new String(Objects.requireNonNull(dictionary.get(0)), StandardCharsets.UTF_8)); + assertEquals("abc", new String(Objects.requireNonNull(dictionary.get(1)), StandardCharsets.UTF_8)); assertNull(dictionary.get(2)); - assertEquals("world", new String(dictionary.get(3))); - assertEquals("12", new String(dictionary.get(4))); - assertEquals("dictionary", new String(dictionary.get(5))); - assertEquals("good", new String(dictionary.get(6))); + assertEquals("world", new String(Objects.requireNonNull(dictionary.get(3)), StandardCharsets.UTF_8)); + assertEquals("12", new String(Objects.requireNonNull(dictionary.get(4)), StandardCharsets.UTF_8)); + assertEquals("dictionary", new String(Objects.requireNonNull(dictionary.get(5)), StandardCharsets.UTF_8)); + assertEquals("good", new String(Objects.requireNonNull(dictionary.get(6)), StandardCharsets.UTF_8)); } } @@ -97,16 +100,16 @@ public void testBuildVariableWidthDictionaryWithoutNull() { dictionary.allocateNew(); // fill data - vec.set(0, "hello".getBytes()); - vec.set(1, "abc".getBytes()); + vec.set(0, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(1, "abc".getBytes(StandardCharsets.UTF_8)); vec.setNull(2); - vec.set(3, "world".getBytes()); - vec.set(4, "12".getBytes()); - vec.set(5, "dictionary".getBytes()); + vec.set(3, "world".getBytes(StandardCharsets.UTF_8)); + vec.set(4, "12".getBytes(StandardCharsets.UTF_8)); + vec.set(5, "dictionary".getBytes(StandardCharsets.UTF_8)); vec.setNull(6); - vec.set(7, "hello".getBytes()); - vec.set(8, "good".getBytes()); - vec.set(9, "abc".getBytes()); + vec.set(7, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(8, "good".getBytes(StandardCharsets.UTF_8)); + vec.set(9, "abc".getBytes(StandardCharsets.UTF_8)); HashTableBasedDictionaryBuilder dictionaryBuilder = new HashTableBasedDictionaryBuilder<>(dictionary, false); @@ -116,12 +119,12 @@ public void testBuildVariableWidthDictionaryWithoutNull() { assertEquals(6, result); assertEquals(6, dictionary.getValueCount()); - assertEquals("hello", new String(dictionary.get(0))); - assertEquals("abc", new String(dictionary.get(1))); - assertEquals("world", new String(dictionary.get(2))); - assertEquals("12", new String(dictionary.get(3))); - assertEquals("dictionary", new String(dictionary.get(4))); - assertEquals("good", new String(dictionary.get(5))); + assertEquals("hello", new String(Objects.requireNonNull(dictionary.get(0)), StandardCharsets.UTF_8)); + assertEquals("abc", new String(Objects.requireNonNull(dictionary.get(1)), StandardCharsets.UTF_8)); + assertEquals("world", new String(Objects.requireNonNull(dictionary.get(2)), StandardCharsets.UTF_8)); + assertEquals("12", new String(Objects.requireNonNull(dictionary.get(3)), StandardCharsets.UTF_8)); + assertEquals("dictionary", new String(Objects.requireNonNull(dictionary.get(4)), StandardCharsets.UTF_8)); + assertEquals("good", new String(Objects.requireNonNull(dictionary.get(5)), StandardCharsets.UTF_8)); } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableDictionaryEncoder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableDictionaryEncoder.java index dd22ac96fac88..60efbf58bebda 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableDictionaryEncoder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableDictionaryEncoder.java @@ -76,7 +76,7 @@ public void testEncodeAndDecode() { dictionary.allocateNew(); for (int i = 0; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); @@ -84,7 +84,7 @@ public void testEncodeAndDecode() { rawVector.allocateNew(10 * VECTOR_LENGTH, VECTOR_LENGTH); for (int i = 0; i < VECTOR_LENGTH; i++) { int val = (random.nextInt() & Integer.MAX_VALUE) % DICTIONARY_LENGTH; - rawVector.set(i, String.valueOf(val).getBytes()); + rawVector.set(i, String.valueOf(val).getBytes(StandardCharsets.UTF_8)); } rawVector.setValueCount(VECTOR_LENGTH); @@ -98,7 +98,7 @@ public void testEncodeAndDecode() { // verify encoding results assertEquals(rawVector.getValueCount(), encodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes()); + assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } // perform decoding @@ -108,7 +108,8 @@ public void testEncodeAndDecode() { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(), decodedVector.get(i)); + assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + decodedVector.get(i)); } } } @@ -126,7 +127,7 @@ public void testEncodeAndDecodeWithNull() { dictionary.setNull(0); for (int i = 1; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); @@ -137,7 +138,7 @@ public void testEncodeAndDecodeWithNull() { rawVector.setNull(i); } else { int val = (random.nextInt() & Integer.MAX_VALUE) % (DICTIONARY_LENGTH - 1) + 1; - rawVector.set(i, String.valueOf(val).getBytes()); + rawVector.set(i, String.valueOf(val).getBytes(StandardCharsets.UTF_8)); } } rawVector.setValueCount(VECTOR_LENGTH); @@ -155,7 +156,7 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertEquals(0, encodedVector.get(i)); } else { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes()); + assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } } @@ -168,7 +169,8 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertTrue(decodedVector.isNull(i)); } else { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(), decodedVector.get(i)); + assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + decodedVector.get(i)); } } } @@ -185,7 +187,7 @@ public void testEncodeNullWithoutNullInDictionary() { dictionary.allocateNew(); for (int i = 0; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestLinearDictionaryEncoder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestLinearDictionaryEncoder.java index 104d1b35b0660..a76aedffa308d 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestLinearDictionaryEncoder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestLinearDictionaryEncoder.java @@ -77,7 +77,7 @@ public void testEncodeAndDecode() { dictionary.allocateNew(); for (int i = 0; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); @@ -85,7 +85,7 @@ public void testEncodeAndDecode() { rawVector.allocateNew(10 * VECTOR_LENGTH, VECTOR_LENGTH); for (int i = 0; i < VECTOR_LENGTH; i++) { int val = (random.nextInt() & Integer.MAX_VALUE) % DICTIONARY_LENGTH; - rawVector.set(i, String.valueOf(val).getBytes()); + rawVector.set(i, String.valueOf(val).getBytes(StandardCharsets.UTF_8)); } rawVector.setValueCount(VECTOR_LENGTH); @@ -99,7 +99,7 @@ public void testEncodeAndDecode() { // verify encoding results assertEquals(rawVector.getValueCount(), encodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes()); + assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } // perform decoding @@ -109,7 +109,8 @@ public void testEncodeAndDecode() { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(), decodedVector.get(i)); + assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + decodedVector.get(i)); } } } @@ -127,7 +128,7 @@ public void testEncodeAndDecodeWithNull() { dictionary.setNull(0); for (int i = 1; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); @@ -138,7 +139,7 @@ public void testEncodeAndDecodeWithNull() { rawVector.setNull(i); } else { int val = (random.nextInt() & Integer.MAX_VALUE) % (DICTIONARY_LENGTH - 1) + 1; - rawVector.set(i, String.valueOf(val).getBytes()); + rawVector.set(i, String.valueOf(val).getBytes(StandardCharsets.UTF_8)); } } rawVector.setValueCount(VECTOR_LENGTH); @@ -156,7 +157,7 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertEquals(0, encodedVector.get(i)); } else { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes()); + assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } } @@ -170,7 +171,8 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertTrue(decodedVector.isNull(i)); } else { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(), decodedVector.get(i)); + assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + decodedVector.get(i)); } } } @@ -187,7 +189,7 @@ public void testEncodeNullWithoutNullInDictionary() { dictionary.allocateNew(); for (int i = 0; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchDictionaryEncoder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchDictionaryEncoder.java index a156e987c20ce..e01c2e7905b46 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchDictionaryEncoder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchDictionaryEncoder.java @@ -78,7 +78,7 @@ public void testEncodeAndDecode() { dictionary.allocateNew(); for (int i = 0; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); @@ -86,7 +86,7 @@ public void testEncodeAndDecode() { rawVector.allocateNew(10 * VECTOR_LENGTH, VECTOR_LENGTH); for (int i = 0; i < VECTOR_LENGTH; i++) { int val = (random.nextInt() & Integer.MAX_VALUE) % DICTIONARY_LENGTH; - rawVector.set(i, String.valueOf(val).getBytes()); + rawVector.set(i, String.valueOf(val).getBytes(StandardCharsets.UTF_8)); } rawVector.setValueCount(VECTOR_LENGTH); @@ -101,7 +101,7 @@ public void testEncodeAndDecode() { // verify encoding results assertEquals(rawVector.getValueCount(), encodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes()); + assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } // perform decoding @@ -111,7 +111,8 @@ public void testEncodeAndDecode() { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(), decodedVector.get(i)); + assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + decodedVector.get(i)); } } } @@ -129,7 +130,7 @@ public void testEncodeAndDecodeWithNull() { dictionary.setNull(0); for (int i = 1; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); @@ -140,7 +141,7 @@ public void testEncodeAndDecodeWithNull() { rawVector.setNull(i); } else { int val = (random.nextInt() & Integer.MAX_VALUE) % (DICTIONARY_LENGTH - 1) + 1; - rawVector.set(i, String.valueOf(val).getBytes()); + rawVector.set(i, String.valueOf(val).getBytes(StandardCharsets.UTF_8)); } } rawVector.setValueCount(VECTOR_LENGTH); @@ -159,7 +160,7 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertEquals(0, encodedVector.get(i)); } else { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes()); + assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } } @@ -173,7 +174,8 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertTrue(decodedVector.isNull(i)); } else { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(), decodedVector.get(i)); + assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + decodedVector.get(i)); } } } @@ -190,7 +192,7 @@ public void testEncodeNullWithoutNullInDictionary() { dictionary.allocateNew(); for (int i = 0; i < DICTIONARY_LENGTH; i++) { // encode "i" as i - dictionary.setSafe(i, String.valueOf(i).getBytes()); + dictionary.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionary.setValueCount(DICTIONARY_LENGTH); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchTreeBasedDictionaryBuilder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchTreeBasedDictionaryBuilder.java index d8e9edce83b7f..340b7e67e861f 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchTreeBasedDictionaryBuilder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchTreeBasedDictionaryBuilder.java @@ -20,6 +20,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -60,16 +63,16 @@ public void testBuildVariableWidthDictionaryWithNull() { sortedDictionary.allocateNew(); // fill data - vec.set(0, "hello".getBytes()); - vec.set(1, "abc".getBytes()); + vec.set(0, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(1, "abc".getBytes(StandardCharsets.UTF_8)); vec.setNull(2); - vec.set(3, "world".getBytes()); - vec.set(4, "12".getBytes()); - vec.set(5, "dictionary".getBytes()); + vec.set(3, "world".getBytes(StandardCharsets.UTF_8)); + vec.set(4, "12".getBytes(StandardCharsets.UTF_8)); + vec.set(5, "dictionary".getBytes(StandardCharsets.UTF_8)); vec.setNull(6); - vec.set(7, "hello".getBytes()); - vec.set(8, "good".getBytes()); - vec.set(9, "abc".getBytes()); + vec.set(7, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(8, "good".getBytes(StandardCharsets.UTF_8)); + vec.set(9, "abc".getBytes(StandardCharsets.UTF_8)); VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); SearchTreeBasedDictionaryBuilder dictionaryBuilder = @@ -83,12 +86,12 @@ public void testBuildVariableWidthDictionaryWithNull() { dictionaryBuilder.populateSortedDictionary(sortedDictionary); assertTrue(sortedDictionary.isNull(0)); - assertEquals("12", new String(sortedDictionary.get(1))); - assertEquals("abc", new String(sortedDictionary.get(2))); - assertEquals("dictionary", new String(sortedDictionary.get(3))); - assertEquals("good", new String(sortedDictionary.get(4))); - assertEquals("hello", new String(sortedDictionary.get(5))); - assertEquals("world", new String(sortedDictionary.get(6))); + assertEquals("12", new String(Objects.requireNonNull(sortedDictionary.get(1)), StandardCharsets.UTF_8)); + assertEquals("abc", new String(Objects.requireNonNull(sortedDictionary.get(2)), StandardCharsets.UTF_8)); + assertEquals("dictionary", new String(Objects.requireNonNull(sortedDictionary.get(3)), StandardCharsets.UTF_8)); + assertEquals("good", new String(Objects.requireNonNull(sortedDictionary.get(4)), StandardCharsets.UTF_8)); + assertEquals("hello", new String(Objects.requireNonNull(sortedDictionary.get(5)), StandardCharsets.UTF_8)); + assertEquals("world", new String(Objects.requireNonNull(sortedDictionary.get(6)), StandardCharsets.UTF_8)); } } @@ -105,16 +108,16 @@ public void testBuildVariableWidthDictionaryWithoutNull() { sortedDictionary.allocateNew(); // fill data - vec.set(0, "hello".getBytes()); - vec.set(1, "abc".getBytes()); + vec.set(0, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(1, "abc".getBytes(StandardCharsets.UTF_8)); vec.setNull(2); - vec.set(3, "world".getBytes()); - vec.set(4, "12".getBytes()); - vec.set(5, "dictionary".getBytes()); + vec.set(3, "world".getBytes(StandardCharsets.UTF_8)); + vec.set(4, "12".getBytes(StandardCharsets.UTF_8)); + vec.set(5, "dictionary".getBytes(StandardCharsets.UTF_8)); vec.setNull(6); - vec.set(7, "hello".getBytes()); - vec.set(8, "good".getBytes()); - vec.set(9, "abc".getBytes()); + vec.set(7, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(8, "good".getBytes(StandardCharsets.UTF_8)); + vec.set(9, "abc".getBytes(StandardCharsets.UTF_8)); VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); SearchTreeBasedDictionaryBuilder dictionaryBuilder = @@ -127,12 +130,12 @@ public void testBuildVariableWidthDictionaryWithoutNull() { dictionaryBuilder.populateSortedDictionary(sortedDictionary); - assertEquals("12", new String(sortedDictionary.get(0))); - assertEquals("abc", new String(sortedDictionary.get(1))); - assertEquals("dictionary", new String(sortedDictionary.get(2))); - assertEquals("good", new String(sortedDictionary.get(3))); - assertEquals("hello", new String(sortedDictionary.get(4))); - assertEquals("world", new String(sortedDictionary.get(5))); + assertEquals("12", new String(Objects.requireNonNull(sortedDictionary.get(0)), StandardCharsets.UTF_8)); + assertEquals("abc", new String(Objects.requireNonNull(sortedDictionary.get(1)), StandardCharsets.UTF_8)); + assertEquals("dictionary", new String(Objects.requireNonNull(sortedDictionary.get(2)), StandardCharsets.UTF_8)); + assertEquals("good", new String(Objects.requireNonNull(sortedDictionary.get(3)), StandardCharsets.UTF_8)); + assertEquals("hello", new String(Objects.requireNonNull(sortedDictionary.get(4)), StandardCharsets.UTF_8)); + assertEquals("world", new String(Objects.requireNonNull(sortedDictionary.get(5)), StandardCharsets.UTF_8)); } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/misc/TestPartialSumUtils.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/misc/TestPartialSumUtils.java index 4e2d5900f8ccc..630dd80b44084 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/misc/TestPartialSumUtils.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/misc/TestPartialSumUtils.java @@ -67,7 +67,7 @@ public void testToPartialSumVector() { // verify results assertEquals(PARTIAL_SUM_VECTOR_LENGTH, partialSum.getValueCount()); for (int i = 0; i < partialSum.getValueCount(); i++) { - assertEquals(i * 3 + sumBase, partialSum.get(i)); + assertEquals(i * 3L + sumBase, partialSum.get(i)); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/rank/TestVectorRank.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/rank/TestVectorRank.java index f372a809bab53..0e6627eb4822a 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/rank/TestVectorRank.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/rank/TestVectorRank.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -89,16 +91,16 @@ public void testVariableWidthRank() { vector.allocateNew(VECTOR_LENGTH * 5, VECTOR_LENGTH); vector.setValueCount(VECTOR_LENGTH); - vector.set(0, String.valueOf(1).getBytes()); - vector.set(1, String.valueOf(5).getBytes()); - vector.set(2, String.valueOf(3).getBytes()); - vector.set(3, String.valueOf(7).getBytes()); - vector.set(4, String.valueOf(9).getBytes()); - vector.set(5, String.valueOf(8).getBytes()); - vector.set(6, String.valueOf(2).getBytes()); - vector.set(7, String.valueOf(0).getBytes()); - vector.set(8, String.valueOf(4).getBytes()); - vector.set(9, String.valueOf(6).getBytes()); + vector.set(0, String.valueOf(1).getBytes(StandardCharsets.UTF_8)); + vector.set(1, String.valueOf(5).getBytes(StandardCharsets.UTF_8)); + vector.set(2, String.valueOf(3).getBytes(StandardCharsets.UTF_8)); + vector.set(3, String.valueOf(7).getBytes(StandardCharsets.UTF_8)); + vector.set(4, String.valueOf(9).getBytes(StandardCharsets.UTF_8)); + vector.set(5, String.valueOf(8).getBytes(StandardCharsets.UTF_8)); + vector.set(6, String.valueOf(2).getBytes(StandardCharsets.UTF_8)); + vector.set(7, String.valueOf(0).getBytes(StandardCharsets.UTF_8)); + vector.set(8, String.valueOf(4).getBytes(StandardCharsets.UTF_8)); + vector.set(9, String.valueOf(6).getBytes(StandardCharsets.UTF_8)); VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vector); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestParallelSearcher.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestParallelSearcher.java index 767935aaa4bae..9ccecfa84a73a 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestParallelSearcher.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestParallelSearcher.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -130,8 +131,8 @@ public void testParallelStringSearch() throws ExecutionException, InterruptedExc : DefaultVectorComparators.createDefaultComparator(targetVector); for (int i = 0; i < VECTOR_LENGTH; i++) { - targetVector.setSafe(i, String.valueOf(i).getBytes()); - keyVector.setSafe(i, String.valueOf(i * 2).getBytes()); + targetVector.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); + keyVector.setSafe(i, String.valueOf(i * 2).getBytes(StandardCharsets.UTF_8)); } targetVector.setValueCount(VECTOR_LENGTH); keyVector.setValueCount(VECTOR_LENGTH); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorRangeSearcher.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorRangeSearcher.java index d7659dc4cfa03..18f4fa0355f4f 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorRangeSearcher.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorRangeSearcher.java @@ -81,7 +81,7 @@ public void testGetLowerBounds() { VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(intVector); for (int i = 0; i < maxValue; i++) { int result = VectorRangeSearcher.getFirstMatch(intVector, comparator, intVector, i * repeat); - assertEquals(i * repeat, result); + assertEquals(i * ((long) repeat), result); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorSearcher.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorSearcher.java index 2847ddbb8ada6..32fa10bbd98d0 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorSearcher.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorSearcher.java @@ -20,6 +20,8 @@ import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.OFFSET_WIDTH; import static org.junit.Assert.assertEquals; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -142,7 +144,7 @@ public void testBinarySearchVarChar() { rawVector.set(i, content); } } - negVector.set(0, "abcd".getBytes()); + negVector.set(0, "abcd".getBytes(StandardCharsets.UTF_8)); // do search VectorValueComparator comparator = @@ -181,7 +183,7 @@ public void testLinearSearchVarChar() { rawVector.set(i, content); } } - negVector.set(0, "abcd".getBytes()); + negVector.set(0, "abcd".getBytes(StandardCharsets.UTF_8)); // do search VectorValueComparator comparator = diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestCompositeVectorComparator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestCompositeVectorComparator.java index cac9933cc0bc2..9624432924b5a 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestCompositeVectorComparator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestCompositeVectorComparator.java @@ -17,8 +17,10 @@ package org.apache.arrow.algorithm.sort; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import org.apache.arrow.memory.BufferAllocator; @@ -67,9 +69,9 @@ public void testCompareVectorSchemaRoot() { for (int i = 0; i < vectorLength; i++) { intVec1.set(i, i); - strVec1.set(i, new String("a" + i).getBytes()); + strVec1.set(i, ("a" + i).getBytes(StandardCharsets.UTF_8)); intVec2.set(i, i); - strVec2.set(i, new String("a5").getBytes()); + strVec2.set(i, "a5".getBytes(StandardCharsets.UTF_8)); } VectorValueComparator innerComparator1 = @@ -86,7 +88,7 @@ public void testCompareVectorSchemaRoot() { // verify results // both elements are equal, the result is equal - assertTrue(comparator.compare(5, 5) == 0); + assertEquals(0, comparator.compare(5, 5)); // the first element being equal, the second is smaller, and the result is smaller assertTrue(comparator.compare(1, 1) < 0); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java index 43c634b7647fb..c40854fb17410 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java @@ -65,6 +65,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; /** * Test cases for {@link DefaultVectorComparators}. @@ -258,7 +259,8 @@ public void testCompareUInt2() { vec.allocateNew(10); ValueVectorDataPopulator.setVector( - vec, null, (char) -2, (char) -1, (char) 0, (char) 1, (char) 2, (char) -2, null, + vec, null, (char) (Character.MAX_VALUE - 1), Character.MAX_VALUE, (char) 0, (char) 1, + (char) 2, (char) (Character.MAX_VALUE - 1), null, '\u7FFF', // value for the max 16-byte signed integer '\u8000' // value for the min 16-byte signed integer ); @@ -272,8 +274,8 @@ public void testCompareUInt2() { assertTrue(comparator.compare(1, 3) > 0); assertTrue(comparator.compare(2, 5) > 0); assertTrue(comparator.compare(4, 5) < 0); - assertTrue(comparator.compare(1, 6) == 0); - assertTrue(comparator.compare(0, 7) == 0); + Assertions.assertEquals(0, comparator.compare(1, 6)); + Assertions.assertEquals(0, comparator.compare(0, 7)); assertTrue(comparator.compare(8, 9) < 0); assertTrue(comparator.compare(4, 8) < 0); assertTrue(comparator.compare(5, 9) < 0); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java index ba2a341bf44a0..80c72b4e21a27 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java @@ -131,37 +131,37 @@ public static Collection getParameters() { for (boolean inPlace : new boolean[] {true, false}) { params.add(new Object[] { length, nullFrac, inPlace, "TinyIntVector", - (Function) (allocator -> new TinyIntVector("vector", allocator)), + (Function) allocator -> new TinyIntVector("vector", allocator), TestSortingUtil.TINY_INT_GENERATOR }); params.add(new Object[] { length, nullFrac, inPlace, "SmallIntVector", - (Function) (allocator -> new SmallIntVector("vector", allocator)), + (Function) allocator -> new SmallIntVector("vector", allocator), TestSortingUtil.SMALL_INT_GENERATOR }); params.add(new Object[] { length, nullFrac, inPlace, "IntVector", - (Function) (allocator -> new IntVector("vector", allocator)), + (Function) allocator -> new IntVector("vector", allocator), TestSortingUtil.INT_GENERATOR }); params.add(new Object[] { length, nullFrac, inPlace, "BigIntVector", - (Function) (allocator -> new BigIntVector("vector", allocator)), + (Function) allocator -> new BigIntVector("vector", allocator), TestSortingUtil.LONG_GENERATOR }); params.add(new Object[] { length, nullFrac, inPlace, "Float4Vector", - (Function) (allocator -> new Float4Vector("vector", allocator)), + (Function) allocator -> new Float4Vector("vector", allocator), TestSortingUtil.FLOAT_GENERATOR }); params.add(new Object[] { length, nullFrac, inPlace, "Float8Vector", - (Function) (allocator -> new Float8Vector("vector", allocator)), + (Function) allocator -> new Float8Vector("vector", allocator), TestSortingUtil.DOUBLE_GENERATOR }); } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java index ea86551061d56..e22b22d4e6757 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.lang.reflect.Array; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Random; import java.util.function.BiConsumer; @@ -122,7 +123,7 @@ static String generateRandomString(int length) { str[i] = (byte) (r % (upper - lower + 1) + lower); } - return new String(str); + return new String(str, StandardCharsets.UTF_8); } /** diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestStableVectorComparator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestStableVectorComparator.java index 07419359427f9..f2de5d23fce89 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestStableVectorComparator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestStableVectorComparator.java @@ -20,12 +20,16 @@ import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VarCharVector; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; /** * Test cases for {@link StableVectorComparator}. @@ -51,11 +55,11 @@ public void testCompare() { vec.setValueCount(10); // fill data to sort - vec.set(0, "ba".getBytes()); - vec.set(1, "abc".getBytes()); - vec.set(2, "aa".getBytes()); - vec.set(3, "abc".getBytes()); - vec.set(4, "a".getBytes()); + vec.set(0, "ba".getBytes(StandardCharsets.UTF_8)); + vec.set(1, "abc".getBytes(StandardCharsets.UTF_8)); + vec.set(2, "aa".getBytes(StandardCharsets.UTF_8)); + vec.set(3, "abc".getBytes(StandardCharsets.UTF_8)); + vec.set(4, "a".getBytes(StandardCharsets.UTF_8)); VectorValueComparator comparator = new TestVarCharSorter(); VectorValueComparator stableComparator = new StableVectorComparator<>(comparator); @@ -66,7 +70,7 @@ public void testCompare() { assertTrue(stableComparator.compare(2, 3) < 0); assertTrue(stableComparator.compare(1, 3) < 0); assertTrue(stableComparator.compare(3, 1) > 0); - assertTrue(stableComparator.compare(3, 3) == 0); + Assertions.assertEquals(0, stableComparator.compare(3, 3)); } } @@ -77,16 +81,16 @@ public void testStableSortString() { vec.setValueCount(10); // fill data to sort - vec.set(0, "a".getBytes()); - vec.set(1, "abc".getBytes()); - vec.set(2, "aa".getBytes()); - vec.set(3, "a1".getBytes()); - vec.set(4, "abcdefg".getBytes()); - vec.set(5, "accc".getBytes()); - vec.set(6, "afds".getBytes()); - vec.set(7, "0".getBytes()); - vec.set(8, "01".getBytes()); - vec.set(9, "0c".getBytes()); + vec.set(0, "a".getBytes(StandardCharsets.UTF_8)); + vec.set(1, "abc".getBytes(StandardCharsets.UTF_8)); + vec.set(2, "aa".getBytes(StandardCharsets.UTF_8)); + vec.set(3, "a1".getBytes(StandardCharsets.UTF_8)); + vec.set(4, "abcdefg".getBytes(StandardCharsets.UTF_8)); + vec.set(5, "accc".getBytes(StandardCharsets.UTF_8)); + vec.set(6, "afds".getBytes(StandardCharsets.UTF_8)); + vec.set(7, "0".getBytes(StandardCharsets.UTF_8)); + vec.set(8, "01".getBytes(StandardCharsets.UTF_8)); + vec.set(9, "0c".getBytes(StandardCharsets.UTF_8)); // sort the vector VariableWidthOutOfPlaceVectorSorter sorter = new VariableWidthOutOfPlaceVectorSorter(); @@ -103,16 +107,16 @@ public void testStableSortString() { // verify results // the results are stable - assertEquals("0", new String(sortedVec.get(0))); - assertEquals("01", new String(sortedVec.get(1))); - assertEquals("0c", new String(sortedVec.get(2))); - assertEquals("a", new String(sortedVec.get(3))); - assertEquals("abc", new String(sortedVec.get(4))); - assertEquals("aa", new String(sortedVec.get(5))); - assertEquals("a1", new String(sortedVec.get(6))); - assertEquals("abcdefg", new String(sortedVec.get(7))); - assertEquals("accc", new String(sortedVec.get(8))); - assertEquals("afds", new String(sortedVec.get(9))); + assertEquals("0", new String(Objects.requireNonNull(sortedVec.get(0)), StandardCharsets.UTF_8)); + assertEquals("01", new String(Objects.requireNonNull(sortedVec.get(1)), StandardCharsets.UTF_8)); + assertEquals("0c", new String(Objects.requireNonNull(sortedVec.get(2)), StandardCharsets.UTF_8)); + assertEquals("a", new String(Objects.requireNonNull(sortedVec.get(3)), StandardCharsets.UTF_8)); + assertEquals("abc", new String(Objects.requireNonNull(sortedVec.get(4)), StandardCharsets.UTF_8)); + assertEquals("aa", new String(Objects.requireNonNull(sortedVec.get(5)), StandardCharsets.UTF_8)); + assertEquals("a1", new String(Objects.requireNonNull(sortedVec.get(6)), StandardCharsets.UTF_8)); + assertEquals("abcdefg", new String(Objects.requireNonNull(sortedVec.get(7)), StandardCharsets.UTF_8)); + assertEquals("accc", new String(Objects.requireNonNull(sortedVec.get(8)), StandardCharsets.UTF_8)); + assertEquals("afds", new String(Objects.requireNonNull(sortedVec.get(9)), StandardCharsets.UTF_8)); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthOutOfPlaceVectorSorter.java index 8f4e3b8e19426..2486034f1fa32 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthOutOfPlaceVectorSorter.java @@ -20,6 +20,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BaseVariableWidthVector; @@ -62,16 +65,16 @@ public void testSortString() { vec.setValueCount(10); // fill data to sort - vec.set(0, "hello".getBytes()); - vec.set(1, "abc".getBytes()); + vec.set(0, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(1, "abc".getBytes(StandardCharsets.UTF_8)); vec.setNull(2); - vec.set(3, "world".getBytes()); - vec.set(4, "12".getBytes()); - vec.set(5, "dictionary".getBytes()); + vec.set(3, "world".getBytes(StandardCharsets.UTF_8)); + vec.set(4, "12".getBytes(StandardCharsets.UTF_8)); + vec.set(5, "dictionary".getBytes(StandardCharsets.UTF_8)); vec.setNull(6); - vec.set(7, "hello".getBytes()); - vec.set(8, "good".getBytes()); - vec.set(9, "yes".getBytes()); + vec.set(7, "hello".getBytes(StandardCharsets.UTF_8)); + vec.set(8, "good".getBytes(StandardCharsets.UTF_8)); + vec.set(9, "yes".getBytes(StandardCharsets.UTF_8)); // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); @@ -93,14 +96,14 @@ public void testSortString() { assertTrue(sortedVec.isNull(0)); assertTrue(sortedVec.isNull(1)); - assertEquals("12", new String(sortedVec.get(2))); - assertEquals("abc", new String(sortedVec.get(3))); - assertEquals("dictionary", new String(sortedVec.get(4))); - assertEquals("good", new String(sortedVec.get(5))); - assertEquals("hello", new String(sortedVec.get(6))); - assertEquals("hello", new String(sortedVec.get(7))); - assertEquals("world", new String(sortedVec.get(8))); - assertEquals("yes", new String(sortedVec.get(9))); + assertEquals("12", new String(Objects.requireNonNull(sortedVec.get(2)), StandardCharsets.UTF_8)); + assertEquals("abc", new String(Objects.requireNonNull(sortedVec.get(3)), StandardCharsets.UTF_8)); + assertEquals("dictionary", new String(Objects.requireNonNull(sortedVec.get(4)), StandardCharsets.UTF_8)); + assertEquals("good", new String(Objects.requireNonNull(sortedVec.get(5)), StandardCharsets.UTF_8)); + assertEquals("hello", new String(Objects.requireNonNull(sortedVec.get(6)), StandardCharsets.UTF_8)); + assertEquals("hello", new String(Objects.requireNonNull(sortedVec.get(7)), StandardCharsets.UTF_8)); + assertEquals("world", new String(Objects.requireNonNull(sortedVec.get(8)), StandardCharsets.UTF_8)); + assertEquals("yes", new String(Objects.requireNonNull(sortedVec.get(9)), StandardCharsets.UTF_8)); sortedVec.close(); } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java index 068fe8b69a883..7951c39d550d2 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -94,7 +95,7 @@ void sortOutOfPlace() { VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vector); try (V sortedVec = (V) vector.getField().getFieldType().createNewSingleVector("", allocator, null)) { - int dataSize = vector.getOffsetBuffer().getInt(vector.getValueCount() * 4); + int dataSize = vector.getOffsetBuffer().getInt(vector.getValueCount() * 4L); sortedVec.allocateNew(dataSize, vector.getValueCount()); sortedVec.setValueCount(vector.getValueCount()); @@ -113,7 +114,7 @@ public static Collection getParameters() { for (double nullFrac : NULL_FRACTIONS) { params.add(new Object[]{ length, nullFrac, "VarCharVector", - (Function) (allocator -> new VarCharVector("vector", allocator)), + (Function) allocator -> new VarCharVector("vector", allocator), TestSortingUtil.STRING_GENERATOR }); } @@ -130,7 +131,7 @@ public static void verifyResults(V vector, String[] expe if (expected[i] == null) { assertTrue(vector.isNull(i)); } else { - assertArrayEquals(((Text) vector.getObject(i)).getBytes(), expected[i].getBytes()); + assertArrayEquals(((Text) vector.getObject(i)).getBytes(), expected[i].getBytes(StandardCharsets.UTF_8)); } } } @@ -151,8 +152,8 @@ public int compare(String str1, String str2) { return str1 == null ? -1 : 1; } - byte[] bytes1 = str1.getBytes(); - byte[] bytes2 = str2.getBytes(); + byte[] bytes1 = str1.getBytes(StandardCharsets.UTF_8); + byte[] bytes2 = str2.getBytes(StandardCharsets.UTF_8); for (int i = 0; i < bytes1.length && i < bytes2.length; i++) { if (bytes1[i] != bytes2[i]) { diff --git a/java/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java b/java/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java index 01156fa2b0e0b..5fff4fafd677e 100644 --- a/java/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java +++ b/java/compression/src/test/java/org/apache/arrow/compression/TestCompressionCodec.java @@ -175,7 +175,7 @@ void testCompressVariableWidthBuffers(int vectorLength, CompressionCodec codec) if (i % 10 == 0) { origVec.setNull(i); } else { - origVec.setSafe(i, String.valueOf(i).getBytes()); + origVec.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } } origVec.setValueCount(vectorLength); @@ -199,7 +199,7 @@ void testCompressVariableWidthBuffers(int vectorLength, CompressionCodec codec) if (i % 10 == 0) { assertTrue(newVec.isNull(i)); } else { - assertArrayEquals(String.valueOf(i).getBytes(), newVec.get(i)); + assertArrayEquals(String.valueOf(i).getBytes(StandardCharsets.UTF_8), newVec.get(i)); } } diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index b7624d7748e7f..0346172f610a6 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -86,6 +86,10 @@ com.google.protobuf protobuf-java + + com.google.protobuf + protobuf-java-util + io.grpc grpc-api diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java index b4ee835dee4a0..46cb282e9f3ce 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ArrowMessage.java @@ -35,8 +35,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.util.Preconditions; -import org.apache.arrow.vector.compression.NoCompressionCodec; -import org.apache.arrow.vector.ipc.message.ArrowBodyCompression; import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; import org.apache.arrow.vector.ipc.message.IpcOption; @@ -144,7 +142,6 @@ public static HeaderType getHeader(byte b) { private final MessageMetadataResult message; private final ArrowBuf appMetadata; private final List bufs; - private final ArrowBodyCompression bodyCompression; private final boolean tryZeroCopyWrite; public ArrowMessage(FlightDescriptor descriptor, Schema schema, IpcOption option) { @@ -155,7 +152,6 @@ public ArrowMessage(FlightDescriptor descriptor, Schema schema, IpcOption option bufs = ImmutableList.of(); this.descriptor = descriptor; this.appMetadata = null; - this.bodyCompression = NoCompressionCodec.DEFAULT_BODY_COMPRESSION; this.tryZeroCopyWrite = false; } @@ -172,7 +168,6 @@ public ArrowMessage(ArrowRecordBatch batch, ArrowBuf appMetadata, boolean tryZer this.bufs = ImmutableList.copyOf(batch.getBuffers()); this.descriptor = null; this.appMetadata = appMetadata; - this.bodyCompression = batch.getBodyCompression(); this.tryZeroCopyWrite = tryZeroCopy; } @@ -186,7 +181,6 @@ public ArrowMessage(ArrowDictionaryBatch batch, IpcOption option) { this.bufs = ImmutableList.copyOf(batch.getDictionary().getBuffers()); this.descriptor = null; this.appMetadata = null; - this.bodyCompression = batch.getDictionary().getBodyCompression(); this.tryZeroCopyWrite = false; } @@ -201,7 +195,6 @@ public ArrowMessage(ArrowBuf appMetadata) { this.bufs = ImmutableList.of(); this.descriptor = null; this.appMetadata = appMetadata; - this.bodyCompression = NoCompressionCodec.DEFAULT_BODY_COMPRESSION; this.tryZeroCopyWrite = false; } @@ -212,7 +205,6 @@ public ArrowMessage(FlightDescriptor descriptor) { this.bufs = ImmutableList.of(); this.descriptor = descriptor; this.appMetadata = null; - this.bodyCompression = NoCompressionCodec.DEFAULT_BODY_COMPRESSION; this.tryZeroCopyWrite = false; } @@ -227,7 +219,6 @@ private ArrowMessage(FlightDescriptor descriptor, MessageMetadataResult message, this.descriptor = descriptor; this.appMetadata = appMetadata; this.bufs = buf == null ? ImmutableList.of() : ImmutableList.of(buf); - this.bodyCompression = NoCompressionCodec.DEFAULT_BODY_COMPRESSION; this.tryZeroCopyWrite = false; } @@ -370,7 +361,7 @@ private static int readRawVarint32(InputStream is) throws IOException { * * @return InputStream */ - private InputStream asInputStream(BufferAllocator allocator) { + private InputStream asInputStream() { if (message == null) { // If we have no IPC message, it's a pure-metadata message final FlightData.Builder builder = FlightData.newBuilder(); @@ -422,7 +413,7 @@ private InputStream asInputStream(BufferAllocator allocator) { // Arrow buffer. This is susceptible to use-after-free, so we subclass CompositeByteBuf // below to tie the Arrow buffer refcnt to the Netty buffer refcnt allBufs.add(Unpooled.wrappedBuffer(b.nioBuffer()).retain()); - size += b.readableBytes(); + size += (int) b.readableBytes(); // [ARROW-4213] These buffers must be aligned to an 8-byte boundary in order to be readable from C++. if (b.readableBytes() % 8 != 0) { int paddingBytes = (int) (8 - (b.readableBytes() % 8)); @@ -543,7 +534,7 @@ public ArrowMessageHolderMarshaller(BufferAllocator allocator) { @Override public InputStream stream(ArrowMessage value) { - return value.asInputStream(allocator); + return value.asInputStream(); } @Override diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CancelFlightInfoResult.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CancelFlightInfoResult.java index eff5afdeeb788..165afdff553df 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CancelFlightInfoResult.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CancelFlightInfoResult.java @@ -105,7 +105,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof CancelFlightInfoResult)) { return false; } CancelFlightInfoResult that = (CancelFlightInfoResult) o; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ErrorFlightMetadata.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ErrorFlightMetadata.java index 6669ce4655010..6e19d2750cb67 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ErrorFlightMetadata.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ErrorFlightMetadata.java @@ -61,7 +61,7 @@ public Iterable getAllByte(String key) { @Override public void insert(String key, String value) { - metadata.put(key, value.getBytes()); + metadata.put(key, value.getBytes(StandardCharsets.UTF_8)); } @Override diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightCallHeaders.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightCallHeaders.java index dd26d190872ac..93b89e775507e 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightCallHeaders.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightCallHeaders.java @@ -17,6 +17,7 @@ package org.apache.arrow.flight; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Set; import java.util.stream.Collectors; @@ -46,7 +47,7 @@ public String get(String key) { } if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { - return new String((byte[]) Iterables.get(values, 0)); + return new String((byte[]) Iterables.get(values, 0), StandardCharsets.UTF_8); } return (String) Iterables.get(values, 0); @@ -63,13 +64,14 @@ public byte[] getByte(String key) { return (byte[]) Iterables.get(values, 0); } - return ((String) Iterables.get(values, 0)).getBytes(); + return ((String) Iterables.get(values, 0)).getBytes(StandardCharsets.UTF_8); } @Override public Iterable getAll(String key) { if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { - return this.keysAndValues.get(key).stream().map(o -> new String((byte[]) o)).collect(Collectors.toList()); + return this.keysAndValues.get(key).stream().map(o -> new String((byte[]) o, StandardCharsets.UTF_8)) + .collect(Collectors.toList()); } return (Collection) (Collection) this.keysAndValues.get(key); } @@ -79,7 +81,8 @@ public Iterable getAllByte(String key) { if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { return (Collection) (Collection) this.keysAndValues.get(key); } - return this.keysAndValues.get(key).stream().map(o -> ((String) o).getBytes()).collect(Collectors.toList()); + return this.keysAndValues.get(key).stream().map(o -> ((String) o).getBytes(StandardCharsets.UTF_8)) + .collect(Collectors.toList()); } @Override @@ -105,6 +108,7 @@ public boolean containsKey(String key) { return this.keysAndValues.containsKey(key); } + @Override public String toString() { return this.keysAndValues.toString(); } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java index 91e3b4d052f39..fc491ebe0df98 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightClient.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -627,6 +628,7 @@ default boolean isCancelled() { /** * Shut down this client. */ + @Override public void close() throws InterruptedException { channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); allocator.close(); @@ -746,19 +748,24 @@ public FlightClient build() { try { // Linux builder.channelType( - (Class) Class.forName("io.netty.channel.epoll.EpollDomainSocketChannel")); - final EventLoopGroup elg = (EventLoopGroup) Class.forName("io.netty.channel.epoll.EpollEventLoopGroup") - .newInstance(); + Class.forName("io.netty.channel.epoll.EpollDomainSocketChannel") + .asSubclass(ServerChannel.class)); + final EventLoopGroup elg = + Class.forName("io.netty.channel.epoll.EpollEventLoopGroup").asSubclass(EventLoopGroup.class) + .getDeclaredConstructor().newInstance(); builder.eventLoopGroup(elg); } catch (ClassNotFoundException e) { // BSD builder.channelType( - (Class) Class.forName("io.netty.channel.kqueue.KQueueDomainSocketChannel")); - final EventLoopGroup elg = (EventLoopGroup) Class.forName("io.netty.channel.kqueue.KQueueEventLoopGroup") - .newInstance(); + Class.forName("io.netty.channel.kqueue.KQueueDomainSocketChannel") + .asSubclass(ServerChannel.class)); + final EventLoopGroup elg = Class.forName("io.netty.channel.kqueue.KQueueEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getDeclaredConstructor().newInstance(); builder.eventLoopGroup(elg); } - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | + NoSuchMethodException | InvocationTargetException e) { throw new UnsupportedOperationException( "Could not find suitable Netty native transport implementation for domain socket address."); } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightDescriptor.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightDescriptor.java index 3eff011d9fe77..1836f2edd94c0 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightDescriptor.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightDescriptor.java @@ -152,7 +152,7 @@ public boolean equals(Object obj) { if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (!(obj instanceof FlightDescriptor)) { return false; } FlightDescriptor other = (FlightDescriptor) obj; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightEndpoint.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightEndpoint.java index 1967fe1d91c34..41ead8e1fcddf 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightEndpoint.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightEndpoint.java @@ -33,6 +33,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; /** * POJO to convert to/from the underlying protobuf FlightEndpoint. @@ -85,11 +86,11 @@ private FlightEndpoint(Ticket ticket, Instant expirationTime, byte[] appMetadata } if (flt.hasExpirationTime()) { this.expirationTime = Instant.ofEpochSecond( - flt.getExpirationTime().getSeconds(), flt.getExpirationTime().getNanos()); + flt.getExpirationTime().getSeconds(), Timestamps.toNanos(flt.getExpirationTime())); } else { this.expirationTime = null; } - this.appMetadata = (flt.getAppMetadata().size() == 0 ? null : flt.getAppMetadata().toByteArray()); + this.appMetadata = (flt.getAppMetadata().isEmpty() ? null : flt.getAppMetadata().toByteArray()); this.ticket = new Ticket(flt.getTicket()); } @@ -163,7 +164,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof FlightEndpoint)) { return false; } FlightEndpoint that = (FlightEndpoint) o; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightInfo.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightInfo.java index b5279a304c865..39e5f5e3a3ed6 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightInfo.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightInfo.java @@ -249,7 +249,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof FlightInfo)) { return false; } FlightInfo that = (FlightInfo) o; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java index 234c9bdcaacc1..d873f7d2828d0 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightServer.java @@ -21,6 +21,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -134,6 +135,7 @@ public boolean awaitTermination(final long timeout, final TimeUnit unit) throws } /** Shutdown the server, waits for up to 6 seconds for successful shutdown before returning. */ + @Override public void close() throws InterruptedException { shutdown(); final boolean terminated = awaitTermination(3000, TimeUnit.MILLISECONDS); @@ -146,7 +148,7 @@ public void close() throws InterruptedException { server.shutdownNow(); int count = 0; - while (!server.isTerminated() & count < 30) { + while (!server.isTerminated() && count < 30) { count++; logger.debug("Waiting for termination"); Thread.sleep(100); @@ -216,22 +218,23 @@ public FlightServer build() { try { try { // Linux - builder.channelType( - (Class) Class - .forName("io.netty.channel.epoll.EpollServerDomainSocketChannel")); - final EventLoopGroup elg = (EventLoopGroup) Class.forName("io.netty.channel.epoll.EpollEventLoopGroup") - .newInstance(); + builder.channelType(Class + .forName("io.netty.channel.epoll.EpollServerDomainSocketChannel") + .asSubclass(ServerChannel.class)); + final EventLoopGroup elg = Class.forName("io.netty.channel.epoll.EpollEventLoopGroup") + .asSubclass(EventLoopGroup.class).getConstructor().newInstance(); builder.bossEventLoopGroup(elg).workerEventLoopGroup(elg); } catch (ClassNotFoundException e) { // BSD builder.channelType( - (Class) Class - .forName("io.netty.channel.kqueue.KQueueServerDomainSocketChannel")); - final EventLoopGroup elg = (EventLoopGroup) Class.forName("io.netty.channel.kqueue.KQueueEventLoopGroup") - .newInstance(); + Class.forName("io.netty.channel.kqueue.KQueueServerDomainSocketChannel") + .asSubclass(ServerChannel.class)); + final EventLoopGroup elg = Class.forName("io.netty.channel.kqueue.KQueueEventLoopGroup") + .asSubclass(EventLoopGroup.class).getConstructor().newInstance(); builder.bossEventLoopGroup(elg).workerEventLoopGroup(elg); } - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException | + InvocationTargetException e) { throw new UnsupportedOperationException( "Could not find suitable Netty native transport implementation for domain socket address."); } @@ -342,7 +345,8 @@ private void closeInputStreamIfNotNull(InputStream stream) { if (stream != null) { try { stream.close(); - } catch (IOException ignored) { + } catch (IOException expected) { + // stream closes gracefully, doesn't expect an exception. } } } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightService.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightService.java index 5231a7aaf76e4..f55b47d2a945b 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightService.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightService.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.Map; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.function.BooleanSupplier; import java.util.function.Consumer; @@ -142,7 +143,7 @@ public void listActions(Flight.Empty request, StreamObserver } private static class GetListener extends OutboundStreamListenerImpl implements ServerStreamListener { - private ServerCallStreamObserver responseObserver; + private final ServerCallStreamObserver serverCallResponseObserver; private final Consumer errorHandler; private Runnable onCancelHandler = null; private Runnable onReadyHandler = null; @@ -152,10 +153,10 @@ public GetListener(ServerCallStreamObserver responseObserver, Cons super(null, responseObserver); this.errorHandler = errorHandler; this.completed = false; - this.responseObserver = responseObserver; - this.responseObserver.setOnCancelHandler(this::onCancel); - this.responseObserver.setOnReadyHandler(this::onReady); - this.responseObserver.disableAutoInboundFlowControl(); + this.serverCallResponseObserver = responseObserver; + this.serverCallResponseObserver.setOnCancelHandler(this::onCancel); + this.serverCallResponseObserver.setOnReadyHandler(this::onReady); + this.serverCallResponseObserver.disableAutoInboundFlowControl(); } private void onCancel() { @@ -183,7 +184,7 @@ public void setOnReadyHandler(Runnable handler) { @Override public boolean isCancelled() { - return responseObserver.isCancelled(); + return serverCallResponseObserver.isCancelled(); } @Override @@ -228,7 +229,7 @@ public StreamObserver doPutCustom(final StreamObserver observer = fs.asObserver(); - executors.submit(() -> { + Future unused = executors.submit(() -> { try { producer.acceptPut(makeContext(responseObserver), fs, ackStream).run(); } catch (Throwable ex) { @@ -277,7 +278,8 @@ public void pollFlightInfo(Flight.FlightDescriptor request, StreamObserver, FlightServerMiddleware> middleware = ServerInterceptorAdapter.SERVER_MIDDLEWARE_KEY.get(); + final Map, FlightServerMiddleware> middleware = ServerInterceptorAdapter + .SERVER_MIDDLEWARE_KEY.get(); if (middleware == null || middleware.isEmpty()) { logger.error("Uncaught exception in Flight method body", t); return; @@ -377,7 +379,7 @@ public StreamObserver doExchangeCustom(StreamObserver observer = fs.asObserver(); try { - executors.submit(() -> { + Future unused = executors.submit(() -> { try { producer.doExchange(makeContext(responseObserver), fs, listener); } catch (Exception ex) { @@ -416,8 +418,9 @@ public boolean isCancelled() { } @Override - public T getMiddleware(Key key) { - final Map, FlightServerMiddleware> middleware = ServerInterceptorAdapter.SERVER_MIDDLEWARE_KEY.get(); + public T getMiddleware(FlightServerMiddleware.Key key) { + final Map, FlightServerMiddleware> middleware = ServerInterceptorAdapter + .SERVER_MIDDLEWARE_KEY.get(); if (middleware == null) { return null; } @@ -430,8 +433,9 @@ public T getMiddleware(Key key) { } @Override - public Map, FlightServerMiddleware> getMiddleware() { - final Map, FlightServerMiddleware> middleware = ServerInterceptorAdapter.SERVER_MIDDLEWARE_KEY.get(); + public Map, FlightServerMiddleware> getMiddleware() { + final Map, FlightServerMiddleware> middleware = + ServerInterceptorAdapter.SERVER_MIDDLEWARE_KEY.get(); if (middleware == null) { return Collections.emptyMap(); } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightStream.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightStream.java index ad4ffcbebdec1..7a5a941603ace 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightStream.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightStream.java @@ -27,6 +27,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.arrow.flight.ArrowMessage.HeaderType; import org.apache.arrow.flight.grpc.StatusUtils; @@ -56,9 +57,17 @@ */ public class FlightStream implements AutoCloseable { // Use AutoCloseable sentinel objects to simplify logic in #close - private final AutoCloseable DONE = () -> { + private final AutoCloseable DONE = new AutoCloseable() { + @Override + public void close() throws Exception { + + } }; - private final AutoCloseable DONE_EX = () -> { + private final AutoCloseable DONE_EX = new AutoCloseable() { + @Override + public void close() throws Exception { + + } }; private final BufferAllocator allocator; @@ -76,7 +85,7 @@ public class FlightStream implements AutoCloseable { // we don't block forever trying to write to a server that has rejected a call. final CompletableFuture cancelled; - private volatile int pending = 1; + private final AtomicInteger pending = new AtomicInteger(); private volatile VectorSchemaRoot fulfilledRoot; private DictionaryProvider.MapDictionaryProvider dictionaries; private volatile VectorLoader loader; @@ -169,6 +178,7 @@ public FlightDescriptor getDescriptor() { * *

If the stream isn't complete and is cancellable, this method will cancel and drain the stream first. */ + @Override public void close() throws Exception { final List closeables = new ArrayList<>(); Throwable suppressor = null; @@ -227,7 +237,7 @@ public boolean next() { return false; } - pending--; + pending.decrementAndGet(); requestOutstanding(); Object data = queue.take(); @@ -359,9 +369,9 @@ public ArrowBuf getLatestMetadata() { } private synchronized void requestOutstanding() { - if (pending < pendingTarget) { - requestor.request(pendingTarget - pending); - pending = pendingTarget; + if (pending.get() < pendingTarget) { + requestor.request(pendingTarget - pending.get()); + pending.set(pendingTarget); } } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/Location.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/Location.java index 9dba773bf3386..fe192aa0c3f9d 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/Location.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/Location.java @@ -71,7 +71,7 @@ public SocketAddress toSocketAddress() { case LocationSchemes.GRPC_DOMAIN_SOCKET: { try { // This dependency is not available on non-Unix platforms. - return (SocketAddress) Class.forName("io.netty.channel.unix.DomainSocketAddress") + return Class.forName("io.netty.channel.unix.DomainSocketAddress").asSubclass(SocketAddress.class) .getConstructor(String.class) .newInstance(uri.getPath()); } catch (InstantiationException | ClassNotFoundException | InvocationTargetException | @@ -144,7 +144,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof Location)) { return false; } Location location = (Location) o; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/PollInfo.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/PollInfo.java index 2bb3c6db69569..59150d8814cd9 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/PollInfo.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/PollInfo.java @@ -27,6 +27,7 @@ import org.apache.arrow.flight.impl.Flight; import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; /** * A POJO representation of the execution of a long-running query. @@ -57,7 +58,7 @@ public PollInfo(FlightInfo flightInfo, FlightDescriptor flightDescriptor, Double this.flightDescriptor = flt.hasFlightDescriptor() ? new FlightDescriptor(flt.getFlightDescriptor()) : null; this.progress = flt.hasProgress() ? flt.getProgress() : null; this.expirationTime = flt.hasExpirationTime() ? - Instant.ofEpochSecond(flt.getExpirationTime().getSeconds(), flt.getExpirationTime().getNanos()) : + Instant.ofEpochSecond(flt.getExpirationTime().getSeconds(), Timestamps.toNanos(flt.getExpirationTime())) : null; } @@ -133,7 +134,8 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + + if (!(o instanceof PollInfo)) { return false; } PollInfo pollInfo = (PollInfo) o; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/Ticket.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/Ticket.java index a93cd087905db..eb2f4af70d781 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/Ticket.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/Ticket.java @@ -88,7 +88,7 @@ public boolean equals(Object obj) { if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (!(obj instanceof Ticket)) { return false; } Ticket other = (Ticket) obj; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/AuthConstants.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/AuthConstants.java index ac55872e5b18b..e3ccdc626d71b 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/AuthConstants.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth/AuthConstants.java @@ -20,8 +20,8 @@ import org.apache.arrow.flight.FlightConstants; import io.grpc.Context; +import io.grpc.Metadata; import io.grpc.Metadata.BinaryMarshaller; -import io.grpc.Metadata.Key; import io.grpc.MethodDescriptor; /** @@ -32,7 +32,7 @@ public final class AuthConstants { public static final String HANDSHAKE_DESCRIPTOR_NAME = MethodDescriptor .generateFullMethodName(FlightConstants.SERVICE, "Handshake"); public static final String TOKEN_NAME = "Auth-Token-bin"; - public static final Key TOKEN_KEY = Key.of(TOKEN_NAME, new BinaryMarshaller() { + public static final Metadata.Key TOKEN_KEY = Metadata.Key.of(TOKEN_NAME, new BinaryMarshaller() { @Override public byte[] toBytes(byte[] value) { diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerTokenAuthenticator.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerTokenAuthenticator.java index 2006e0a2b1241..5eb5863e792d4 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerTokenAuthenticator.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/auth2/BearerTokenAuthenticator.java @@ -55,7 +55,6 @@ public AuthResult authenticate(CallHeaders incomingHeaders) { * Validate the bearer token. * @param bearerToken The bearer token to validate. * @return A successful AuthResult if validation succeeded. - * @throws Exception If the token validation fails. */ protected abstract AuthResult validateBearer(String bearerToken); diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ClientInterceptorAdapter.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ClientInterceptorAdapter.java index ae11e52605623..db27aa481ec75 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ClientInterceptorAdapter.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ClientInterceptorAdapter.java @@ -23,7 +23,6 @@ import org.apache.arrow.flight.CallInfo; import org.apache.arrow.flight.CallStatus; import org.apache.arrow.flight.FlightClientMiddleware; -import org.apache.arrow.flight.FlightClientMiddleware.Factory; import org.apache.arrow.flight.FlightMethod; import org.apache.arrow.flight.FlightRuntimeException; import org.apache.arrow.flight.FlightStatusCode; @@ -46,9 +45,9 @@ */ public class ClientInterceptorAdapter implements ClientInterceptor { - private final List factories; + private final List factories; - public ClientInterceptorAdapter(List factories) { + public ClientInterceptorAdapter(List factories) { this.factories = factories; } @@ -59,7 +58,7 @@ public ClientCall interceptCall(MethodDescriptor getAll(String key) { - return this.metadata.getAll(Key.of(key, Metadata.ASCII_STRING_MARSHALLER)); + return this.metadata.getAll(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER)); } @Override public Iterable getAllByte(String key) { if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { - return this.metadata.getAll(Key.of(key, Metadata.BINARY_BYTE_MARSHALLER)); + return this.metadata.getAll(Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER)); } return StreamSupport.stream(getAll(key).spliterator(), false) .map(String::getBytes).collect(Collectors.toList()); @@ -69,12 +70,12 @@ public Iterable getAllByte(String key) { @Override public void insert(String key, String value) { - this.metadata.put(Key.of(key, Metadata.ASCII_STRING_MARSHALLER), value); + this.metadata.put(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER), value); } @Override public void insert(String key, byte[] value) { - this.metadata.put(Key.of(key, Metadata.BINARY_BYTE_MARSHALLER), value); + this.metadata.put(Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER), value); } @Override @@ -85,13 +86,14 @@ public Set keys() { @Override public boolean containsKey(String key) { if (key.endsWith("-bin")) { - final Key grpcKey = Key.of(key, Metadata.BINARY_BYTE_MARSHALLER); + final Metadata.Key grpcKey = Metadata.Key.of(key, Metadata.BINARY_BYTE_MARSHALLER); return this.metadata.containsKey(grpcKey); } - final Key grpcKey = Key.of(key, Metadata.ASCII_STRING_MARSHALLER); + final Metadata.Key grpcKey = Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER); return this.metadata.containsKey(grpcKey); } + @Override public String toString() { return this.metadata.toString(); } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ServerInterceptorAdapter.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ServerInterceptorAdapter.java index 9b038b9d49272..70c667df56020 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ServerInterceptorAdapter.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/ServerInterceptorAdapter.java @@ -61,7 +61,7 @@ public static class KeyFactory { private final FlightServerMiddleware.Key key; private final FlightServerMiddleware.Factory factory; - public KeyFactory(Key key, Factory factory) { + public KeyFactory(FlightServerMiddleware.Key key, FlightServerMiddleware.Factory factory) { this.key = key; this.factory = factory; } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/StatusUtils.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/StatusUtils.java index 55e8418642d36..7f0dcf2da3f0d 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/StatusUtils.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/StatusUtils.java @@ -17,6 +17,7 @@ package org.apache.arrow.flight.grpc; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.Objects; import java.util.function.Function; @@ -171,7 +172,7 @@ private static ErrorFlightMetadata parseTrailers(Metadata trailers) { if (key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { metadata.insert(key, trailers.get(keyOfBinary(key))); } else { - metadata.insert(key, Objects.requireNonNull(trailers.get(keyOfAscii(key))).getBytes()); + metadata.insert(key, Objects.requireNonNull(trailers.get(keyOfAscii(key))).getBytes(StandardCharsets.UTF_8)); } } return metadata; diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java index 11510dbd32058..393fa086775ed 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java @@ -23,7 +23,6 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.List; -import java.util.Random; import org.apache.arrow.vector.test.util.ArrowTestDataUtil; import org.junit.jupiter.api.Assertions; @@ -34,8 +33,6 @@ */ public class FlightTestUtil { - private static final Random RANDOM = new Random(); - public static final String LOCALHOST = "localhost"; static Path getFlightTestDataRoot() { diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestApplicationMetadata.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestApplicationMetadata.java index 07db301309e3d..77c039afd87a0 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestApplicationMetadata.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestApplicationMetadata.java @@ -20,6 +20,7 @@ import static org.apache.arrow.flight.FlightTestUtil.LOCALHOST; import static org.apache.arrow.flight.Location.forGrpcInsecure; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.concurrent.ExecutionException; @@ -45,7 +46,7 @@ public class TestApplicationMetadata { // The command used to trigger the test for ARROW-6136. - private static final byte[] COMMAND_ARROW_6136 = "ARROW-6136".getBytes(); + private static final byte[] COMMAND_ARROW_6136 = "ARROW-6136".getBytes(StandardCharsets.UTF_8); // The expected error message. private static final String MESSAGE_ARROW_6136 = "The stream should not be double-closed."; diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBackPressure.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBackPressure.java index 7586d50c8e713..596debcf89dd2 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBackPressure.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBackPressure.java @@ -22,6 +22,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; @@ -153,7 +154,7 @@ public void getStream(CallContext context, Ticket ticket, ServerStreamListener l loadData.run(); } else { final ExecutorService service = Executors.newSingleThreadExecutor(); - service.submit(loadData); + Future unused = service.submit(loadData); service.shutdown(); } } @@ -237,7 +238,8 @@ public WaitResult waitForListener(long timeout) { try { Thread.sleep(1); sleepTime.addAndGet(1L); - } catch (InterruptedException ignore) { + } catch (InterruptedException expected) { + // it is expected and no action needed } } return WaitResult.READY; diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java index 41b3a4693e579..ae520ee9b991b 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestBasicOperation.java @@ -17,6 +17,7 @@ package org.apache.arrow.flight; + import static org.apache.arrow.flight.FlightTestUtil.LOCALHOST; import static org.apache.arrow.flight.Location.forGrpcInsecure; @@ -114,11 +115,11 @@ public void roundTripInfo() throws Exception { Field.nullable("b", new ArrowType.FixedSizeBinary(32)) ), metadata); final FlightInfo info1 = FlightInfo.builder(schema, FlightDescriptor.path(), Collections.emptyList()) - .setAppMetadata("foo".getBytes()).build(); + .setAppMetadata("foo".getBytes(StandardCharsets.UTF_8)).build(); final FlightInfo info2 = new FlightInfo(schema, FlightDescriptor.command(new byte[2]), Collections.singletonList( FlightEndpoint.builder(new Ticket(new byte[10]), Location.forGrpcDomainSocket("/tmp/test.sock")) - .setAppMetadata("bar".getBytes()).build() + .setAppMetadata("bar".getBytes(StandardCharsets.UTF_8)).build() ), 200, 500); final FlightInfo info3 = new FlightInfo(schema, FlightDescriptor.path("a", "b"), Arrays.asList(new FlightEndpoint( @@ -160,7 +161,7 @@ public void roundTripDescriptor() throws Exception { public void getDescriptors() throws Exception { test(c -> { int count = 0; - for (FlightInfo i : c.listFlights(Criteria.ALL)) { + for (FlightInfo unused : c.listFlights(Criteria.ALL)) { count += 1; } Assertions.assertEquals(1, count); @@ -171,7 +172,8 @@ public void getDescriptors() throws Exception { public void getDescriptorsWithCriteria() throws Exception { test(c -> { int count = 0; - for (FlightInfo i : c.listFlights(new Criteria(new byte[]{1}))) { + for (FlightInfo unused : c.listFlights(new Criteria(new byte[]{1}))) { + count += 1; } Assertions.assertEquals(0, count); diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java index 8b1a897467d58..41df36c863325 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestCallOptions.java @@ -21,6 +21,7 @@ import static org.apache.arrow.flight.Location.forGrpcInsecure; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.Iterator; @@ -87,8 +88,8 @@ public void multipleProperties() { @Test public void binaryProperties() { final FlightCallHeaders headers = new FlightCallHeaders(); - headers.insert("key-bin", "value".getBytes()); - headers.insert("key3-bin", "ëfßæ".getBytes()); + headers.insert("key-bin", "value".getBytes(StandardCharsets.UTF_8)); + headers.insert("key3-bin", "ëfßæ".getBytes(StandardCharsets.UTF_8)); testHeaders(headers); } @@ -96,7 +97,7 @@ public void binaryProperties() { public void mixedProperties() { final FlightCallHeaders headers = new FlightCallHeaders(); headers.insert("key", "value"); - headers.insert("key3-bin", "ëfßæ".getBytes()); + headers.insert("key3-bin", "ëfßæ".getBytes(StandardCharsets.UTF_8)); testHeaders(headers); } diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDictionaryUtils.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDictionaryUtils.java index b3a716ab3cec5..40930131e0ca8 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDictionaryUtils.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDictionaryUtils.java @@ -18,7 +18,8 @@ package org.apache.arrow.flight; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; import java.util.TreeSet; @@ -54,7 +55,7 @@ public void testReuseSchema() { Schema newSchema = DictionaryUtils.generateSchema(schema, null, new TreeSet<>()); // assert that no new schema is created. - assertTrue(schema == newSchema); + assertSame(schema, newSchema); } @Test @@ -78,7 +79,7 @@ public void testCreateSchema() { Schema newSchema = DictionaryUtils.generateSchema(schema, dictProvider, dictionaryUsed); // assert that a new schema is created. - assertTrue(schema != newSchema); + assertNotSame(schema, newSchema); // assert the column is converted as expected ArrowType newColType = newSchema.getFields().get(0).getType(); diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDoExchange.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDoExchange.java index f9db9bfd23a88..b70353df8e9a7 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDoExchange.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestDoExchange.java @@ -476,7 +476,7 @@ public void doExchange(CallContext context, FlightStream reader, ServerStreamLis } /** Emulate DoGet. */ - private void doGet(CallContext context, FlightStream reader, ServerStreamListener writer) { + private void doGet(CallContext unusedContext, FlightStream unusedReader, ServerStreamListener writer) { try (VectorSchemaRoot root = VectorSchemaRoot.create(SCHEMA, allocator)) { writer.start(root); root.allocateNew(); @@ -493,7 +493,7 @@ private void doGet(CallContext context, FlightStream reader, ServerStreamListene } /** Emulate DoPut. */ - private void doPut(CallContext context, FlightStream reader, ServerStreamListener writer) { + private void doPut(CallContext unusedContext, FlightStream reader, ServerStreamListener writer) { int counter = 0; while (reader.next()) { if (!reader.hasRoot()) { @@ -510,7 +510,7 @@ private void doPut(CallContext context, FlightStream reader, ServerStreamListene } /** Exchange metadata without ever exchanging data. */ - private void metadataOnly(CallContext context, FlightStream reader, ServerStreamListener writer) { + private void metadataOnly(CallContext unusedContext, FlightStream reader, ServerStreamListener writer) { final ArrowBuf buf = allocator.buffer(4); buf.writeInt(42); writer.putMetadata(buf); @@ -522,7 +522,7 @@ private void metadataOnly(CallContext context, FlightStream reader, ServerStream } /** Echo the client's response back to it. */ - private void echo(CallContext context, FlightStream reader, ServerStreamListener writer) { + private void echo(CallContext unusedContext, FlightStream reader, ServerStreamListener writer) { VectorSchemaRoot root = null; VectorLoader loader = null; while (reader.next()) { @@ -555,7 +555,7 @@ private void echo(CallContext context, FlightStream reader, ServerStreamListener } /** Accept a set of messages, then return some result. */ - private void transform(CallContext context, FlightStream reader, ServerStreamListener writer) { + private void transform(CallContext unusedContext, FlightStream reader, ServerStreamListener writer) { final Schema schema = reader.getSchema(); for (final Field field : schema.getFields()) { if (!(field.getType() instanceof ArrowType.Int)) { @@ -597,11 +597,11 @@ private void transform(CallContext context, FlightStream reader, ServerStreamLis } /** Immediately cancel the call. */ - private void cancel(CallContext context, FlightStream reader, ServerStreamListener writer) { + private void cancel(CallContext unusedContext, FlightStream unusedReader, ServerStreamListener writer) { writer.error(CallStatus.CANCELLED.withDescription("expected").toRuntimeException()); } - private void error(CallContext context, FlightStream reader, ServerStreamListener writer) { + private void error(CallContext unusedContext, FlightStream reader, ServerStreamListener writer) { VectorSchemaRoot root = null; VectorLoader loader = null; while (reader.next()) { diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java index 1987d98196e9d..4ec7301466228 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestErrorMetadata.java @@ -20,6 +20,8 @@ import static org.apache.arrow.flight.FlightTestUtil.LOCALHOST; import static org.apache.arrow.flight.Location.forGrpcInsecure; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.flight.perf.impl.PerfOuterClass; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -53,7 +55,7 @@ public void testGrpcMetadata() throws Exception { .start(); final FlightClient client = FlightClient.builder(allocator, s.getLocation()).build()) { final CallStatus flightStatus = FlightTestUtil.assertCode(FlightStatusCode.CANCELLED, () -> { - FlightStream stream = client.getStream(new Ticket("abs".getBytes())); + FlightStream stream = client.getStream(new Ticket("abs".getBytes(StandardCharsets.UTF_8))); stream.next(); }); PerfOuterClass.Perf newPerf = null; diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java index 9010f2d4a98f0..2569d2ac2b384 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java @@ -176,7 +176,7 @@ public void testProxyChannelWithClosedChannel() throws IOException, InterruptedE /** * Private class used for testing purposes that overrides service behavior. */ - private class TestServiceAdapter extends TestServiceGrpc.TestServiceImplBase { + private static class TestServiceAdapter extends TestServiceGrpc.TestServiceImplBase { /** * gRPC service that receives an empty object & returns and empty protobuf object. diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java index 0e4669f29ce43..de1b7750da3bf 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java @@ -21,6 +21,7 @@ import static org.apache.arrow.flight.Location.forGrpcInsecure; import static org.junit.jupiter.api.Assertions.fail; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Optional; @@ -139,7 +140,7 @@ public void supportsNullSchemas() throws Exception public FlightInfo getFlightInfo(CallContext context, FlightDescriptor descriptor) { return new FlightInfo(null, descriptor, Collections.emptyList(), - 0, 0, false, IpcOption.DEFAULT, "foo".getBytes()); + 0, 0, false, IpcOption.DEFAULT, "foo".getBytes(StandardCharsets.UTF_8)); } }; @@ -149,7 +150,7 @@ public FlightInfo getFlightInfo(CallContext context, FlightInfo flightInfo = client.getInfo(FlightDescriptor.path("test")); Assertions.assertEquals(Optional.empty(), flightInfo.getSchemaOptional()); Assertions.assertEquals(new Schema(Collections.emptyList()), flightInfo.getSchema()); - Assertions.assertArrayEquals(flightInfo.getAppMetadata(), "foo".getBytes()); + Assertions.assertArrayEquals(flightInfo.getAppMetadata(), "foo".getBytes(StandardCharsets.UTF_8)); Exception e = Assertions.assertThrows( FlightRuntimeException.class, diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLargeMessage.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLargeMessage.java index 1feb6afcf8f05..430dc29a7d0c2 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLargeMessage.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestLargeMessage.java @@ -17,6 +17,7 @@ package org.apache.arrow.flight; +import static com.google.common.collect.ImmutableList.toImmutableList; import static org.apache.arrow.flight.FlightTestUtil.LOCALHOST; import static org.apache.arrow.flight.Location.forGrpcInsecure; @@ -89,7 +90,7 @@ private static VectorSchemaRoot generateData(BufferAllocator allocator) { final Stream fields = fieldNames .stream() .map(fieldName -> new Field(fieldName, FieldType.nullable(new ArrowType.Int(32, true)), null)); - final Schema schema = new Schema(fields::iterator, null); + final Schema schema = new Schema(fields.collect(toImmutableList()), null); final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); root.allocateNew(); diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestServerMiddleware.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestServerMiddleware.java index 0e19468d2b409..3bc8f2f90a612 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestServerMiddleware.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestServerMiddleware.java @@ -38,15 +38,13 @@ public class TestServerMiddleware { - private static final RuntimeException EXPECTED_EXCEPTION = new RuntimeException("test"); - /** * Make sure errors in DoPut are intercepted. */ @Test public void doPutErrors() { test( - new ErrorProducer(EXPECTED_EXCEPTION), + new ErrorProducer(new RuntimeException("test")), (allocator, client) -> { final FlightDescriptor descriptor = FlightDescriptor.path("test"); try (final VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(Collections.emptyList()), allocator)) { @@ -91,7 +89,7 @@ public void doPutCustomCode() { */ @Test public void doPutUncaught() { - test(new ServerErrorProducer(EXPECTED_EXCEPTION), + test(new ServerErrorProducer(new RuntimeException("test")), (allocator, client) -> { final FlightDescriptor descriptor = FlightDescriptor.path("test"); try (final VectorSchemaRoot root = VectorSchemaRoot.create(new Schema(Collections.emptyList()), allocator)) { @@ -106,13 +104,13 @@ public void doPutUncaught() { Assertions.assertEquals(FlightStatusCode.OK, status.code()); Assertions.assertNull(status.cause()); Assertions.assertNotNull(err); - Assertions.assertEquals(EXPECTED_EXCEPTION.getMessage(), err.getMessage()); + Assertions.assertEquals("test", err.getMessage()); }); } @Test public void listFlightsUncaught() { - test(new ServerErrorProducer(EXPECTED_EXCEPTION), + test(new ServerErrorProducer(new RuntimeException("test")), (allocator, client) -> client.listFlights(new Criteria(new byte[0])).forEach((action) -> { }), (recorder) -> { final CallStatus status = recorder.statusFuture.get(); @@ -121,13 +119,13 @@ public void listFlightsUncaught() { Assertions.assertEquals(FlightStatusCode.OK, status.code()); Assertions.assertNull(status.cause()); Assertions.assertNotNull(err); - Assertions.assertEquals(EXPECTED_EXCEPTION.getMessage(), err.getMessage()); + Assertions.assertEquals("test", err.getMessage()); }); } @Test public void doActionUncaught() { - test(new ServerErrorProducer(EXPECTED_EXCEPTION), + test(new ServerErrorProducer(new RuntimeException("test")), (allocator, client) -> client.doAction(new Action("test")).forEachRemaining(result -> { }), (recorder) -> { final CallStatus status = recorder.statusFuture.get(); @@ -136,13 +134,13 @@ public void doActionUncaught() { Assertions.assertEquals(FlightStatusCode.OK, status.code()); Assertions.assertNull(status.cause()); Assertions.assertNotNull(err); - Assertions.assertEquals(EXPECTED_EXCEPTION.getMessage(), err.getMessage()); + Assertions.assertEquals("test", err.getMessage()); }); } @Test public void listActionsUncaught() { - test(new ServerErrorProducer(EXPECTED_EXCEPTION), + test(new ServerErrorProducer(new RuntimeException("test")), (allocator, client) -> client.listActions().forEach(result -> { }), (recorder) -> { final CallStatus status = recorder.statusFuture.get(); @@ -151,13 +149,13 @@ public void listActionsUncaught() { Assertions.assertEquals(FlightStatusCode.OK, status.code()); Assertions.assertNull(status.cause()); Assertions.assertNotNull(err); - Assertions.assertEquals(EXPECTED_EXCEPTION.getMessage(), err.getMessage()); + Assertions.assertEquals("test", err.getMessage()); }); } @Test public void getFlightInfoUncaught() { - test(new ServerErrorProducer(EXPECTED_EXCEPTION), + test(new ServerErrorProducer(new RuntimeException("test")), (allocator, client) -> { FlightTestUtil.assertCode(FlightStatusCode.INTERNAL, () -> client.getInfo(FlightDescriptor.path("test"))); }, (recorder) -> { @@ -165,13 +163,13 @@ public void getFlightInfoUncaught() { Assertions.assertNotNull(status); Assertions.assertEquals(FlightStatusCode.INTERNAL, status.code()); Assertions.assertNotNull(status.cause()); - Assertions.assertEquals(EXPECTED_EXCEPTION.getMessage(), status.cause().getMessage()); + Assertions.assertEquals(new RuntimeException("test").getMessage(), status.cause().getMessage()); }); } @Test public void doGetUncaught() { - test(new ServerErrorProducer(EXPECTED_EXCEPTION), + test(new ServerErrorProducer(new RuntimeException("test")), (allocator, client) -> { try (final FlightStream stream = client.getStream(new Ticket(new byte[0]))) { while (stream.next()) { @@ -186,7 +184,7 @@ public void doGetUncaught() { Assertions.assertEquals(FlightStatusCode.OK, status.code()); Assertions.assertNull(status.cause()); Assertions.assertNotNull(err); - Assertions.assertEquals(EXPECTED_EXCEPTION.getMessage(), err.getMessage()); + Assertions.assertEquals("test", err.getMessage()); }); } @@ -305,7 +303,7 @@ static class ServerMiddlewarePair { final FlightServerMiddleware.Key key; final FlightServerMiddleware.Factory factory; - ServerMiddlewarePair(Key key, Factory factory) { + ServerMiddlewarePair(FlightServerMiddleware.Key key, FlightServerMiddleware.Factory factory) { this.key = key; this.factory = factory; } @@ -339,7 +337,7 @@ static void test(FlightProducer producer, BiConsumer verify) { final ErrorRecorder.Factory factory = new ErrorRecorder.Factory(); final List> middleware = Collections - .singletonList(new ServerMiddlewarePair<>(Key.of("m"), factory)); + .singletonList(new ServerMiddlewarePair<>(FlightServerMiddleware.Key.of("m"), factory)); test(producer, middleware, (allocator, client) -> { body.accept(allocator, client); try { diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestTls.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestTls.java index 0f6697a8e519c..75bc5f6e61589 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestTls.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestTls.java @@ -27,7 +27,6 @@ import java.util.Iterator; import java.util.function.Consumer; -import org.apache.arrow.flight.FlightClient.Builder; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.junit.jupiter.api.Assertions; @@ -105,7 +104,7 @@ public void connectTlsDisableServerVerification() { }); } - void test(Consumer testFn) { + void test(Consumer testFn) { final FlightTestUtil.CertKeyPair certKey = FlightTestUtil.exampleTlsCerts().get(0); try ( BufferAllocator a = new RootAllocator(Long.MAX_VALUE); @@ -113,7 +112,8 @@ void test(Consumer testFn) { FlightServer s = FlightServer.builder(a, forGrpcInsecure(LOCALHOST, 0), producer) .useTls(certKey.cert, certKey.key) .build().start()) { - final Builder builder = FlightClient.builder(a, Location.forGrpcTls(FlightTestUtil.LOCALHOST, s.getPort())); + final FlightClient.Builder builder = FlightClient.builder(a, Location.forGrpcTls(FlightTestUtil.LOCALHOST, + s.getPort())); testFn.accept(builder); } catch (InterruptedException | IOException e) { throw new RuntimeException(e); diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/client/TestCookieHandling.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/client/TestCookieHandling.java index 4b8a11870dab6..d34a3a2d3a51e 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/client/TestCookieHandling.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/client/TestCookieHandling.java @@ -251,6 +251,7 @@ public ClientCookieMiddleware onCallStarted(CallInfo info) { private void startServerAndClient() throws IOException { final FlightProducer flightProducer = new NoOpFlightProducer() { + @Override public void listFlights(CallContext context, Criteria criteria, StreamListener listener) { listener.onCompleted(); diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java index 0ded2f7065f9c..b1e83ea61ed53 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.apache.arrow.flight.BackpressureStrategy; import org.apache.arrow.flight.FlightDescriptor; @@ -48,10 +49,7 @@ public class PerformanceTestServer implements AutoCloseable { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PerformanceTestServer.class); - private final FlightServer flightServer; - private final Location location; private final BufferAllocator allocator; private final PerfProducer producer; private final boolean isNonBlocking; @@ -78,7 +76,6 @@ public WaitResult waitForListener(long timeout) { public PerformanceTestServer(BufferAllocator incomingAllocator, Location location, BackpressureStrategy bpStrategy, boolean isNonBlocking) { this.allocator = incomingAllocator.newChildAllocator("perf-server", 0, Long.MAX_VALUE); - this.location = location; this.producer = new PerfProducer(bpStrategy); this.flightServer = FlightServer.builder(this.allocator, location, producer).build(); this.isNonBlocking = isNonBlocking; @@ -110,16 +107,18 @@ public void getStream(CallContext context, Ticket ticket, ServerStreamListener listener) { bpStrategy.register(listener); final Runnable loadData = () -> { - VectorSchemaRoot root = null; + Token token = null; try { - Token token = Token.parseFrom(ticket.getBytes()); - Perf perf = token.getDefinition(); - Schema schema = Schema.deserializeMessage(perf.getSchema().asReadOnlyByteBuffer()); - root = VectorSchemaRoot.create(schema, allocator); - BigIntVector a = (BigIntVector) root.getVector("a"); - BigIntVector b = (BigIntVector) root.getVector("b"); - BigIntVector c = (BigIntVector) root.getVector("c"); - BigIntVector d = (BigIntVector) root.getVector("d"); + token = Token.parseFrom(ticket.getBytes()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + Perf perf = token.getDefinition(); + Schema schema = Schema.deserializeMessage(perf.getSchema().asReadOnlyByteBuffer()); + try ( + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator); + BigIntVector a = (BigIntVector) root.getVector("a") + ) { listener.setUseZeroCopy(true); listener.start(root); root.allocateNew(); @@ -158,14 +157,6 @@ public void getStream(CallContext context, Ticket ticket, listener.putNext(); } listener.completed(); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } finally { - try { - AutoCloseables.close(root); - } catch (Exception e) { - throw new RuntimeException(e); - } } }; @@ -173,7 +164,7 @@ public void getStream(CallContext context, Ticket ticket, loadData.run(); } else { final ExecutorService service = Executors.newSingleThreadExecutor(); - service.submit(loadData); + Future unused = service.submit(loadData); service.shutdown(); } } diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/TestPerf.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/TestPerf.java index 17c83c205feb0..290e82de36c57 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/TestPerf.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/perf/TestPerf.java @@ -110,14 +110,14 @@ public void throughput() throws Exception { double seconds = r.nanos * 1.0d / 1000 / 1000 / 1000; throughPuts[i] = (r.bytes * 1.0d / 1024 / 1024) / seconds; - System.out.println(String.format( - "Transferred %d records totaling %s bytes at %f MiB/s. %f record/s. %f batch/s.", + System.out.printf( + "Transferred %d records totaling %s bytes at %f MiB/s. %f record/s. %f batch/s.%n", r.rows, r.bytes, throughPuts[i], (r.rows * 1.0d) / seconds, (r.batches * 1.0d) / seconds - )); + ); } } pool.shutdown(); @@ -126,11 +126,11 @@ public void throughput() throws Exception { double average = Arrays.stream(throughPuts).sum() / numRuns; double sqrSum = Arrays.stream(throughPuts).map(val -> val - average).map(val -> val * val).sum(); double stddev = Math.sqrt(sqrSum / numRuns); - System.out.println(String.format("Average throughput: %f MiB/s, standard deviation: %f MiB/s", - average, stddev)); + System.out.printf("Average throughput: %f MiB/s, standard deviation: %f MiB/s%n", + average, stddev); } - private final class Consumer implements Callable { + private static final class Consumer implements Callable { private final FlightClient client; private final Ticket ticket; @@ -157,7 +157,7 @@ public Result call() throws Exception { aSum += a.get(i); } } - r.bytes += rows * 32; + r.bytes += rows * 32L; r.rows += rows; r.aSum = aSum; r.batches++; @@ -173,7 +173,7 @@ public Result call() throws Exception { } - private final class Result { + private static final class Result { private long rows; private long aSum; private long bytes; diff --git a/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/IntegrationTestClient.java b/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/IntegrationTestClient.java index 2ea9874f3dec3..64b5882c0f50d 100644 --- a/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/IntegrationTestClient.java +++ b/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/IntegrationTestClient.java @@ -98,14 +98,14 @@ private void run(String[] args) throws Exception { Scenarios.getScenario(cmd.getOptionValue("scenario")).client(allocator, defaultLocation, client); } else { final String inputPath = cmd.getOptionValue("j"); - testStream(allocator, defaultLocation, client, inputPath); + testStream(allocator, client, inputPath); } } catch (InterruptedException e) { throw new RuntimeException(e); } } - private static void testStream(BufferAllocator allocator, Location server, FlightClient client, String inputPath) + private static void testStream(BufferAllocator allocator, FlightClient client, String inputPath) throws IOException { // 1. Read data from JSON and upload to server. FlightDescriptor descriptor = FlightDescriptor.path(inputPath); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java index aa1b460fc136a..183e3d5c7b055 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java @@ -43,6 +43,7 @@ import org.apache.calcite.avatica.Meta; import org.apache.calcite.avatica.UnregisteredDriver; + /** * JDBC driver for querying data from an Apache Arrow Flight server. */ @@ -99,6 +100,7 @@ protected String getFactoryClassName(final JdbcVersion jdbcVersion) { } @Override + @SuppressWarnings("StringSplitter") protected DriverVersion createDriverVersion() { if (version == null) { final InputStream flightProperties = this.getClass().getResourceAsStream("/properties/flight.properties"); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java index 382750914992f..d25f03ac27b48 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightMetaImpl.java @@ -255,11 +255,6 @@ private static final class StatementHandleKey { public final String connectionId; public final int id; - StatementHandleKey(String connectionId, int id) { - this.connectionId = connectionId; - this.id = id; - } - StatementHandleKey(StatementHandle statementHandle) { this.connectionId = statementHandle.connectionId; this.id = statementHandle.id; diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBaseIntVectorAccessor.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBaseIntVectorAccessor.java index aea9b75fa6c3f..8d2fe1cc70319 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBaseIntVectorAccessor.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBaseIntVectorAccessor.java @@ -46,59 +46,57 @@ public class ArrowFlightJdbcBaseIntVectorAccessor extends ArrowFlightJdbcAccesso private final MinorType type; private final boolean isUnsigned; - private final int bytesToAllocate; private final Getter getter; private final NumericHolder holder; public ArrowFlightJdbcBaseIntVectorAccessor(UInt1Vector vector, IntSupplier currentRowSupplier, ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { - this(vector, currentRowSupplier, true, UInt1Vector.TYPE_WIDTH, setCursorWasNull); + this(vector, currentRowSupplier, true, setCursorWasNull); } public ArrowFlightJdbcBaseIntVectorAccessor(UInt2Vector vector, IntSupplier currentRowSupplier, ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { - this(vector, currentRowSupplier, true, UInt2Vector.TYPE_WIDTH, setCursorWasNull); + this(vector, currentRowSupplier, true, setCursorWasNull); } public ArrowFlightJdbcBaseIntVectorAccessor(UInt4Vector vector, IntSupplier currentRowSupplier, ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { - this(vector, currentRowSupplier, true, UInt4Vector.TYPE_WIDTH, setCursorWasNull); + this(vector, currentRowSupplier, true, setCursorWasNull); } public ArrowFlightJdbcBaseIntVectorAccessor(UInt8Vector vector, IntSupplier currentRowSupplier, ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { - this(vector, currentRowSupplier, true, UInt8Vector.TYPE_WIDTH, setCursorWasNull); + this(vector, currentRowSupplier, true, setCursorWasNull); } public ArrowFlightJdbcBaseIntVectorAccessor(TinyIntVector vector, IntSupplier currentRowSupplier, ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { - this(vector, currentRowSupplier, false, TinyIntVector.TYPE_WIDTH, setCursorWasNull); + this(vector, currentRowSupplier, false, setCursorWasNull); } public ArrowFlightJdbcBaseIntVectorAccessor(SmallIntVector vector, IntSupplier currentRowSupplier, ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { - this(vector, currentRowSupplier, false, SmallIntVector.TYPE_WIDTH, setCursorWasNull); + this(vector, currentRowSupplier, false, setCursorWasNull); } public ArrowFlightJdbcBaseIntVectorAccessor(IntVector vector, IntSupplier currentRowSupplier, ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { - this(vector, currentRowSupplier, false, IntVector.TYPE_WIDTH, setCursorWasNull); + this(vector, currentRowSupplier, false, setCursorWasNull); } public ArrowFlightJdbcBaseIntVectorAccessor(BigIntVector vector, IntSupplier currentRowSupplier, ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { - this(vector, currentRowSupplier, false, BigIntVector.TYPE_WIDTH, setCursorWasNull); + this(vector, currentRowSupplier, false, setCursorWasNull); } private ArrowFlightJdbcBaseIntVectorAccessor(BaseIntVector vector, IntSupplier currentRowSupplier, - boolean isUnsigned, int bytesToAllocate, - ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { + boolean isUnsigned, + ArrowFlightJdbcAccessorFactory.WasNullConsumer setCursorWasNull) { super(currentRowSupplier, setCursorWasNull); this.type = vector.getMinorType(); this.holder = new NumericHolder(); this.getter = createGetter(vector); this.isUnsigned = isUnsigned; - this.bytesToAllocate = bytesToAllocate; } @Override diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBitVectorAccessor.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBitVectorAccessor.java index f55fd12f9a517..67d98c2e69847 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBitVectorAccessor.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBitVectorAccessor.java @@ -32,7 +32,6 @@ public class ArrowFlightJdbcBitVectorAccessor extends ArrowFlightJdbcAccessor { private final BitVector vector; private final NullableBitHolder holder; - private static final int BYTES_T0_ALLOCATE = 1; /** * Constructor for the BitVectorAccessor. diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java index 6237a8b58d68a..e95cf00bc7a21 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java @@ -33,6 +33,7 @@ import org.apache.calcite.avatica.ConnectionConfigImpl; import org.apache.calcite.avatica.ConnectionProperty; + /** * A {@link ConnectionConfig} for the {@link ArrowFlightConnection}. */ diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConnectionWrapper.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConnectionWrapper.java index 5ee43ce012e94..c28071490caa6 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConnectionWrapper.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConnectionWrapper.java @@ -248,13 +248,13 @@ public PreparedStatement prepareStatement(final String sqlQuery, final int autoG } @Override - public PreparedStatement prepareStatement(final String sqlQuery, final int... columnIndices) + public PreparedStatement prepareStatement(final String sqlQuery, final int[] columnIndices) throws SQLException { return realConnection.prepareStatement(sqlQuery, columnIndices); } @Override - public PreparedStatement prepareStatement(final String sqlQuery, final String... columnNames) + public PreparedStatement prepareStatement(final String sqlQuery, final String[] columnNames) throws SQLException { return realConnection.prepareStatement(sqlQuery, columnNames); } @@ -306,12 +306,12 @@ public Properties getClientInfo() throws SQLException { } @Override - public Array createArrayOf(final String typeName, final Object... elements) throws SQLException { + public Array createArrayOf(final String typeName, final Object[] elements) throws SQLException { return realConnection.createArrayOf(typeName, elements); } @Override - public Struct createStruct(final String typeName, final Object... attributes) + public Struct createStruct(final String typeName, final Object[] attributes) throws SQLException { return realConnection.createStruct(typeName, attributes); } diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java index b21b03340e9f9..843fe0cb89d9f 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ConvertUtils.java @@ -49,7 +49,6 @@ import org.apache.calcite.avatica.AvaticaParameter; import org.apache.calcite.avatica.ColumnMetaData; import org.apache.calcite.avatica.proto.Common; -import org.apache.calcite.avatica.proto.Common.ColumnMetaData.Builder; /** * Convert objects between Arrow and Avatica. @@ -71,7 +70,7 @@ public static List convertArrowFieldsToColumnMetaDataList(final final Field field = fields.get(index); final ArrowType fieldType = field.getType(); - final Builder builder = Common.ColumnMetaData.newBuilder() + final Common.ColumnMetaData.Builder builder = Common.ColumnMetaData.newBuilder() .setOrdinal(index) .setColumnName(field.getName()) .setLabel(field.getName()); @@ -90,10 +89,10 @@ public static List convertArrowFieldsToColumnMetaDataList(final /** * Set on Column MetaData Builder. * - * @param builder {@link Builder} + * @param builder {@link Common.ColumnMetaData.Builder} * @param metadataMap {@link Map} */ - public static void setOnColumnMetaDataBuilder(final Builder builder, + public static void setOnColumnMetaDataBuilder(final Common.ColumnMetaData.Builder builder, final Map metadataMap) { final FlightSqlColumnMetadata columnMetadata = new FlightSqlColumnMetadata(metadataMap); final String catalogName = columnMetadata.getCatalogName(); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/UrlParser.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/UrlParser.java index e52251f53918a..64255e2213a1a 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/UrlParser.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/UrlParser.java @@ -21,10 +21,10 @@ import java.net.URLDecoder; import java.util.HashMap; import java.util.Map; - /** * URL Parser for extracting key values from a connection string. */ + public final class UrlParser { private UrlParser() { } @@ -37,6 +37,7 @@ private UrlParser() { * @param url {@link String} * @return {@link Map} */ + @SuppressWarnings("StringSplitter") public static Map parse(String url, String separator) { Map resultMap = new HashMap<>(); if (url != null) { diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java index 90c926612f15a..76f01514c9501 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcArrayTest.java @@ -140,7 +140,7 @@ public void testShouldGetResultSetReturnValidResultSetWithOffsets() throws SQLEx Assert.assertEquals((Object) resultSet.getInt(1), dataVector.getObject(count + 3)); count++; } - Assert.assertEquals(count, 5); + Assert.assertEquals(5, count); } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java index 784fd5b292b27..e1f64c9dd8732 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriverTest.java @@ -181,6 +181,8 @@ public void testConnectWithInsensitiveCasePropertyKeys2() throws Exception { /** * Tests whether an exception is thrown upon attempting to connect to a * malformed URI. + * + * @throws SQLException If an error occurs. */ @Test(expected = SQLException.class) public void testShouldThrowExceptionWhenAttemptingToConnectToMalformedUrl() throws SQLException { @@ -194,7 +196,7 @@ public void testShouldThrowExceptionWhenAttemptingToConnectToMalformedUrl() thro * Tests whether an exception is thrown upon attempting to connect to a * malformed URI. * - * @throws Exception If an error occurs. + * @throws SQLException If an error occurs. */ @Test(expected = SQLException.class) public void testShouldThrowExceptionWhenAttemptingToConnectToUrlNoPrefix() throws SQLException { diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java index cc44cc57be9b3..03f15d77ade11 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ConnectionMutualTlsTest.java @@ -50,8 +50,6 @@ public class ConnectionMutualTlsTest { @ClassRule public static final FlightServerTestRule FLIGHT_SERVER_TEST_RULE; private static final String tlsRootCertsPath; - - private static final String serverMTlsCACertPath; private static final String clientMTlsCertPath; private static final String badClientMTlsCertPath; private static final String clientMTlsKeyPath; @@ -68,8 +66,6 @@ public class ConnectionMutualTlsTest { final File serverMTlsCACert = FlightSqlTestCertificates.exampleCACert(); - serverMTlsCACertPath = serverMTlsCACert.getPath(); - final FlightSqlTestCertificates.CertKeyPair clientMTlsCertKey = FlightSqlTestCertificates.exampleTlsCerts().get(1); diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java index 231371a923a28..0e3e015a04636 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java @@ -94,10 +94,6 @@ private static void resultSetNextUntilDone(ResultSet resultSet) throws SQLExcept } } - private static void setMaxRowsLimit(int maxRowsLimit, Statement statement) throws SQLException { - statement.setLargeMaxRows(maxRowsLimit); - } - /** * Tests whether the {@link ArrowFlightJdbcDriver} can run a query successfully. * @@ -411,9 +407,9 @@ public void testPartitionedFlightServer() throws Exception { // Construct the data-only nodes first. FlightProducer firstProducer = new PartitionedFlightSqlProducer.DataOnlyFlightSqlProducer( - new Ticket("first".getBytes()), firstPartition); + new Ticket("first".getBytes(StandardCharsets.UTF_8)), firstPartition); FlightProducer secondProducer = new PartitionedFlightSqlProducer.DataOnlyFlightSqlProducer( - new Ticket("second".getBytes()), secondPartition); + new Ticket("second".getBytes(StandardCharsets.UTF_8)), secondPartition); final FlightServer.Builder firstBuilder = FlightServer.builder( allocator, forGrpcInsecure("localhost", 0), firstProducer); @@ -427,10 +423,10 @@ public void testPartitionedFlightServer() throws Exception { firstServer.start(); secondServer.start(); final FlightEndpoint firstEndpoint = - new FlightEndpoint(new Ticket("first".getBytes()), firstServer.getLocation()); + new FlightEndpoint(new Ticket("first".getBytes(StandardCharsets.UTF_8)), firstServer.getLocation()); final FlightEndpoint secondEndpoint = - new FlightEndpoint(new Ticket("second".getBytes()), secondServer.getLocation()); + new FlightEndpoint(new Ticket("second".getBytes(StandardCharsets.UTF_8)), secondServer.getLocation()); // Finally start the root node. try (final PartitionedFlightSqlProducer rootProducer = new PartitionedFlightSqlProducer( diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorTest.java index 099b0122179f1..a9b5c46e01e9b 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/ArrowFlightJdbcAccessorTest.java @@ -123,7 +123,7 @@ public void testShouldGetObjectWithBooleanClassReturnGetBoolean() throws SQLExce when(accessor.getObject(Boolean.class)).thenCallRealMethod(); - Assert.assertEquals(accessor.getObject(Boolean.class), true); + Assert.assertEquals(true, accessor.getObject(Boolean.class)); verify(accessor).getBoolean(); } @@ -134,7 +134,7 @@ public void testShouldGetObjectWithBigDecimalClassReturnGetBigDecimal() throws S when(accessor.getObject(BigDecimal.class)).thenCallRealMethod(); - Assert.assertEquals(accessor.getObject(BigDecimal.class), expected); + Assert.assertEquals(expected, accessor.getObject(BigDecimal.class)); verify(accessor).getBigDecimal(); } @@ -145,7 +145,7 @@ public void testShouldGetObjectWithStringClassReturnGetString() throws SQLExcept when(accessor.getObject(String.class)).thenCallRealMethod(); - Assert.assertEquals(accessor.getObject(String.class), expected); + Assert.assertEquals(expected, accessor.getObject(String.class)); verify(accessor).getString(); } @@ -167,7 +167,7 @@ public void testShouldGetObjectWithObjectClassReturnGetObject() throws SQLExcept when(accessor.getObject(Object.class)).thenCallRealMethod(); - Assert.assertEquals(accessor.getObject(Object.class), expected); + Assert.assertEquals(expected, accessor.getObject(Object.class)); verify(accessor).getObject(); } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java index 38d842724b9c1..e2c17b2f085ae 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcTimeStampVectorAccessorTest.java @@ -179,7 +179,7 @@ public void testShouldGetTimestampReturnValidTimestampWithCalendar() throws Exce final Timestamp resultWithoutCalendar = accessor.getTimestamp(null); final Timestamp result = accessor.getTimestamp(calendar); - long offset = timeZone.getOffset(resultWithoutCalendar.getTime()) - + long offset = (long) timeZone.getOffset(resultWithoutCalendar.getTime()) - timeZoneForVector.getOffset(resultWithoutCalendar.getTime()); collector.checkThat(resultWithoutCalendar.getTime() - result.getTime(), is(offset)); @@ -212,7 +212,7 @@ public void testShouldGetDateReturnValidDateWithCalendar() throws Exception { final Date resultWithoutCalendar = accessor.getDate(null); final Date result = accessor.getDate(calendar); - long offset = timeZone.getOffset(resultWithoutCalendar.getTime()) - + long offset = (long) timeZone.getOffset(resultWithoutCalendar.getTime()) - timeZoneForVector.getOffset(resultWithoutCalendar.getTime()); collector.checkThat(resultWithoutCalendar.getTime() - result.getTime(), is(offset)); @@ -245,7 +245,7 @@ public void testShouldGetTimeReturnValidTimeWithCalendar() throws Exception { final Time resultWithoutCalendar = accessor.getTime(null); final Time result = accessor.getTime(calendar); - long offset = timeZone.getOffset(resultWithoutCalendar.getTime()) - + long offset = (long) timeZone.getOffset(resultWithoutCalendar.getTime()) - timeZoneForVector.getOffset(resultWithoutCalendar.getTime()); collector.checkThat(resultWithoutCalendar.getTime() - result.getTime(), is(offset)); diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java index b2eb8f1dbee8f..e958fb60ba41e 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcListAccessorTest.java @@ -114,7 +114,7 @@ public void testShouldGetObjectReturnValidList() throws Exception { accessorIterator.assertAccessorGetter(vector, AbstractArrowFlightJdbcListVectorAccessor::getObject, (accessor, currentRow) -> equalTo( - Arrays.asList(0, (currentRow), (currentRow) * 2, (currentRow) * 3, (currentRow) * 4))); + Arrays.asList(0, currentRow, currentRow * 2, currentRow * 3, currentRow * 4))); } @Test @@ -137,7 +137,7 @@ public void testShouldGetArrayReturnValidArray() throws Exception { Object[] arrayObject = (Object[]) array.getArray(); collector.checkThat(arrayObject, equalTo( - new Object[] {0, currentRow, (currentRow) * 2, (currentRow) * 3, (currentRow) * 4})); + new Object[] {0, currentRow, currentRow * 2, currentRow * 3, currentRow * 4})); }); } @@ -161,7 +161,7 @@ public void testShouldGetArrayReturnValidArrayPassingOffsets() throws Exception Object[] arrayObject = (Object[]) array.getArray(1, 3); collector.checkThat(arrayObject, equalTo( - new Object[] {currentRow, (currentRow) * 2, (currentRow) * 3})); + new Object[] {currentRow, currentRow * 2, currentRow * 3})); }); } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcStructVectorAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcStructVectorAccessorTest.java index b3c85fc0ab1f3..735fe9f40ba0e 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcStructVectorAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/ArrowFlightJdbcStructVectorAccessorTest.java @@ -202,8 +202,8 @@ public void testShouldGetObjectWorkWithNestedComplexData() throws SQLException { new ArrowFlightJdbcStructVectorAccessor(rootVector, () -> 0, (boolean wasNull) -> { }); - Assert.assertEquals(accessor.getObject(), expected); - Assert.assertEquals(accessor.getString(), expected.toString()); + Assert.assertEquals(expected, accessor.getObject()); + Assert.assertEquals(expected.toString(), accessor.getString()); } } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBitVectorAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBitVectorAccessorTest.java index 809d6e8d35386..00537bfa028e9 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBitVectorAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcBitVectorAccessorTest.java @@ -68,7 +68,7 @@ private void iterate(final CheckedFunction is(arrayToAssert[currentRow] ? result : resultIfFalse)) + (accessor, currentRow) -> is(arrayToAssert[currentRow] ? result : resultIfFalse) ); } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java index c165bfb7ce336..52a397edab18f 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/MockFlightSqlProducer.java @@ -159,7 +159,7 @@ public void addSelectQuery(final String sqlCommand, final Schema schema, * @param updatedRows the number of rows affected. */ public void addUpdateQuery(final String sqlCommand, final long updatedRows) { - addUpdateQuery(sqlCommand, ((flightStream, putResultStreamListener) -> { + addUpdateQuery(sqlCommand, (flightStream, putResultStreamListener) -> { final DoPutUpdateResult result = DoPutUpdateResult.newBuilder().setRecordCount(updatedRows).build(); try (final BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); @@ -171,7 +171,7 @@ public void addUpdateQuery(final String sqlCommand, final long updatedRows) { } finally { putResultStreamListener.onCompleted(); } - })); + }); } /** diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestRule.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestRule.java index a200fc8d39c15..fd8fb57fcafde 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestRule.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/RootAllocatorTestRule.java @@ -18,6 +18,7 @@ package org.apache.arrow.driver.jdbc.utils; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -456,9 +457,9 @@ public VarBinaryVector createVarBinaryVector() { public VarBinaryVector createVarBinaryVector(final String fieldName) { VarBinaryVector valueVector = new VarBinaryVector(fieldName, this.getRootAllocator()); valueVector.allocateNew(3); - valueVector.setSafe(0, (fieldName + "__BINARY_DATA_0001").getBytes()); - valueVector.setSafe(1, (fieldName + "__BINARY_DATA_0002").getBytes()); - valueVector.setSafe(2, (fieldName + "__BINARY_DATA_0003").getBytes()); + valueVector.setSafe(0, (fieldName + "__BINARY_DATA_0001").getBytes(StandardCharsets.UTF_8)); + valueVector.setSafe(1, (fieldName + "__BINARY_DATA_0002").getBytes(StandardCharsets.UTF_8)); + valueVector.setSafe(2, (fieldName + "__BINARY_DATA_0003").getBytes(StandardCharsets.UTF_8)); valueVector.setValueCount(3); return valueVector; @@ -472,9 +473,9 @@ public VarBinaryVector createVarBinaryVector(final String fieldName) { public LargeVarBinaryVector createLargeVarBinaryVector() { LargeVarBinaryVector valueVector = new LargeVarBinaryVector("", this.getRootAllocator()); valueVector.allocateNew(3); - valueVector.setSafe(0, "BINARY_DATA_0001".getBytes()); - valueVector.setSafe(1, "BINARY_DATA_0002".getBytes()); - valueVector.setSafe(2, "BINARY_DATA_0003".getBytes()); + valueVector.setSafe(0, "BINARY_DATA_0001".getBytes(StandardCharsets.UTF_8)); + valueVector.setSafe(1, "BINARY_DATA_0002".getBytes(StandardCharsets.UTF_8)); + valueVector.setSafe(2, "BINARY_DATA_0003".getBytes(StandardCharsets.UTF_8)); valueVector.setValueCount(3); return valueVector; @@ -488,9 +489,9 @@ public LargeVarBinaryVector createLargeVarBinaryVector() { public FixedSizeBinaryVector createFixedSizeBinaryVector() { FixedSizeBinaryVector valueVector = new FixedSizeBinaryVector("", this.getRootAllocator(), 16); valueVector.allocateNew(3); - valueVector.setSafe(0, "BINARY_DATA_0001".getBytes()); - valueVector.setSafe(1, "BINARY_DATA_0002".getBytes()); - valueVector.setSafe(2, "BINARY_DATA_0003".getBytes()); + valueVector.setSafe(0, "BINARY_DATA_0001".getBytes(StandardCharsets.UTF_8)); + valueVector.setSafe(1, "BINARY_DATA_0002".getBytes(StandardCharsets.UTF_8)); + valueVector.setSafe(2, "BINARY_DATA_0003".getBytes(StandardCharsets.UTF_8)); valueVector.setValueCount(3); return valueVector; diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ThrowableAssertionUtils.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ThrowableAssertionUtils.java index f1bd44539ac58..48334dc0f92e2 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ThrowableAssertionUtils.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/ThrowableAssertionUtils.java @@ -25,7 +25,7 @@ public class ThrowableAssertionUtils { private ThrowableAssertionUtils() { } - public static void simpleAssertThrowableClass( + public static void simpleAssertThrowableClass( final Class expectedThrowable, final ThrowingRunnable runnable) { try { runnable.run(); diff --git a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlProducer.java b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlProducer.java index e2d79129c1fc9..dbe39ab1d07b4 100644 --- a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlProducer.java +++ b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlProducer.java @@ -433,7 +433,7 @@ default void cancelFlightInfo(CancelFlightInfoRequest request, CallContext conte * @param info The FlightInfo of the query to cancel. * @param context Per-call context. * @param listener Whether cancellation succeeded. - * @deprecated Prefer {@link #cancelFlightInfo(FlightInfo, CallContext, StreamListener)}. + * @deprecated Prefer {@link #cancelFlightInfo(CancelFlightInfoRequest, CallContext, StreamListener)}. */ @Deprecated default void cancelQuery(FlightInfo info, CallContext context, StreamListener listener) { diff --git a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/SqlInfoBuilder.java b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/SqlInfoBuilder.java index 251a709f63965..338a60e2ae6df 100644 --- a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/SqlInfoBuilder.java +++ b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/SqlInfoBuilder.java @@ -17,7 +17,6 @@ package org.apache.arrow.flight.sql; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.IntStream.range; import static org.apache.arrow.flight.FlightProducer.ServerStreamListener; import static org.apache.arrow.flight.sql.impl.FlightSql.SqlSupportedTransaction; @@ -81,7 +80,7 @@ public class SqlInfoBuilder { * @return a new {@link NullableVarCharHolder} with the provided input data {@code string}. */ public static NullableVarCharHolder getHolderForUtf8(final String string, final ArrowBuf buf) { - final byte[] bytes = string.getBytes(UTF_8); + final byte[] bytes = string.getBytes(StandardCharsets.UTF_8); buf.setBytes(0, bytes); final NullableVarCharHolder holder = new NullableVarCharHolder(); holder.buffer = buf; @@ -1051,7 +1050,7 @@ private void setDataVarCharListField(final VectorSchemaRoot root, final int inde final int length = values.length; range(0, length) .forEach(i -> onCreateArrowBuf(buf -> { - final byte[] bytes = values[i].getBytes(UTF_8); + final byte[] bytes = values[i].getBytes(StandardCharsets.UTF_8); buf.setBytes(0, bytes); writer.writeVarChar(0, bytes.length, buf); })); diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java index 11f38ded5fcdd..1d43728b789f5 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java @@ -69,6 +69,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -382,6 +383,7 @@ private static int saveToVectors(final Map ve return saveToVectors(vectorToColumnName, data, emptyToNull, alwaysTrue); } + @SuppressWarnings("StringSplitter") private static int saveToVectors(final Map vectorToColumnName, final ResultSet data, boolean emptyToNull, Predicate resultSetPredicate) @@ -512,7 +514,7 @@ private static VectorSchemaRoot getTypeInfoRoot(CommandGetXdbcTypeInfo request, } }; } else { - predicate = (resultSet -> true); + predicate = resultSet -> true; } int rows = saveToVectors(mapper, typeInfo, true, predicate); @@ -685,7 +687,7 @@ public void getStreamPreparedStatement(final CommandPreparedStatementQuery comma public void closePreparedStatement(final ActionClosePreparedStatementRequest request, final CallContext context, final StreamListener listener) { // Running on another thread - executorService.submit(() -> { + Future unused = executorService.submit(() -> { try { preparedStatementLoadingCache.invalidate(request.getPreparedStatementHandle()); } catch (final Exception e) { @@ -774,7 +776,7 @@ public void listFlights(CallContext context, Criteria criteria, StreamListener listener) { // Running on another thread - executorService.submit(() -> { + Future unused = executorService.submit(() -> { try { final ByteString preparedStatementHandle = copyFrom(randomUUID().toString().getBytes(UTF_8)); // Ownership of the connection will be passed to the context. Do NOT close! diff --git a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java index c49b04c855846..1201d0f760524 100644 --- a/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java +++ b/java/tools/src/main/java/org/apache/arrow/tools/FileRoundtrip.java @@ -43,11 +43,9 @@ public class FileRoundtrip { private static final Logger LOGGER = LoggerFactory.getLogger(FileRoundtrip.class); private final Options options; - private final PrintStream out; private final PrintStream err; - FileRoundtrip(PrintStream out, PrintStream err) { - this.out = out; + FileRoundtrip(PrintStream err) { this.err = err; this.options = new Options(); this.options.addOption("i", "in", true, "input file"); @@ -56,7 +54,7 @@ public class FileRoundtrip { } public static void main(String[] args) { - System.exit(new FileRoundtrip(System.out, System.err).run(args)); + System.exit(new FileRoundtrip(System.err).run(args)); } private File validateFile(String type, String fileName) { diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java index 178a0834fa44f..1bc7ead7b73bb 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java @@ -35,7 +35,6 @@ import org.apache.arrow.vector.ipc.ArrowFileReader; import org.apache.arrow.vector.ipc.ArrowFileWriter; import org.apache.arrow.vector.ipc.message.ArrowBlock; -import org.apache.arrow.vector.types.pojo.Schema; import org.junit.Assert; public class ArrowFileTestFixtures { @@ -63,7 +62,6 @@ static void validateOutput(File testOutFile, BufferAllocator allocator) throws E ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) { VectorSchemaRoot root = arrowReader.getVectorSchemaRoot(); - Schema schema = root.getSchema(); for (ArrowBlock rbBlock : arrowReader.getRecordBlocks()) { if (!arrowReader.loadRecordBatch(rbBlock)) { throw new IOException("Expected to read record batch"); diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index 714cb416bf996..9cf893ee5c283 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -142,7 +142,6 @@ public void basicTest() throws InterruptedException, IOException { Collections.emptyList()); TinyIntVector vector = new TinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc); - Schema schema = new Schema(asList(field)); // Try an empty stream, just the header. testEchoServer(serverPort, field, vector, 0); diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java index ddac6f79384d9..a5d6c9658fd4f 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java @@ -56,7 +56,7 @@ public void test() throws Exception { writeInput(testInFile, allocator); String[] args = {"-i", testInFile.getAbsolutePath(), "-o", testOutFile.getAbsolutePath()}; - int result = new FileRoundtrip(System.out, System.err).run(args); + int result = new FileRoundtrip(System.err).run(args); assertEquals(0, result); validateOutput(testOutFile, allocator); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index 90229460111c3..c456c625389ba 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -110,7 +110,7 @@ public String getName() { */ @Override public long getValidityBufferAddress() { - return (validityBuffer.memoryAddress()); + return validityBuffer.memoryAddress(); } /** @@ -120,7 +120,7 @@ public long getValidityBufferAddress() { */ @Override public long getDataBufferAddress() { - return (valueBuffer.memoryAddress()); + return valueBuffer.memoryAddress(); } /** @@ -298,6 +298,7 @@ public boolean allocateNewSafe() { * @param valueCount the desired number of elements in the vector * @throws org.apache.arrow.memory.OutOfMemoryException on error */ + @Override public void allocateNew(int valueCount) { computeAndCheckBufferSize(valueCount); @@ -521,6 +522,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers * * @return the inner buffers. */ + @Override public List getFieldBuffers() { List result = new ArrayList<>(2); setReaderAndWriterIndex(); @@ -597,6 +599,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * @param allocator allocator for the target vector * @return TransferPair */ + @Override public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); /** @@ -605,6 +608,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * @param allocator allocator for the target vector * @return TransferPair */ + @Override public abstract TransferPair getTransferPair(Field field, BufferAllocator allocator); /** @@ -911,6 +915,7 @@ public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { * * @param index position of element */ + @Override public void setNull(int index) { handleSafe(index); // not really needed to set the bit to 0 as long as diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index a77278138f28c..c239edbcc3c29 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -228,6 +228,7 @@ private void initOffsetBuffer() { * Reset the vector to initial state. Same as {@link #zeroVector()}. * Note that this method doesn't release any memory. */ + @Override public void reset() { zeroVector(); lastSet = -1; @@ -318,6 +319,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers * Get the buffers belonging to this vector. * @return the inner buffers. */ + @Override public List getFieldBuffers() { // before flight/IPC, we must bring the vector to a consistent state. // this is because, it is possible that the offset buffers of some trailing values @@ -471,6 +473,7 @@ private void allocateValidityBuffer(final long size) { * Resize the vector to increase the capacity. The internal behavior is to * double the current value capacity. */ + @Override public void reAlloc() { reallocDataBuffer(); reallocValidityAndOffsetBuffers(); @@ -691,6 +694,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * @param allocator allocator for the target vector * @return TransferPair */ + @Override public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); /** @@ -699,6 +703,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * @param allocator allocator for the target vector * @return TransferPair */ + @Override public abstract TransferPair getTransferPair(Field field, BufferAllocator allocator); /** @@ -835,6 +840,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * * @return the number of null elements. */ + @Override public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); } @@ -856,6 +862,7 @@ public boolean isSafe(int index) { * @param index position of element * @return true if element at given index is null */ + @Override public boolean isNull(int index) { return (isSet(index) == 0); } @@ -879,6 +886,7 @@ public int isSet(int index) { * * @return valueCount for the vector */ + @Override public int getValueCount() { return valueCount; } @@ -888,6 +896,7 @@ public int getValueCount() { * * @param valueCount value count */ + @Override public void setValueCount(int valueCount) { assert valueCount >= 0; this.valueCount = valueCount; @@ -1091,6 +1100,7 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { * * @param index position of element */ + @Override public void setNull(int index) { // We need to check and realloc both validity and offset buffer while (index >= getValueCapacity()) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 679e5d06c016e..070919c356791 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -28,15 +28,12 @@ import org.apache.arrow.vector.util.DataSizeRoundingUtil; import org.apache.arrow.vector.util.TransferPair; import org.apache.arrow.vector.util.ValueVectorUtility; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Base class for other Arrow Vector Types. Provides basic functionality around * memory management. */ public abstract class BaseValueVector implements ValueVector { - private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class); public static final String MAX_ALLOCATION_SIZE_PROPERTY = "arrow.vector.max_allocation_bytes"; public static final long MAX_ALLOCATION_SIZE = Long.getLong(MAX_ALLOCATION_SIZE_PROPERTY, Long.MAX_VALUE); @@ -160,6 +157,7 @@ long computeCombinedBufferSize(int valueCount, int typeWidth) { * * @return Concrete instance of FieldReader by using double-checked locking. */ + @Override public FieldReader getReader() { FieldReader reader = fieldReader; @@ -178,7 +176,7 @@ public FieldReader getReader() { /** * Container for primitive vectors (1 for the validity bit-mask and one to hold the values). */ - class DataAndValidityBuffers { + static class DataAndValidityBuffers { private ArrowBuf dataBuf; private ArrowBuf validityBuf; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index 46bc9815f037a..4cf495a349f02 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -247,6 +247,7 @@ private void initOffsetBuffer() { * Reset the vector to initial state. Same as {@link #zeroVector()}. * Note that this method doesn't release any memory. */ + @Override public void reset() { zeroVector(); lastSet = -1; @@ -337,6 +338,7 @@ public void loadFieldBuffers(ArrowFieldNode fieldNode, List ownBuffers * Get the buffers belonging to this vector. * @return the inner buffers. */ + @Override public List getFieldBuffers() { // before flight/IPC, we must bring the vector to a consistent state. // this is because, it is possible that the offset buffers of some trailing values @@ -493,6 +495,7 @@ private void allocateValidityBuffer(final long size) { * Resize the vector to increase the capacity. The internal behavior is to * double the current value capacity. */ + @Override public void reAlloc() { reallocDataBuffer(); reallocValidityAndOffsetBuffers(); @@ -732,6 +735,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * @param allocator allocator for the target vector * @return TransferPair */ + @Override public abstract TransferPair getTransferPair(String ref, BufferAllocator allocator); /** @@ -740,6 +744,7 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * @param allocator allocator for the target vector * @return TransferPair */ + @Override public abstract TransferPair getTransferPair(Field field, BufferAllocator allocator); /** @@ -796,7 +801,8 @@ private void splitAndTransferOffsetBuffer(int startIndex, int length, BaseVariab final int dataLength = end - start; if (start == 0) { - final ArrowBuf slicedOffsetBuffer = offsetBuffer.slice(startIndex * OFFSET_WIDTH, (1 + length) * OFFSET_WIDTH); + final ArrowBuf slicedOffsetBuffer = offsetBuffer.slice(startIndex * ((long) OFFSET_WIDTH), + (1 + length) * ((long) OFFSET_WIDTH)); target.offsetBuffer = transferBuffer(slicedOffsetBuffer, target.allocator); } else { target.allocateOffsetBuffer((long) (length + 1) * OFFSET_WIDTH); @@ -883,6 +889,7 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, * * @return the number of null elements. */ + @Override public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); } @@ -904,6 +911,7 @@ public boolean isSafe(int index) { * @param index position of element * @return true if element at given index is null */ + @Override public boolean isNull(int index) { return (isSet(index) == 0); } @@ -927,6 +935,7 @@ public int isSet(int index) { * * @return valueCount for the vector */ + @Override public int getValueCount() { return valueCount; } @@ -936,6 +945,7 @@ public int getValueCount() { * * @param valueCount value count */ + @Override public void setValueCount(int valueCount) { assert valueCount >= 0; this.valueCount = valueCount; @@ -1016,7 +1026,7 @@ public void setValueLengthSafe(int index, int length) { handleSafe(index, length); fillHoles(index); final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + length); lastSet = index; } @@ -1119,7 +1129,7 @@ public void set(int index, ByteBuffer value, int start, int length) { fillHoles(index); BitVectorHelper.setBit(validityBuffer, index); final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + length); valueBuffer.setBytes(startOffset, value, start, length); lastSet = index; } @@ -1140,7 +1150,7 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { fillHoles(index); BitVectorHelper.setBit(validityBuffer, index); final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + length); valueBuffer.setBytes(startOffset, value, start, length); lastSet = index; } @@ -1150,6 +1160,7 @@ public void setSafe(int index, ByteBuffer value, int start, int length) { * * @param index position of element */ + @Override public void setNull(int index) { // We need to check and realloc both validity and offset buffer while (index >= getValueCapacity()) { @@ -1174,7 +1185,7 @@ public void set(int index, int isSet, int start, int end, ArrowBuf buffer) { fillHoles(index); BitVectorHelper.setValidityBit(validityBuffer, index, isSet); final int startOffset = offsetBuffer.getInt((long) index * OFFSET_WIDTH); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, buffer, start, dataLength); lastSet = index; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java index b0052e7e33009..095d98aa265fe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BigIntVector.java @@ -129,6 +129,7 @@ public void get(int index, NullableBigIntHolder holder) { * @param index position of element * @return element at given index */ + @Override public Long getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 104819147b109..a34df8cf6f68b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -105,7 +105,7 @@ public MinorType getMinorType() { @Override public void setInitialCapacity(int valueCount) { final int size = getValidityBufferSizeFromCount(valueCount); - if (size * 2 > MAX_ALLOCATION_SIZE) { + if (size * 2L > MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } lastValueCapacity = valueCount; @@ -149,15 +149,14 @@ public int getBufferSize() { * @param length length of the split. * @param target destination vector */ + @Override public void splitAndTransferTo(int startIndex, int length, BaseFixedWidthVector target) { Preconditions.checkArgument(startIndex >= 0 && length >= 0 && startIndex + length <= valueCount, "Invalid parameters startIndex: %s, length: %s for valueCount: %s", startIndex, length, valueCount); compareTypes(target, "splitAndTransferTo"); target.clear(); - target.validityBuffer = splitAndTransferBuffer(startIndex, length, target, - validityBuffer, target.validityBuffer); - target.valueBuffer = splitAndTransferBuffer(startIndex, length, target, - valueBuffer, target.valueBuffer); + target.validityBuffer = splitAndTransferBuffer(startIndex, length, validityBuffer, target.validityBuffer); + target.valueBuffer = splitAndTransferBuffer(startIndex, length, valueBuffer, target.valueBuffer); target.refreshValueCapacity(); target.setValueCount(length); @@ -166,7 +165,6 @@ public void splitAndTransferTo(int startIndex, int length, BaseFixedWidthVector private ArrowBuf splitAndTransferBuffer( int startIndex, int length, - BaseFixedWidthVector target, ArrowBuf sourceBuffer, ArrowBuf destBuffer) { int firstByteSource = BitVectorHelper.byteIndex(startIndex); @@ -276,11 +274,12 @@ public void get(int index, NullableBitHolder holder) { * @param index position of element * @return element at given index */ + @Override public Boolean getObject(int index) { if (isSet(index) == 0) { return null; } else { - return new Boolean(getBit(index) != 0); + return getBit(index) != 0; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java index 09c874e398022..9725693348a48 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java @@ -144,7 +144,7 @@ public boolean equals(Object obj) { if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (!(obj instanceof BufferLayout)) { return false; } BufferLayout other = (BufferLayout) obj; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java index c99c5786058b7..13645d3b26004 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java @@ -131,6 +131,7 @@ public void get(int index, NullableDateDayHolder holder) { * @param index position of element * @return element at given index */ + @Override public Integer getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java index 6ab8ac4eed229..1333fb0adcefa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateMilliVector.java @@ -133,6 +133,7 @@ public void get(int index, NullableDateMilliHolder holder) { * @param index position of element * @return element at given index */ + @Override public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java index fe650c7d28074..931c4eea0afb1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java @@ -151,6 +151,7 @@ public void get(int index, NullableDecimal256Holder holder) { * @param index position of element * @return element at given index */ + @Override public BigDecimal getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index 7c3662c86748b..eefcee837f719 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -150,6 +150,7 @@ public void get(int index, NullableDecimalHolder holder) { * @param index position of element * @return element at given index */ + @Override public BigDecimal getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java index b6abc16194b77..636afef1e9f7b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java @@ -143,6 +143,7 @@ public void get(int index, NullableDurationHolder holder) { * @param index position of element * @return element at given index */ + @Override public Duration getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java index 4b56a22f2d0c4..46f9447be2938 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float4Vector.java @@ -131,6 +131,7 @@ public void get(int index, NullableFloat4Holder holder) { * @param index position of element * @return element at given index */ + @Override public Float getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java index 7e4fae7087ba5..840f9d4ba087b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Float8Vector.java @@ -131,6 +131,7 @@ public void get(int index, NullableFloat8Holder holder) { * @param index position of element * @return element at given index */ + @Override public Double getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java index efebfd83543d7..6cda18a8a53d3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/GenerateSampleData.java @@ -108,8 +108,8 @@ private static void writeTimeStampData(TimeStampVector vector, int valueCount) { } private static void writeDecimalData(DecimalVector vector, int valueCount) { - final BigDecimal even = new BigDecimal(0.0543278923); - final BigDecimal odd = new BigDecimal(2.0543278923); + final BigDecimal even = new BigDecimal("0.0543278923"); + final BigDecimal odd = new BigDecimal("2.0543278923"); for (int i = 0; i < valueCount; i++) { if (i % 2 == 0) { vector.setSafe(i, even); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java index 5c8ef440e8ea4..08ead148af312 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntVector.java @@ -131,6 +131,7 @@ public void get(int index, NullableIntHolder holder) { * @param index position of element * @return element at given index */ + @Override public Integer getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java index 7c0d19baa9a6f..f53eb37138dcb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java @@ -164,6 +164,7 @@ public void get(int index, NullableIntervalDayHolder holder) { * @param index position of element * @return element at given index */ + @Override public Duration getObject(int index) { if (isSet(index) == 0) { return null; @@ -206,23 +207,23 @@ private StringBuilder getAsStringBuilderHelper(int index) { final int days = valueBuffer.getInt(startIndex); int millis = valueBuffer.getInt(startIndex + MILLISECOND_OFFSET); - final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis); + final int hours = millis / org.apache.arrow.vector.util.DateUtility.hoursToMillis; + millis = millis % org.apache.arrow.vector.util.DateUtility.hoursToMillis; - final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis); + final int minutes = millis / org.apache.arrow.vector.util.DateUtility.minutesToMillis; + millis = millis % org.apache.arrow.vector.util.DateUtility.minutesToMillis; - final int seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis); - millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis); + final int seconds = millis / org.apache.arrow.vector.util.DateUtility.secondsToMillis; + millis = millis % org.apache.arrow.vector.util.DateUtility.secondsToMillis; final String dayString = (Math.abs(days) == 1) ? " day " : " days "; - return (new StringBuilder() + return new StringBuilder() .append(days).append(dayString) .append(hours).append(":") .append(minutes).append(":") .append(seconds).append(".") - .append(millis)); + .append(millis); } /*----------------------------------------------------------------* diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalMonthDayNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalMonthDayNanoVector.java index fc0aa9d27b1c3..716af6fec9cd8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalMonthDayNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalMonthDayNanoVector.java @@ -186,6 +186,7 @@ public void get(int index, NullableIntervalMonthDayNanoHolder holder) { * @param index position of element * @return element at given index */ + @Override public PeriodDuration getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java index 7fe572f3ff1f8..c5f384604aa83 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java @@ -147,6 +147,7 @@ public void get(int index, NullableIntervalYearHolder holder) { * @param index position of element * @return element at given index */ + @Override public Period getObject(int index) { if (isSet(index) == 0) { return null; @@ -181,11 +182,11 @@ private StringBuilder getAsStringBuilderHelper(int index) { final String yearString = (Math.abs(years) == 1) ? " year " : " years "; final String monthString = (Math.abs(months) == 1) ? " month " : " months "; - return (new StringBuilder() + return new StringBuilder() .append(years) .append(yearString) .append(months) - .append(monthString)); + .append(monthString); } /*----------------------------------------------------------------* diff --git a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java index 0750f68f4f716..8560ba3a68b04 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java @@ -131,6 +131,7 @@ public void read(int index, ReusableBuffer buffer) { * @param index position of element to get * @return byte array for non-null element, null otherwise */ + @Override public byte[] getObject(int index) { return get(index); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java index 6f08fcb81fee1..df424c87488a0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarCharVector.java @@ -121,6 +121,7 @@ public byte[] get(int index) { * @param index position of element to get * @return Text object for non-null element, null otherwise */ + @Override public Text getObject(int index) { assert index >= 0; if (NULL_CHECKING_ENABLED && isSet(index) == 0) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java index 518ee707396ea..37a6fe110401e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/SmallIntVector.java @@ -131,6 +131,7 @@ public void get(int index, NullableSmallIntHolder holder) { * @param index position of element * @return element at given index */ + @Override public Short getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java index 86738cd221ec4..c463dc36336c8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMicroVector.java @@ -131,6 +131,7 @@ public void get(int index, NullableTimeMicroHolder holder) { * @param index position of element * @return element at given index */ + @Override public Long getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java index 480add91097bb..1e745d9b9923b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeMilliVector.java @@ -133,6 +133,7 @@ public void get(int index, NullableTimeMilliHolder holder) { * @param index position of element * @return element at given index */ + @Override public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java index 82609cdc446ed..426e865a5c18b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeNanoVector.java @@ -131,6 +131,7 @@ public void get(int index, NullableTimeNanoHolder holder) { * @param index position of element * @return element at given index */ + @Override public Long getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java index 9b7614e55b6e8..c760ed29e04e6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeSecVector.java @@ -131,6 +131,7 @@ public void get(int index, NullableTimeSecHolder holder) { * @param index position of element * @return element at given index */ + @Override public Integer getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java index a37b444d1a368..b515f8e2c83c0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroTZVector.java @@ -133,6 +133,7 @@ public void get(int index, NullableTimeStampMicroTZHolder holder) { * @param index position of element * @return element at given index */ + @Override public Long getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java index 88ce27a187ebc..2f65921f22b26 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMicroVector.java @@ -119,6 +119,7 @@ public void get(int index, NullableTimeStampMicroHolder holder) { * @param index position of element * @return element at given index */ + @Override public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java index 775594ceea640..d0293099432a9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliTZVector.java @@ -133,6 +133,7 @@ public void get(int index, NullableTimeStampMilliTZHolder holder) { * @param index position of element * @return element at given index */ + @Override public Long getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java index a42773269f8b5..96440fd5ac3f7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampMilliVector.java @@ -119,6 +119,7 @@ public void get(int index, NullableTimeStampMilliHolder holder) { * @param index position of element * @return element at given index */ + @Override public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java index af43cf6fc9b64..f93ec9b24c43a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoTZVector.java @@ -133,6 +133,7 @@ public void get(int index, NullableTimeStampNanoTZHolder holder) { * @param index position of element * @return element at given index */ + @Override public Long getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java index 7b02b1c87d3fb..723e62f8d6e02 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampNanoVector.java @@ -119,6 +119,7 @@ public void get(int index, NullableTimeStampNanoHolder holder) { * @param index position of element * @return element at given index */ + @Override public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java index 1e249140335d2..2de01fd52e457 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecVector.java @@ -119,6 +119,7 @@ public void get(int index, NullableTimeStampSecHolder holder) { * @param index position of element * @return element at given index */ + @Override public LocalDateTime getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java index 4c4eee1342ff0..e9ea59298d093 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TinyIntVector.java @@ -131,6 +131,7 @@ public void get(int index, NullableTinyIntHolder holder) { * @param index position of element * @return element at given index */ + @Override public Byte getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java index 60fe2a6a6ee63..ae465418cf2fd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java @@ -55,7 +55,7 @@ public class TypeLayout { /** - * Constructs a new {@TypeLayout} for the given arrowType. + * Constructs a new {@link TypeLayout} for the given arrowType. */ public static TypeLayout getTypeLayout(final ArrowType arrowType) { TypeLayout layout = arrowType.accept(new ArrowTypeVisitor() { @@ -421,6 +421,7 @@ public List getBufferTypes() { return types; } + @Override public String toString() { return bufferLayouts.toString(); } @@ -438,7 +439,7 @@ public boolean equals(Object obj) { if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (!(obj instanceof TypeLayout)) { return false; } TypeLayout other = (TypeLayout) obj; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java index 777df3fb1efe7..fcb04eaf08821 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt1Vector.java @@ -136,6 +136,7 @@ public void get(int index, NullableUInt1Holder holder) { * @param index position of element * @return element at given index */ + @Override public Byte getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java index e5b95be191df1..a9708a4faa9a7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt2Vector.java @@ -127,6 +127,7 @@ public void get(int index, NullableUInt2Holder holder) { * @param index position of element * @return element at given index */ + @Override public Character getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java index bda98b12005ce..f9bed0c013a2a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt4Vector.java @@ -136,6 +136,7 @@ public void get(int index, NullableUInt4Holder holder) { * @param index position of element * @return element at given index */ + @Override public Integer getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java index 5e7c18902f0ae..a3e16b5e30dde 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/UInt8Vector.java @@ -136,6 +136,7 @@ public void get(int index, NullableUInt8Holder holder) { * @param index position of element * @return element at given index */ + @Override public Long getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java index 87790c1168bd0..ab67ebad965aa 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java @@ -132,6 +132,7 @@ public void read(int index, ReusableBuffer buffer) { * @param index position of element to get * @return byte array for non-null element, null otherwise */ + @Override public byte[] getObject(int index) { return get(index); } @@ -176,7 +177,7 @@ public void set(int index, VarBinaryHolder holder) { BitVectorHelper.setBit(validityBuffer, index); final int dataLength = holder.end - holder.start; final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); lastSet = index; } @@ -196,7 +197,7 @@ public void setSafe(int index, VarBinaryHolder holder) { fillHoles(index); BitVectorHelper.setBit(validityBuffer, index); final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); lastSet = index; } @@ -215,10 +216,10 @@ public void set(int index, NullableVarBinaryHolder holder) { final int startOffset = getStartOffset(index); if (holder.isSet != 0) { final int dataLength = holder.end - holder.start; - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); } else { - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset); } lastSet = index; } @@ -238,7 +239,7 @@ public void setSafe(int index, NullableVarBinaryHolder holder) { handleSafe(index, dataLength); fillHoles(index); final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); } else { fillEmpties(index + 1); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java index 7350dc99bbda8..c6d5a7090bc6f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java @@ -118,6 +118,7 @@ public byte[] get(int index) { * @param index position of element to get * @return Text object for non-null element, null otherwise */ + @Override public Text getObject(int index) { assert index >= 0; if (NULL_CHECKING_ENABLED && isSet(index) == 0) { @@ -182,7 +183,7 @@ public void set(int index, VarCharHolder holder) { BitVectorHelper.setBit(validityBuffer, index); final int dataLength = holder.end - holder.start; final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); lastSet = index; } @@ -203,7 +204,7 @@ public void setSafe(int index, VarCharHolder holder) { BitVectorHelper.setBit(validityBuffer, index); final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); lastSet = index; } @@ -222,10 +223,10 @@ public void set(int index, NullableVarCharHolder holder) { final int startOffset = getStartOffset(index); if (holder.isSet != 0) { final int dataLength = holder.end - holder.start; - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); } else { - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset); } lastSet = index; } @@ -245,7 +246,7 @@ public void setSafe(int index, NullableVarCharHolder holder) { handleSafe(index, dataLength); fillHoles(index); final int startOffset = getStartOffset(index); - offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((index + 1) * ((long) OFFSET_WIDTH), startOffset + dataLength); valueBuffer.setBytes(startOffset, holder.buffer, holder.start, dataLength); } else { fillEmpties(index + 1); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index 898bfe3d39780..8e6cdb6c45bc5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -55,6 +55,7 @@ public void allocateNew() throws OutOfMemoryException { } } + @Override public BufferAllocator getAllocator() { return allocator; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java index 797a5af31f9a4..80efea6cbe39e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; import org.apache.arrow.memory.ArrowBuf; @@ -56,7 +57,7 @@ public abstract class AbstractStructVector extends AbstractContainerVector { } ConflictPolicy conflictPolicy; try { - conflictPolicy = ConflictPolicy.valueOf(conflictPolicyStr.toUpperCase()); + conflictPolicy = ConflictPolicy.valueOf(conflictPolicyStr.toUpperCase(Locale.ROOT)); } catch (Exception e) { conflictPolicy = ConflictPolicy.CONFLICT_REPLACE; } @@ -172,6 +173,7 @@ public void reAlloc() { * @return resultant {@link org.apache.arrow.vector.ValueVector} * @throws java.lang.IllegalStateException raised if there is a hard schema change */ + @Override public T addOrGet(String childName, FieldType fieldType, Class clazz) { final ValueVector existing = getChild(childName); boolean create = false; @@ -411,7 +413,7 @@ public int getBufferSize() { for (final ValueVector v : vectors.values()) { for (final ArrowBuf buf : v.getBuffers(false)) { - actualBufSize += buf.writerIndex(); + actualBufSize += (int) buf.writerIndex(); } } return actualBufSize; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java index 95deceb4e75ca..8ba2e48dc2fa3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java @@ -54,7 +54,7 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements public static final byte OFFSET_WIDTH = 4; protected ArrowBuf offsetBuffer; protected FieldVector vector; - protected final CallBack callBack; + protected final CallBack repeatedCallBack; protected int valueCount; protected long offsetAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * OFFSET_WIDTH; private final String name; @@ -70,7 +70,7 @@ protected BaseRepeatedValueVector(String name, BufferAllocator allocator, FieldV this.name = name; this.offsetBuffer = allocator.getEmpty(); this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null"); - this.callBack = callBack; + this.repeatedCallBack = callBack; this.valueCount = 0; } @@ -123,7 +123,7 @@ protected void reallocOffsetBuffer() { } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); - newAllocationSize = Math.min(newAllocationSize, (long) (OFFSET_WIDTH) * Integer.MAX_VALUE); + newAllocationSize = Math.min(newAllocationSize, (long) OFFSET_WIDTH * Integer.MAX_VALUE); assert newAllocationSize >= 1; if (newAllocationSize > MAX_ALLOCATION_SIZE || newAllocationSize <= offsetBuffer.capacity()) { @@ -157,7 +157,7 @@ public FieldVector getDataVector() { @Override public void setInitialCapacity(int numRecords) { - offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; + offsetAllocationSizeInBytes = (numRecords + 1L) * OFFSET_WIDTH; if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) { vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); } else { @@ -194,7 +194,7 @@ public void setInitialCapacity(int numRecords, double density) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } - offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; + offsetAllocationSizeInBytes = (numRecords + 1L) * OFFSET_WIDTH; int innerValueCapacity = Math.max((int) (numRecords * density), 1); @@ -222,7 +222,7 @@ public void setInitialCapacity(int numRecords, double density) { * for in this vector across all records. */ public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; + offsetAllocationSizeInBytes = (numRecords + 1L) * OFFSET_WIDTH; vector.setInitialCapacity(totalNumberOfElements); } @@ -313,13 +313,13 @@ public int size() { public AddOrGetResult addOrGetVector(FieldType fieldType) { boolean created = false; if (vector instanceof NullVector) { - vector = fieldType.createNewSingleVector(defaultDataVectorName, allocator, callBack); + vector = fieldType.createNewSingleVector(defaultDataVectorName, allocator, repeatedCallBack); // returned vector must have the same field created = true; - if (callBack != null && + if (repeatedCallBack != null && // not a schema change if changing from ZeroVector to ZeroVector (fieldType.getType().getTypeID() != ArrowTypeID.Null)) { - callBack.doWork(); + repeatedCallBack.doWork(); } } @@ -355,6 +355,7 @@ public int getInnerValueCountAt(int index) { } /** Return if value at index is null (this implementation is always false). */ + @Override public boolean isNull(int index) { return false; } @@ -376,6 +377,7 @@ public int startNewValue(int index) { } /** Preallocates the number of repeated values. */ + @Override public void setValueCount(int valueCount) { this.valueCount = valueCount; while (valueCount > getOffsetBufferValueCapacity()) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java index 367335436aecd..48b53d7de2e3f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java @@ -234,11 +234,9 @@ public boolean allocateNewSafe() { } finally { if (!success) { clear(); - return false; } } - - return true; + return success; } private void allocateValidityBuffer(final long size) { @@ -257,12 +255,12 @@ public void reAlloc() { private void reallocValidityBuffer() { final int currentBufferCapacity = checkedCastToInt(validityBuffer.capacity()); - long newAllocationSize = currentBufferCapacity * 2; + long newAllocationSize = currentBufferCapacity * 2L; if (newAllocationSize == 0) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2; + newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } @@ -273,7 +271,7 @@ private void reallocValidityBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + final ArrowBuf newBuf = allocator.buffer(newAllocationSize); newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity); validityBuffer.getReferenceManager().release(1); @@ -468,6 +466,7 @@ public List getObject(int index) { /** * Returns whether the value at index null. */ + @Override public boolean isNull(int index) { return (isSet(index) == 0); } @@ -503,6 +502,7 @@ private int getValidityBufferValueCapacity() { /** * Sets the value at index to null. Reallocates if index is larger than capacity. */ + @Override public void setNull(int index) { while (index >= getValidityBufferValueCapacity()) { reallocValidityBuffer(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java index 312bed6ab3349..b934cbd81db16 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListVector.java @@ -194,7 +194,7 @@ public void setInitialCapacity(int numRecords, double density) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } - offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; + offsetAllocationSizeInBytes = (numRecords + 1L) * OFFSET_WIDTH; int innerValueCapacity = Math.max((int) (numRecords * density), 1); @@ -222,7 +222,7 @@ public void setInitialCapacity(int numRecords, double density) { * for in this vector across all records. */ public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - offsetAllocationSizeInBytes = (numRecords + 1) * OFFSET_WIDTH; + offsetAllocationSizeInBytes = (numRecords + 1L) * OFFSET_WIDTH; vector.setInitialCapacity(totalNumberOfElements); } @@ -332,6 +332,7 @@ public void allocateNew() throws OutOfMemoryException { * * @return false if memory allocation fails, true otherwise. */ + @Override public boolean allocateNewSafe() { boolean success = false; try { @@ -347,7 +348,7 @@ public boolean allocateNewSafe() { } catch (Exception e) { e.printStackTrace(); clear(); - return false; + success = false; } finally { if (!dataAlloc) { clear(); @@ -357,10 +358,9 @@ public boolean allocateNewSafe() { } finally { if (!success) { clear(); - return false; } } - return true; + return success; } private void allocateValidityBuffer(final long size) { @@ -408,7 +408,7 @@ protected void reallocOffsetBuffer() { } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); - newAllocationSize = Math.min(newAllocationSize, (long) (OFFSET_WIDTH) * Integer.MAX_VALUE); + newAllocationSize = Math.min(newAllocationSize, (long) OFFSET_WIDTH * Integer.MAX_VALUE); assert newAllocationSize >= 1; if (newAllocationSize > MAX_ALLOCATION_SIZE || newAllocationSize <= offsetBuffer.capacity()) { @@ -425,12 +425,12 @@ protected void reallocOffsetBuffer() { private void reallocValidityBuffer() { final int currentBufferCapacity = checkedCastToInt(validityBuffer.capacity()); - long newAllocationSize = currentBufferCapacity * 2; + long newAllocationSize = currentBufferCapacity * 2L; if (newAllocationSize == 0) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2; + newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -440,7 +440,7 @@ private void reallocValidityBuffer() { throw new OversizedAllocationException("Unable to expand the buffer"); } - final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); + final ArrowBuf newBuf = allocator.buffer(newAllocationSize); newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity); validityBuffer.getReferenceManager().release(1); @@ -526,7 +526,7 @@ public TransferPair makeTransferPair(ValueVector target) { @Override public long getValidityBufferAddress() { - return (validityBuffer.memoryAddress()); + return validityBuffer.memoryAddress(); } @Override @@ -536,7 +536,7 @@ public long getDataBufferAddress() { @Override public long getOffsetBufferAddress() { - return (offsetBuffer.memoryAddress()); + return offsetBuffer.memoryAddress(); } @Override @@ -754,6 +754,7 @@ public UnionLargeListReader getReader() { * Initialize the data vector (and execute callback) if it hasn't already been done, * returns the data vector. */ + @Override public AddOrGetResult addOrGetVector(FieldType fieldType) { boolean created = false; if (vector instanceof NullVector) { @@ -988,6 +989,7 @@ public void setNotNull(int index) { * Sets list at index to be null. * @param index position in vector */ + @Override public void setNull(int index) { while (index >= getValidityAndOffsetValueCapacity()) { reallocValidityAndOffsetBuffers(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java index e5a83921b3135..5154ac17279c5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java @@ -291,6 +291,7 @@ public void allocateNew() throws OutOfMemoryException { * * @return false if memory allocation fails, true otherwise. */ + @Override public boolean allocateNewSafe() { boolean success = false; try { @@ -303,10 +304,9 @@ public boolean allocateNewSafe() { } finally { if (!success) { clear(); - return false; } } - return true; + return success; } protected void allocateValidityBuffer(final long size) { @@ -336,12 +336,23 @@ protected void reallocValidityAndOffsetBuffers() { private void reallocValidityBuffer() { final int currentBufferCapacity = checkedCastToInt(validityBuffer.capacity()); - long newAllocationSize = currentBufferCapacity * 2; + long newAllocationSize = getNewAllocationSize(currentBufferCapacity); + + final ArrowBuf newBuf = allocator.buffer(newAllocationSize); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity); + validityBuffer.getReferenceManager().release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int) newAllocationSize; + } + + private long getNewAllocationSize(int currentBufferCapacity) { + long newAllocationSize = currentBufferCapacity * 2L; if (newAllocationSize == 0) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2; + newAllocationSize = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -350,13 +361,7 @@ private void reallocValidityBuffer() { if (newAllocationSize > MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Unable to expand the buffer"); } - - final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); - newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); - newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity); - validityBuffer.getReferenceManager().release(1); - validityBuffer = newBuf; - validityAllocationSizeInBytes = (int) newAllocationSize; + return newAllocationSize; } /** @@ -425,7 +430,7 @@ public TransferPair makeTransferPair(ValueVector target) { @Override public long getValidityBufferAddress() { - return (validityBuffer.memoryAddress()); + return validityBuffer.memoryAddress(); } @Override @@ -435,7 +440,7 @@ public long getDataBufferAddress() { @Override public long getOffsetBufferAddress() { - return (offsetBuffer.memoryAddress()); + return offsetBuffer.memoryAddress(); } @Override @@ -625,6 +630,7 @@ public UnionListReader getReader() { } /** Initialize the child data vector to field type. */ + @Override public AddOrGetResult addOrGetVector(FieldType fieldType) { AddOrGetResult result = super.addOrGetVector(fieldType); invalidateReader(); @@ -837,6 +843,7 @@ public void setNotNull(int index) { * Sets list at index to be null. * @param index position in vector */ + @Override public void setNull(int index) { while (index >= getValidityAndOffsetValueCapacity()) { reallocValidityAndOffsetBuffers(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java index 27db1574808a3..9d0dc5ca3fd15 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java @@ -495,10 +495,9 @@ public boolean allocateNewSafe() { } finally { if (!success) { clear(); - return false; } } - return true; + return success; } private void allocateValidityBuffer(final long size) { @@ -518,12 +517,23 @@ public void reAlloc() { private void reallocValidityBuffer() { final int currentBufferCapacity = checkedCastToInt(validityBuffer.capacity()); - long newAllocationSize = currentBufferCapacity * 2; + long newAllocationSize = getNewAllocationSize(currentBufferCapacity); + + final ArrowBuf newBuf = allocator.buffer(newAllocationSize); + newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); + newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity); + validityBuffer.getReferenceManager().release(1); + validityBuffer = newBuf; + validityAllocationSizeInBytes = (int) newAllocationSize; + } + + private long getNewAllocationSize(int currentBufferCapacity) { + long newAllocationSize = currentBufferCapacity * 2L; if (newAllocationSize == 0) { if (validityAllocationSizeInBytes > 0) { newAllocationSize = validityAllocationSizeInBytes; } else { - newAllocationSize = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION) * 2; + newAllocationSize = BitVectorHelper.getValidityBufferSize(BaseValueVector.INITIAL_VALUE_ALLOCATION) * 2L; } } newAllocationSize = CommonUtil.nextPowerOfTwo(newAllocationSize); @@ -532,13 +542,7 @@ private void reallocValidityBuffer() { if (newAllocationSize > BaseValueVector.MAX_ALLOCATION_SIZE) { throw new OversizedAllocationException("Unable to expand the buffer"); } - - final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize); - newBuf.setBytes(0, validityBuffer, 0, currentBufferCapacity); - newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity); - validityBuffer.getReferenceManager().release(1); - validityBuffer = newBuf; - validityAllocationSizeInBytes = (int) newAllocationSize; + return newAllocationSize; } @Override @@ -607,6 +611,7 @@ public void get(int index, ComplexHolder holder) { /** * Return the number of null values in the vector. */ + @Override public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); } @@ -614,6 +619,7 @@ public int getNullCount() { /** * Returns true if the value at the provided index is null. */ + @Override public boolean isNull(int index) { return isSet(index) == 0; } @@ -643,6 +649,7 @@ public void setIndexDefined(int index) { /** * Marks the value at index as null/not set. */ + @Override public void setNull(int index) { while (index >= getValidityBufferValueCapacity()) { /* realloc the inner buffers if needed */ diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java index c80fcb89d0cc9..028901ee847da 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java @@ -46,6 +46,7 @@ public int getPosition() { return index; } + @Override public void setPosition(int index) { this.index = index; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index f7be277f592a6..7f724829ef1eb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -19,6 +19,7 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.util.Locale; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.FieldVector; @@ -301,13 +302,15 @@ public boolean isEmptyStruct() { return writer.isEmptyStruct(); } + @Override protected FieldWriter getWriter() { return writer; } private FieldWriter promoteToUnion() { String name = vector.getField().getName(); - TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator()); + TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(Locale.ROOT), + vector.getAllocator()); tp.transfer(); if (parentContainer != null) { // TODO allow dictionaries in complex types diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java index 5c4cd2af98d55..6a217bbc8b547 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java @@ -88,8 +88,9 @@ public StructOrListWriter struct(final String name) { * * @param name Unused. * - * @deprecated use {@link #listOfStruct()} instead. + * @deprecated use {@link #listOfStruct(String)} instead. */ + @Deprecated public StructOrListWriter listoftstruct(final String name) { return listOfStruct(name); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java index ece729ae563af..f69fea3bd5779 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionFixedSizeListReader.java @@ -99,6 +99,7 @@ public boolean next() { } } + @Override public void copyAsValue(ListWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java index faf088b55981d..0f3ba50f2b3a1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionLargeListReader.java @@ -34,7 +34,6 @@ public class UnionLargeListReader extends AbstractFieldReader { private LargeListVector vector; private ValueVector data; - private long index; private static final long OFFSET_WIDTH = 8L; public UnionLargeListReader(LargeListVector vector) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java index 74548bc985f6a..7dadcabdcee88 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListReader.java @@ -60,8 +60,8 @@ public void setPosition(int index) { currentOffset = 0; maxOffset = 0; } else { - currentOffset = vector.getOffsetBuffer().getInt(index * OFFSET_WIDTH) - 1; - maxOffset = vector.getOffsetBuffer().getInt((index + 1) * OFFSET_WIDTH); + currentOffset = vector.getOffsetBuffer().getInt(index * (long) OFFSET_WIDTH) - 1; + maxOffset = vector.getOffsetBuffer().getInt((index + 1) * (long) OFFSET_WIDTH); } } @@ -106,6 +106,7 @@ public boolean next() { } } + @Override public void copyAsValue(ListWriter writer) { ComplexCopier.copy(this, (FieldWriter) writer); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java index 6f40e5814b972..5687e4025acee 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/Dictionary.java @@ -60,7 +60,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof Dictionary)) { return false; } Dictionary that = (Dictionary) o; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/table/package-info.java b/java/vector/src/main/java/org/apache/arrow/vector/table/package-info.java index cdd5093b9f554..b11ada51292f9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/table/package-info.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/table/package-info.java @@ -17,7 +17,7 @@ package org.apache.arrow.vector.table; -/** +/* * Support for Table, an immutable, columnar, tabular data structure based on FieldVectors. * See the Arrow Java documentation for details: Table */ diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java b/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java index c52fc1243d99f..85c2532236866 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/FloatingPointPrecision.java @@ -39,7 +39,7 @@ public enum FloatingPointPrecision { } } - private short flatbufID; + private final short flatbufID; private FloatingPointPrecision(short flatbufID) { this.flatbufID = flatbufID; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java b/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java index 1b17240d016b3..d2314ea7cce3c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/IntervalUnit.java @@ -36,7 +36,7 @@ public enum IntervalUnit { } } - private short flatbufID; + private final short flatbufID; private IntervalUnit(short flatbufID) { this.flatbufID = flatbufID; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java index 8d41b92d867e9..592e18826f09c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/DictionaryEncoding.java @@ -74,7 +74,7 @@ public String toString() { public boolean equals(Object o) { if (this == o) { return true; - } else if (o == null || getClass() != o.getClass()) { + } else if (!(o instanceof DictionaryEncoding)) { return false; } DictionaryEncoding that = (DictionaryEncoding) o; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java index 54c609d4a104f..d3623618e7a55 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java @@ -37,7 +37,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Collections2; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,16 +92,19 @@ private Field( this(name, new FieldType(nullable, type, dictionary, convertMetadata(metadata)), children); } - private Field(String name, FieldType fieldType, List children, TypeLayout typeLayout) { + /** + * Constructs a new Field object. + * + * @param name name of the field + * @param fieldType type of the field + * @param children child fields, if any + */ + public Field(String name, FieldType fieldType, List children) { this.name = name; this.fieldType = checkNotNull(fieldType); this.children = children == null ? Collections.emptyList() : Collections2.toImmutableList(children); } - public Field(String name, FieldType fieldType, List children) { - this(name, fieldType, children, fieldType == null ? null : TypeLayout.getTypeLayout(fieldType.getType())); - } - /** * Construct a new vector of this type using the given allocator. */ @@ -279,7 +281,7 @@ public boolean equals(Object obj) { } Field that = (Field) obj; return Objects.equals(this.name, that.name) && - Objects.equals(this.isNullable(), that.isNullable()) && + this.isNullable() == that.isNullable() && Objects.equals(this.getType(), that.getType()) && Objects.equals(this.getDictionary(), that.getDictionary()) && Objects.equals(this.getMetadata(), that.getMetadata()) && diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java index d5c0d85671fcc..8988993920d79 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java @@ -118,7 +118,7 @@ public boolean equals(Object obj) { return false; } FieldType that = (FieldType) obj; - return Objects.equals(this.isNullable(), that.isNullable()) && + return this.isNullable() == that.isNullable() && Objects.equals(this.getType(), that.getType()) && Objects.equals(this.getDictionary(), that.getDictionary()) && Objects.equals(this.getMetadata(), that.getMetadata()); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java index dcffea0ef5367..392b3c2e2ec73 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java @@ -159,10 +159,10 @@ private Schema(@JsonProperty("fields") Iterable fields, /** * Private constructor to bypass automatic collection copy. - * @param unsafe a ignored argument. Its only purpose is to prevent using the constructor + * @param ignored an ignored argument. Its only purpose is to prevent using the constructor * by accident because of type collisions (List vs Iterable). */ - private Schema(boolean unsafe, List fields, Map metadata) { + private Schema(boolean ignored, List fields, Map metadata) { this.fields = fields; this.metadata = metadata; } @@ -245,13 +245,12 @@ public int getSchema(FlatBufferBuilder builder) { /** * Returns the serialized flatbuffer bytes of the schema wrapped in a message table. - * Use {@link #deserializeMessage() to rebuild the Schema.} + * Use {@link #deserializeMessage(ByteBuffer)} to rebuild the Schema. */ public byte[] serializeAsMessage() { ByteArrayOutputStream out = new ByteArrayOutputStream(); try (WriteChannel channel = new WriteChannel(Channels.newChannel(out))) { - long size = MessageSerializer.serialize( - new WriteChannel(Channels.newChannel(out)), this); + MessageSerializer.serialize(channel, this); return out.toByteArray(); } catch (IOException ex) { throw new RuntimeException(ex); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinalImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinalImpl.java index 7c9c0e9408860..1f18587afdfd1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinalImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinalImpl.java @@ -27,8 +27,6 @@ import java.util.stream.Collectors; import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * An implementation of map that supports constant time look-up by a generic key or an ordinal. @@ -48,7 +46,6 @@ * @param value type */ public class MapWithOrdinalImpl implements MapWithOrdinal { - private static final Logger logger = LoggerFactory.getLogger(MapWithOrdinalImpl.class); private final Map> primary = new LinkedHashMap<>(); private final IntObjectHashMap secondary = new IntObjectHashMap<>(); @@ -93,10 +90,6 @@ public V put(K key, V value) { return oldPair == null ? null : oldPair.getValue(); } - public boolean put(K key, V value, boolean override) { - return put(key, value) != null; - } - @Override public V remove(Object key) { final Entry oldPair = primary.remove(key); @@ -146,6 +139,7 @@ public Set> entrySet() { * @param id ordinal value for lookup * @return an instance of V */ + @Override public V getByOrdinal(int id) { return secondary.get(id); } @@ -156,6 +150,7 @@ public V getByOrdinal(int id) { * @param key key for ordinal lookup * @return ordinal value corresponding to key if it exists or -1 */ + @Override public int getOrdinal(K key) { Map.Entry pair = primary.get(key); if (pair != null) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java index 5f5f5d3bd6d22..95e35ce6938c3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java @@ -210,7 +210,7 @@ public void set(String string) { } /** - * Set to a utf8 byte array. + * Set to an utf8 byte array. * * @param utf8 the byte array to initialize from */ diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java index 19648dc9e13fb..8596399e7e08c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -133,7 +134,7 @@ public void testLargeDecimalVector() { for (int i = 0; i < vecLength; i++) { ArrowBuf buf = largeVec.get(i); - assertEquals(buf.capacity(), DecimalVector.TYPE_WIDTH); + assertEquals(DecimalVector.TYPE_WIDTH, buf.capacity()); assertEquals(0, buf.getLong(0)); assertEquals(0, buf.getLong(8)); @@ -215,7 +216,7 @@ public void testLargeVarCharVector() { logger.trace("Successfully allocated a vector with capacity " + vecLength); for (int i = 0; i < vecLength; i++) { - largeVec.setSafe(i, strElement.getBytes()); + largeVec.setSafe(i, strElement.getBytes(StandardCharsets.UTF_8)); if ((i + 1) % 10000 == 0) { logger.trace("Successfully written " + (i + 1) + " values"); @@ -228,7 +229,7 @@ public void testLargeVarCharVector() { for (int i = 0; i < vecLength; i++) { byte[] val = largeVec.get(i); - assertEquals(strElement, new String(val)); + assertEquals(strElement, new String(val, StandardCharsets.UTF_8)); if ((i + 1) % 10000 == 0) { logger.trace("Successfully read " + (i + 1) + " values"); @@ -254,7 +255,7 @@ public void testLargeLargeVarCharVector() { logger.trace("Successfully allocated a vector with capacity " + vecLength); for (int i = 0; i < vecLength; i++) { - largeVec.setSafe(i, strElement.getBytes()); + largeVec.setSafe(i, strElement.getBytes(StandardCharsets.UTF_8)); if ((i + 1) % 10000 == 0) { logger.trace("Successfully written " + (i + 1) + " values"); @@ -267,7 +268,7 @@ public void testLargeLargeVarCharVector() { for (int i = 0; i < vecLength; i++) { byte[] val = largeVec.get(i); - assertEquals(strElement, new String(val)); + assertEquals(strElement, new String(val, StandardCharsets.UTF_8)); if ((i + 1) % 10000 == 0) { logger.trace("Successfully read " + (i + 1) + " values"); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java index 96005dc511cab..1da4a4c4914b9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java @@ -47,7 +47,7 @@ public void testGetNullCount() throws Exception { validityBuffer.setByte(0, 0xFF); count = BitVectorHelper.getNullCount(validityBuffer, 8); - assertEquals(count, 0); + assertEquals(0, count); validityBuffer.close(); // test case 3, 1 null value for 0x7F @@ -55,7 +55,7 @@ public void testGetNullCount() throws Exception { validityBuffer.setByte(0, 0x7F); count = BitVectorHelper.getNullCount(validityBuffer, 8); - assertEquals(count, 1); + assertEquals(1, count); validityBuffer.close(); // test case 4, validity buffer has multiple bytes, 11 items @@ -64,7 +64,7 @@ public void testGetNullCount() throws Exception { validityBuffer.setByte(1, 0b01010101); count = BitVectorHelper.getNullCount(validityBuffer, 11); - assertEquals(count, 5); + assertEquals(5, count); validityBuffer.close(); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java index 8efadad9b3bf4..056b6bdd2b787 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java @@ -21,6 +21,8 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReferenceManager; import org.apache.arrow.memory.RootAllocator; @@ -65,7 +67,7 @@ public void testTransferVariableWidth() { VarCharVector v1 = new VarCharVector("v1", childAllocator1); v1.allocateNew(); - v1.setSafe(4094, "hello world".getBytes(), 0, 11); + v1.setSafe(4094, "hello world".getBytes(StandardCharsets.UTF_8), 0, 11); v1.setValueCount(4001); VarCharVector v2 = new VarCharVector("v2", childAllocator2); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java index 3786f63c31bb6..97de27bec8237 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java @@ -23,9 +23,10 @@ import static org.junit.Assert.assertNull; import java.math.BigDecimal; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Period; +import java.util.Objects; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -84,7 +85,7 @@ public void testCopyFromWithNulls() { if (i % 3 == 0) { continue; } - byte[] b = Integer.toString(i).getBytes(); + byte[] b = Integer.toString(i).getBytes(StandardCharsets.UTF_8); vector.setSafe(i, b, 0, b.length); } @@ -156,7 +157,7 @@ public void testCopyFromWithNulls1() { if (i % 3 == 0) { continue; } - byte[] b = Integer.toString(i).getBytes(); + byte[] b = Integer.toString(i).getBytes(StandardCharsets.UTF_8); vector.setSafe(i, b, 0, b.length); } @@ -950,7 +951,7 @@ public void testCopyFromWithNulls13() { assertEquals(0, vector1.getValueCount()); int initialCapacity = vector1.getValueCapacity(); - final double baseValue = 104567897654.876543654; + final double baseValue = 104567897654.87654; final BigDecimal[] decimals = new BigDecimal[4096]; for (int i = 0; i < initialCapacity; i++) { if ((i & 1) == 0) { @@ -1082,13 +1083,13 @@ public void testCopySafeArrow7837() { // to trigger a reallocation of the vector. vc2.setInitialCapacity(/*valueCount*/20, /*density*/0.5); - vc1.setSafe(0, "1234567890".getBytes(Charset.forName("utf-8"))); + vc1.setSafe(0, "1234567890".getBytes(StandardCharsets.UTF_8)); assertFalse(vc1.isNull(0)); - assertEquals(vc1.getObject(0).toString(), "1234567890"); + assertEquals("1234567890", Objects.requireNonNull(vc1.getObject(0)).toString()); vc2.copyFromSafe(0, 0, vc1); assertFalse(vc2.isNull(0)); - assertEquals(vc2.getObject(0).toString(), "1234567890"); + assertEquals("1234567890", Objects.requireNonNull(vc2.getObject(0)).toString()); vc2.copyFromSafe(0, 5, vc1); assertTrue(vc2.isNull(1)); @@ -1096,7 +1097,7 @@ public void testCopySafeArrow7837() { assertTrue(vc2.isNull(3)); assertTrue(vc2.isNull(4)); assertFalse(vc2.isNull(5)); - assertEquals(vc2.getObject(5).toString(), "1234567890"); + assertEquals("1234567890", Objects.requireNonNull(vc2.getObject(5)).toString()); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java index b703959d2bb1e..fc5dfc38587a4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java @@ -40,8 +40,8 @@ public class TestDecimal256Vector { static { intValues = new long[60]; for (int i = 0; i < intValues.length / 2; i++) { - intValues[i] = 1 << i + 1; - intValues[2 * i] = -1 * (1 << i + 1); + intValues[i] = 1L << (i + 1); + intValues[2 * i] = -1L * (1 << (i + 1)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index ba25cbe8b52a0..572f13fea1ed1 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -40,8 +40,8 @@ public class TestDecimalVector { static { intValues = new long[60]; for (int i = 0; i < intValues.length / 2; i++) { - intValues[i] = 1 << i + 1; - intValues[2 * i] = -1 * (1 << i + 1); + intValues[i] = 1L << (i + 1); + intValues[2 * i] = -1L * (1 << (i + 1)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java index 9cb12481612b2..8fd33eb5a8432 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java @@ -349,16 +349,16 @@ public void testGetFieldTypeInfo() throws Exception { assertEquals(vector.getField(), field); // Union has 2 child vectors - assertEquals(vector.size(), 2); + assertEquals(2, vector.size()); // Check child field 0 VectorWithOrdinal intChild = vector.getChildVectorWithOrdinal("int"); - assertEquals(intChild.ordinal, 0); + assertEquals(0, intChild.ordinal); assertEquals(intChild.vector.getField(), children.get(0)); // Check child field 1 VectorWithOrdinal varcharChild = vector.getChildVectorWithOrdinal("varchar"); - assertEquals(varcharChild.ordinal, 1); + assertEquals(1, varcharChild.ordinal); assertEquals(varcharChild.vector.getField(), children.get(1)); } @@ -458,8 +458,8 @@ public void testMultipleStructs() { // register relative types byte typeId1 = unionVector.registerNewTypeId(structVector1.getField()); byte typeId2 = unionVector.registerNewTypeId(structVector2.getField()); - assertEquals(typeId1, 0); - assertEquals(typeId2, 1); + assertEquals(0, typeId1); + assertEquals(1, typeId2); // add two struct vectors to union vector unionVector.addVector(typeId1, structVector1); @@ -519,8 +519,8 @@ public void testMultipleVarChars() { byte typeId1 = unionVector.registerNewTypeId(childVector1.getField()); byte typeId2 = unionVector.registerNewTypeId(childVector2.getField()); - assertEquals(typeId1, 0); - assertEquals(typeId2, 1); + assertEquals(0, typeId1); + assertEquals(1, typeId2); while (unionVector.getValueCapacity() < 5) { unionVector.reAlloc(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 501059733c616..9ffa79470eeb8 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -552,7 +552,7 @@ public void testEncodeWithEncoderInstance() { // now run through the decoder and verify we get the original back try (ValueVector decoded = encoder.decode(encoded)) { assertEquals(vector.getClass(), decoded.getClass()); - assertEquals(vector.getValueCount(), (decoded).getValueCount()); + assertEquals(vector.getValueCount(), decoded.getValueCount()); for (int i = 0; i < 5; i++) { assertEquals(vector.getObject(i), ((VarCharVector) decoded).getObject(i)); } @@ -591,7 +591,7 @@ public void testEncodeMultiVectors() { // now run through the decoder and verify we get the original back try (ValueVector decoded = encoder.decode(encoded)) { assertEquals(vector1.getClass(), decoded.getClass()); - assertEquals(vector1.getValueCount(), (decoded).getValueCount()); + assertEquals(vector1.getValueCount(), decoded.getValueCount()); for (int i = 0; i < 5; i++) { assertEquals(vector1.getObject(i), ((VarCharVector) decoded).getObject(i)); } @@ -611,7 +611,7 @@ public void testEncodeMultiVectors() { // now run through the decoder and verify we get the original back try (ValueVector decoded = encoder.decode(encoded)) { assertEquals(vector2.getClass(), decoded.getClass()); - assertEquals(vector2.getValueCount(), (decoded).getValueCount()); + assertEquals(vector2.getValueCount(), decoded.getValueCount()); for (int i = 0; i < 3; i++) { assertEquals(vector2.getObject(i), ((VarCharVector) decoded).getObject(i)); } @@ -841,7 +841,8 @@ public void testEncodeStructSubFieldWithCertainColumns() { // initialize dictionaries DictionaryProvider.MapDictionaryProvider provider = new DictionaryProvider.MapDictionaryProvider(); - setVector(dictVector1, "aa".getBytes(), "bb".getBytes(), "cc".getBytes(), "dd".getBytes()); + setVector(dictVector1, "aa".getBytes(StandardCharsets.UTF_8), "bb".getBytes(StandardCharsets.UTF_8), + "cc".getBytes(StandardCharsets.UTF_8), "dd".getBytes(StandardCharsets.UTF_8)); provider.put(new Dictionary(dictVector1, new DictionaryEncoding(1L, false, null))); StructSubfieldEncoder encoder = new StructSubfieldEncoder(allocator, provider); @@ -1049,20 +1050,20 @@ private void testDictionary(Dictionary dictionary, ToIntBiFunction ((UInt2Vector) vector).get(index)); } } @@ -1096,7 +1097,7 @@ public void testDictionaryUInt4() { setVector(dictionaryVector, "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"); Dictionary dictionary4 = new Dictionary(dictionaryVector, new DictionaryEncoding(/*id=*/30L, /*ordered=*/false, - /*indexType=*/new ArrowType.Int(/*indexType=*/32, /*isSigned*/false))); + /*indexType=*/new ArrowType.Int(/*bitWidth=*/32, /*isSigned*/false))); testDictionary(dictionary4, (vector, index) -> ((UInt4Vector) vector).get(index)); } } @@ -1107,7 +1108,7 @@ public void testDictionaryUInt8() { setVector(dictionaryVector, "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"); Dictionary dictionary8 = new Dictionary(dictionaryVector, new DictionaryEncoding(/*id=*/40L, /*ordered=*/false, - /*indexType=*/new ArrowType.Int(/*indexType=*/64, /*isSigned*/false))); + /*indexType=*/new ArrowType.Int(/*bitWidth=*/64, /*isSigned*/false))); testDictionary(dictionary8, (vector, index) -> (int) ((UInt8Vector) vector).get(index)); } } @@ -1119,13 +1120,13 @@ public void testDictionaryUIntOverflow() { try (VarCharVector dictionaryVector = new VarCharVector("dict vector", allocator)) { dictionaryVector.allocateNew(vecLength * 3, vecLength); for (int i = 0; i < vecLength; i++) { - dictionaryVector.set(i, String.valueOf(i).getBytes()); + dictionaryVector.set(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } dictionaryVector.setValueCount(vecLength); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(/*id=*/10L, /*ordered=*/false, - /*indexType=*/new ArrowType.Int(/*indexType=*/8, /*isSigned*/false))); + /*indexType=*/new ArrowType.Int(/*bitWidth=*/8, /*isSigned*/false))); try (VarCharVector vector = new VarCharVector("vector", allocator)) { setVector(vector, "255"); @@ -1137,7 +1138,7 @@ public void testDictionaryUIntOverflow() { try (VarCharVector decodedVector = (VarCharVector) DictionaryEncoder.decode(encodedVector, dictionary)) { assertEquals(1, decodedVector.getValueCount()); - assertArrayEquals("255".getBytes(), decodedVector.get(0)); + assertArrayEquals("255".getBytes(StandardCharsets.UTF_8), decodedVector.get(0)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index 0023b1dddb8e7..bde6dd491dd71 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -25,6 +25,7 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -61,7 +62,7 @@ public void terminate() throws Exception { @Test public void testIntType() { - try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { + try (FixedSizeListVector vector = FixedSizeListVector.empty("list", /*size=*/2, allocator)) { IntVector nested = (IntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector(); vector.allocateNew(); @@ -88,7 +89,7 @@ public void testIntType() { @Test public void testFloatTypeNullable() { - try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) { + try (FixedSizeListVector vector = FixedSizeListVector.empty("list", /*size=*/2, allocator)) { Float4Vector nested = (Float4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())) .getVector(); vector.allocateNew(); @@ -235,7 +236,7 @@ public void testTransferPair() { @Test public void testConsistentChildName() throws Exception { - try (FixedSizeListVector listVector = FixedSizeListVector.empty("sourceVector", 2, allocator)) { + try (FixedSizeListVector listVector = FixedSizeListVector.empty("sourceVector", /*size=*/2, allocator)) { String emptyListStr = listVector.getField().toString(); Assert.assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME)); @@ -251,7 +252,7 @@ public void testUnionFixedSizeListWriterWithNulls() throws Exception { * each list of size 3 and having its data values alternating between null and a non-null. * Read and verify */ - try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/3, allocator)) { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*size=*/3, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -279,7 +280,7 @@ public void testUnionFixedSizeListWriterWithNulls() throws Exception { @Test public void testUnionFixedSizeListWriter() throws Exception { - try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", 3, allocator)) { + try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", /*size=*/3, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -307,7 +308,7 @@ public void testUnionFixedSizeListWriter() throws Exception { @Test public void testWriteDecimal() throws Exception { - try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/3, allocator)) { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*size=*/3, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -335,7 +336,7 @@ public void testWriteDecimal() throws Exception { @Test public void testDecimalIndexCheck() throws Exception { - try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/3, allocator)) { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*size=*/3, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -355,7 +356,7 @@ public void testDecimalIndexCheck() throws Exception { @Test(expected = IllegalStateException.class) public void testWriteIllegalData() throws Exception { - try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", 3, allocator)) { + try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", /*size=*/3, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -378,7 +379,7 @@ public void testWriteIllegalData() throws Exception { @Test public void testSplitAndTransfer() throws Exception { - try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", 3, allocator)) { + try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", /*size=*/3, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -399,9 +400,9 @@ public void testSplitAndTransfer() throws Exception { assertEquals(2, targetVector.getValueCount()); int[] realValue1 = convertListToIntArray(targetVector.getObject(0)); - assertTrue(Arrays.equals(values1, realValue1)); + assertArrayEquals(values1, realValue1); int[] realValue2 = convertListToIntArray(targetVector.getObject(1)); - assertTrue(Arrays.equals(values2, realValue2)); + assertArrayEquals(values2, realValue2); targetVector.clear(); } @@ -409,7 +410,7 @@ public void testSplitAndTransfer() throws Exception { @Test public void testZeroWidthVector() { - try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", 0, allocator)) { + try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", /*size=*/0, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -440,7 +441,7 @@ public void testZeroWidthVector() { @Test public void testVectorWithNulls() { - try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", 4, allocator)) { + try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", /*size=*/4, allocator)) { UnionFixedSizeListWriter writer1 = vector1.getWriter(); writer1.allocate(); @@ -472,7 +473,7 @@ public void testVectorWithNulls() { @Test public void testWriteVarCharHelpers() throws Exception { - try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/4, allocator)) { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*size=*/4, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -491,7 +492,7 @@ public void testWriteVarCharHelpers() throws Exception { @Test public void testWriteLargeVarCharHelpers() throws Exception { - try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/4, allocator)) { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*size=*/4, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); @@ -510,43 +511,47 @@ public void testWriteLargeVarCharHelpers() throws Exception { @Test public void testWriteVarBinaryHelpers() throws Exception { - try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/4, allocator)) { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*size=*/4, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); writer.startList(); - writer.writeVarBinary("row1,1".getBytes()); - writer.writeVarBinary("row1,2".getBytes(), 0, "row1,2".getBytes().length); - writer.writeVarBinary(ByteBuffer.wrap("row1,3".getBytes())); - writer.writeVarBinary(ByteBuffer.wrap("row1,4".getBytes()), 0, "row1,4".getBytes().length); + writer.writeVarBinary("row1,1".getBytes(StandardCharsets.UTF_8)); + writer.writeVarBinary("row1,2".getBytes(StandardCharsets.UTF_8), 0, + "row1,2".getBytes(StandardCharsets.UTF_8).length); + writer.writeVarBinary(ByteBuffer.wrap("row1,3".getBytes(StandardCharsets.UTF_8))); + writer.writeVarBinary(ByteBuffer.wrap("row1,4".getBytes(StandardCharsets.UTF_8)), 0, + "row1,4".getBytes(StandardCharsets.UTF_8).length); writer.endList(); - assertEquals("row1,1", new String((byte[]) (vector.getObject(0).get(0)))); - assertEquals("row1,2", new String((byte[]) (vector.getObject(0).get(1)))); - assertEquals("row1,3", new String((byte[]) (vector.getObject(0).get(2)))); - assertEquals("row1,4", new String((byte[]) (vector.getObject(0).get(3)))); + assertEquals("row1,1", new String((byte[]) vector.getObject(0).get(0), StandardCharsets.UTF_8)); + assertEquals("row1,2", new String((byte[]) vector.getObject(0).get(1), StandardCharsets.UTF_8)); + assertEquals("row1,3", new String((byte[]) vector.getObject(0).get(2), StandardCharsets.UTF_8)); + assertEquals("row1,4", new String((byte[]) vector.getObject(0).get(3), StandardCharsets.UTF_8)); } } @Test public void testWriteLargeVarBinaryHelpers() throws Exception { - try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*listSize=*/4, allocator)) { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("vector", /*size=*/4, allocator)) { UnionFixedSizeListWriter writer = vector.getWriter(); writer.allocate(); writer.startList(); - writer.writeLargeVarBinary("row1,1".getBytes()); - writer.writeLargeVarBinary("row1,2".getBytes(), 0, "row1,2".getBytes().length); - writer.writeLargeVarBinary(ByteBuffer.wrap("row1,3".getBytes())); - writer.writeLargeVarBinary(ByteBuffer.wrap("row1,4".getBytes()), 0, "row1,4".getBytes().length); + writer.writeLargeVarBinary("row1,1".getBytes(StandardCharsets.UTF_8)); + writer.writeLargeVarBinary("row1,2".getBytes(StandardCharsets.UTF_8), 0, + "row1,2".getBytes(StandardCharsets.UTF_8).length); + writer.writeLargeVarBinary(ByteBuffer.wrap("row1,3".getBytes(StandardCharsets.UTF_8))); + writer.writeLargeVarBinary(ByteBuffer.wrap("row1,4".getBytes(StandardCharsets.UTF_8)), 0, + "row1,4".getBytes(StandardCharsets.UTF_8).length); writer.endList(); - assertEquals("row1,1", new String((byte[]) (vector.getObject(0).get(0)))); - assertEquals("row1,2", new String((byte[]) (vector.getObject(0).get(1)))); - assertEquals("row1,3", new String((byte[]) (vector.getObject(0).get(2)))); - assertEquals("row1,4", new String((byte[]) (vector.getObject(0).get(3)))); + assertEquals("row1,1", new String((byte[]) vector.getObject(0).get(0), StandardCharsets.UTF_8)); + assertEquals("row1,2", new String((byte[]) vector.getObject(0).get(1), StandardCharsets.UTF_8)); + assertEquals("row1,3", new String((byte[]) vector.getObject(0).get(2), StandardCharsets.UTF_8)); + assertEquals("row1,4", new String((byte[]) vector.getObject(0).get(3), StandardCharsets.UTF_8)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 993ce0b089769..ffd87c99d508d 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -102,9 +102,9 @@ public void testCopyFrom() throws Exception { Object result = outVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); - assertEquals(new Long(1), resultSet.get(0)); - assertEquals(new Long(2), resultSet.get(1)); - assertEquals(new Long(3), resultSet.get(2)); + assertEquals(Long.valueOf(1), resultSet.get(0)); + assertEquals(Long.valueOf(2), resultSet.get(1)); + assertEquals(Long.valueOf(3), resultSet.get(2)); /* index 1 */ result = outVector.getObject(1); @@ -143,7 +143,7 @@ public void testSetLastSetUsage() throws Exception { assertEquals(-1L, listVector.getLastSet()); int index = 0; - int offset = 0; + int offset; /* write [10, 11, 12] to the list vector at index 0 */ BitVectorHelper.setBit(validityBuffer, index); @@ -222,41 +222,40 @@ public void testSetLastSetUsage() throws Exception { assertEquals(Integer.toString(0), Integer.toString(offset)); Long actual = dataVector.getObject(offset); - assertEquals(new Long(10), actual); + assertEquals(Long.valueOf(10), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(11), actual); + assertEquals(Long.valueOf(11), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(12), actual); + assertEquals(Long.valueOf(12), actual); index++; offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH); assertEquals(Integer.toString(3), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(13), actual); + assertEquals(Long.valueOf(13), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(14), actual); + assertEquals(Long.valueOf(14), actual); index++; offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH); assertEquals(Integer.toString(5), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(15), actual); + assertEquals(Long.valueOf(15), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(16), actual); + assertEquals(Long.valueOf(16), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(17), actual); + assertEquals(Long.valueOf(17), actual); index++; offset = (int) offsetBuffer.getLong(index * LargeListVector.OFFSET_WIDTH); assertEquals(Integer.toString(8), Integer.toString(offset)); - actual = dataVector.getObject(offset); assertNull(actual); } @@ -323,8 +322,8 @@ public void testSplitAndTransfer() throws Exception { /* check the vector output */ int index = 0; - int offset = 0; - Long actual = null; + int offset; + Long actual; /* index 0 */ assertFalse(listVector.isNull(index)); @@ -332,13 +331,13 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(0), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(10), actual); + assertEquals(Long.valueOf(10), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(11), actual); + assertEquals(Long.valueOf(11), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(12), actual); + assertEquals(Long.valueOf(12), actual); /* index 1 */ index++; @@ -347,10 +346,10 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(3), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(13), actual); + assertEquals(Long.valueOf(13), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(14), actual); + assertEquals(Long.valueOf(14), actual); /* index 2 */ index++; @@ -359,16 +358,16 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(5), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(15), actual); + assertEquals(Long.valueOf(15), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(16), actual); + assertEquals(Long.valueOf(16), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(17), actual); + assertEquals(Long.valueOf(17), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(18), actual); + assertEquals(Long.valueOf(18), actual); /* index 3 */ index++; @@ -377,7 +376,7 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(9), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(19), actual); + assertEquals(Long.valueOf(19), actual); /* index 4 */ index++; @@ -386,16 +385,16 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(10), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(20), actual); + assertEquals(Long.valueOf(20), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(21), actual); + assertEquals(Long.valueOf(21), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(22), actual); + assertEquals(Long.valueOf(22), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(23), actual); + assertEquals(Long.valueOf(23), actual); /* index 5 */ index++; @@ -522,15 +521,15 @@ public void testNestedLargeListVector() throws Exception { assertEquals(4, resultSet.get(1).size()); /* size of second inner list */ list = resultSet.get(0); - assertEquals(new Long(50), list.get(0)); - assertEquals(new Long(100), list.get(1)); - assertEquals(new Long(200), list.get(2)); + assertEquals(Long.valueOf(50), list.get(0)); + assertEquals(Long.valueOf(100), list.get(1)); + assertEquals(Long.valueOf(200), list.get(2)); list = resultSet.get(1); - assertEquals(new Long(75), list.get(0)); - assertEquals(new Long(125), list.get(1)); - assertEquals(new Long(150), list.get(2)); - assertEquals(new Long(175), list.get(3)); + assertEquals(Long.valueOf(75), list.get(0)); + assertEquals(Long.valueOf(125), list.get(1)); + assertEquals(Long.valueOf(150), list.get(2)); + assertEquals(Long.valueOf(175), list.get(3)); /* get listVector value at index 1 -- the value itself is a listvector */ result = listVector.getObject(1); @@ -542,16 +541,16 @@ public void testNestedLargeListVector() throws Exception { assertEquals(3, resultSet.get(2).size()); /* size of third inner list */ list = resultSet.get(0); - assertEquals(new Long(10), list.get(0)); + assertEquals(Long.valueOf(10), list.get(0)); list = resultSet.get(1); - assertEquals(new Long(15), list.get(0)); - assertEquals(new Long(20), list.get(1)); + assertEquals(Long.valueOf(15), list.get(0)); + assertEquals(Long.valueOf(20), list.get(1)); list = resultSet.get(2); - assertEquals(new Long(25), list.get(0)); - assertEquals(new Long(30), list.get(1)); - assertEquals(new Long(35), list.get(2)); + assertEquals(Long.valueOf(25), list.get(0)); + assertEquals(Long.valueOf(30), list.get(1)); + assertEquals(Long.valueOf(35), list.get(2)); /* check underlying bitVector */ assertFalse(listVector.isNull(0)); @@ -656,13 +655,13 @@ public void testNestedLargeListVector2() throws Exception { assertEquals(2, resultSet.get(1).size()); /* size of second inner list */ list = resultSet.get(0); - assertEquals(new Long(50), list.get(0)); - assertEquals(new Long(100), list.get(1)); - assertEquals(new Long(200), list.get(2)); + assertEquals(Long.valueOf(50), list.get(0)); + assertEquals(Long.valueOf(100), list.get(1)); + assertEquals(Long.valueOf(200), list.get(2)); list = resultSet.get(1); - assertEquals(new Long(75), list.get(0)); - assertEquals(new Long(125), list.get(1)); + assertEquals(Long.valueOf(75), list.get(0)); + assertEquals(Long.valueOf(125), list.get(1)); /* get listVector value at index 1 -- the value itself is a listvector */ result = listVector.getObject(1); @@ -673,13 +672,13 @@ public void testNestedLargeListVector2() throws Exception { assertEquals(3, resultSet.get(1).size()); /* size of second inner list */ list = resultSet.get(0); - assertEquals(new Long(15), list.get(0)); - assertEquals(new Long(20), list.get(1)); + assertEquals(Long.valueOf(15), list.get(0)); + assertEquals(Long.valueOf(20), list.get(1)); list = resultSet.get(1); - assertEquals(new Long(25), list.get(0)); - assertEquals(new Long(30), list.get(1)); - assertEquals(new Long(35), list.get(2)); + assertEquals(Long.valueOf(25), list.get(0)); + assertEquals(Long.valueOf(30), list.get(1)); + assertEquals(Long.valueOf(35), list.get(2)); /* check underlying bitVector */ assertFalse(listVector.isNull(0)); @@ -723,15 +722,15 @@ public void testGetBufferAddress() throws Exception { Object result = listVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); - assertEquals(new Long(50), resultSet.get(0)); - assertEquals(new Long(100), resultSet.get(1)); - assertEquals(new Long(200), resultSet.get(2)); + assertEquals(Long.valueOf(50), resultSet.get(0)); + assertEquals(Long.valueOf(100), resultSet.get(1)); + assertEquals(Long.valueOf(200), resultSet.get(2)); result = listVector.getObject(1); resultSet = (ArrayList) result; assertEquals(2, resultSet.size()); - assertEquals(new Long(250), resultSet.get(0)); - assertEquals(new Long(300), resultSet.get(1)); + assertEquals(Long.valueOf(250), resultSet.get(0)); + assertEquals(Long.valueOf(300), resultSet.get(1)); List buffers = listVector.getFieldBuffers(); @@ -739,7 +738,7 @@ public void testGetBufferAddress() throws Exception { long offsetAddress = listVector.getOffsetBufferAddress(); try { - long dataAddress = listVector.getDataBufferAddress(); + listVector.getDataBufferAddress(); } catch (UnsupportedOperationException ue) { error = true; } finally { @@ -849,11 +848,11 @@ public void testClearAndReuse() { Object result = vector.getObject(0); ArrayList resultSet = (ArrayList) result; - assertEquals(new Long(7), resultSet.get(0)); + assertEquals(Long.valueOf(7), resultSet.get(0)); result = vector.getObject(1); resultSet = (ArrayList) result; - assertEquals(new Long(8), resultSet.get(0)); + assertEquals(Long.valueOf(8), resultSet.get(0)); // Clear and release the buffers to trigger a realloc when adding next value vector.clear(); @@ -869,11 +868,11 @@ public void testClearAndReuse() { result = vector.getObject(0); resultSet = (ArrayList) result; - assertEquals(new Long(7), resultSet.get(0)); + assertEquals(Long.valueOf(7), resultSet.get(0)); result = vector.getObject(1); resultSet = (ArrayList) result; - assertEquals(new Long(8), resultSet.get(0)); + assertEquals(Long.valueOf(8), resultSet.get(0)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java index ecababde8de3a..36607903b01a2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java @@ -22,7 +22,9 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Objects; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -61,7 +63,7 @@ public void testSetNullableLargeVarBinaryHolder() { String str = "hello"; try (ArrowBuf buf = allocator.buffer(16)) { - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); binHolder.start = 0; binHolder.end = str.length(); @@ -72,7 +74,7 @@ public void testSetNullableLargeVarBinaryHolder() { // verify results assertTrue(vector.isNull(0)); - assertEquals(str, new String(vector.get(1))); + assertEquals(str, new String(Objects.requireNonNull(vector.get(1)), StandardCharsets.UTF_8)); } } } @@ -90,7 +92,7 @@ public void testSetNullableLargeVarBinaryHolderSafe() { String str = "hello world"; try (ArrowBuf buf = allocator.buffer(16)) { - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); binHolder.start = 0; binHolder.end = str.length(); @@ -100,7 +102,7 @@ public void testSetNullableLargeVarBinaryHolderSafe() { vector.setSafe(1, nullHolder); // verify results - assertEquals(str, new String(vector.get(0))); + assertEquals(str, new String(Objects.requireNonNull(vector.get(0)), StandardCharsets.UTF_8)); assertTrue(vector.isNull(1)); } } @@ -113,18 +115,18 @@ public void testGetBytesRepeatedly() { final String str = "hello world"; final String str2 = "foo"; - vector.setSafe(0, str.getBytes()); - vector.setSafe(1, str2.getBytes()); + vector.setSafe(0, str.getBytes(StandardCharsets.UTF_8)); + vector.setSafe(1, str2.getBytes(StandardCharsets.UTF_8)); // verify results ReusableByteArray reusableByteArray = new ReusableByteArray(); vector.read(0, reusableByteArray); byte[] oldBuffer = reusableByteArray.getBuffer(); - assertArrayEquals(str.getBytes(), Arrays.copyOfRange(reusableByteArray.getBuffer(), + assertArrayEquals(str.getBytes(StandardCharsets.UTF_8), Arrays.copyOfRange(reusableByteArray.getBuffer(), 0, (int) reusableByteArray.getLength())); vector.read(1, reusableByteArray); - assertArrayEquals(str2.getBytes(), Arrays.copyOfRange(reusableByteArray.getBuffer(), + assertArrayEquals(str2.getBytes(StandardCharsets.UTF_8), Arrays.copyOfRange(reusableByteArray.getBuffer(), 0, (int) reusableByteArray.getLength())); // There should not have been any reallocation since the newer value is smaller in length. @@ -137,7 +139,7 @@ public void testGetTransferPairWithField() { try (BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 1000000, 1000000); LargeVarBinaryVector v1 = new LargeVarBinaryVector("v1", childAllocator1)) { v1.allocateNew(); - v1.setSafe(4094, "hello world".getBytes(), 0, 11); + v1.setSafe(4094, "hello world".getBytes(StandardCharsets.UTF_8), 0, 11); v1.setValueCount(4001); TransferPair tp = v1.getTransferPair(v1.getField(), allocator); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java index 7d074c393648f..62d09da86d652 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java @@ -27,6 +27,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -48,12 +49,12 @@ public class TestLargeVarCharVector { - private static final byte[] STR1 = "AAAAA1".getBytes(); - private static final byte[] STR2 = "BBBBBBBBB2".getBytes(); - private static final byte[] STR3 = "CCCC3".getBytes(); - private static final byte[] STR4 = "DDDDDDDD4".getBytes(); - private static final byte[] STR5 = "EEE5".getBytes(); - private static final byte[] STR6 = "FFFFF6".getBytes(); + private static final byte[] STR1 = "AAAAA1".getBytes(StandardCharsets.UTF_8); + private static final byte[] STR2 = "BBBBBBBBB2".getBytes(StandardCharsets.UTF_8); + private static final byte[] STR3 = "CCCC3".getBytes(StandardCharsets.UTF_8); + private static final byte[] STR4 = "DDDDDDDD4".getBytes(StandardCharsets.UTF_8); + private static final byte[] STR5 = "EEE5".getBytes(StandardCharsets.UTF_8); + private static final byte[] STR6 = "FFFFF6".getBytes(StandardCharsets.UTF_8); private BufferAllocator allocator; @@ -74,7 +75,7 @@ public void testTransfer() { LargeVarCharVector v1 = new LargeVarCharVector("v1", childAllocator1); LargeVarCharVector v2 = new LargeVarCharVector("v2", childAllocator2);) { v1.allocateNew(); - v1.setSafe(4094, "hello world".getBytes(), 0, 11); + v1.setSafe(4094, "hello world".getBytes(StandardCharsets.UTF_8), 0, 11); v1.setValueCount(4001); long memoryBeforeTransfer = childAllocator1.getAllocatedMemory(); @@ -207,12 +208,12 @@ public void testSizeOfValueBuffer() { @Test public void testSetLastSetUsage() { - final byte[] STR1 = "AAAAA1".getBytes(); - final byte[] STR2 = "BBBBBBBBB2".getBytes(); - final byte[] STR3 = "CCCC3".getBytes(); - final byte[] STR4 = "DDDDDDDD4".getBytes(); - final byte[] STR5 = "EEE5".getBytes(); - final byte[] STR6 = "FFFFF6".getBytes(); + final byte[] STR1 = "AAAAA1".getBytes(StandardCharsets.UTF_8); + final byte[] STR2 = "BBBBBBBBB2".getBytes(StandardCharsets.UTF_8); + final byte[] STR3 = "CCCC3".getBytes(StandardCharsets.UTF_8); + final byte[] STR4 = "DDDDDDDD4".getBytes(StandardCharsets.UTF_8); + final byte[] STR5 = "EEE5".getBytes(StandardCharsets.UTF_8); + final byte[] STR6 = "FFFFF6".getBytes(StandardCharsets.UTF_8); try (final LargeVarCharVector vector = new LargeVarCharVector("myvector", allocator)) { vector.allocateNew(1024 * 10, 1024); @@ -353,7 +354,7 @@ public void testSplitAndTransfer() { for (int i = 0; i < length; i++) { final boolean expectedSet = ((start + i) % 3) == 0; if (expectedSet) { - final byte[] expectedValue = compareArray[start + i].getBytes(); + final byte[] expectedValue = compareArray[start + i].getBytes(StandardCharsets.UTF_8); assertFalse(newLargeVarCharVector.isNull(i)); assertArrayEquals(expectedValue, newLargeVarCharVector.get(i)); } else { @@ -367,8 +368,8 @@ public void testSplitAndTransfer() { @Test public void testReallocAfterVectorTransfer() { - final byte[] STR1 = "AAAAA1".getBytes(); - final byte[] STR2 = "BBBBBBBBB2".getBytes(); + final byte[] STR1 = "AAAAA1".getBytes(StandardCharsets.UTF_8); + final byte[] STR2 = "BBBBBBBBB2".getBytes(StandardCharsets.UTF_8); try (final LargeVarCharVector vector = new LargeVarCharVector("vector", allocator)) { /* 4096 values with 10 byte per record */ @@ -675,7 +676,7 @@ public void testSetNullableLargeVarCharHolder() { String str = "hello"; ArrowBuf buf = allocator.buffer(16); - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); stringHolder.start = 0; stringHolder.end = str.length(); @@ -686,7 +687,7 @@ public void testSetNullableLargeVarCharHolder() { // verify results assertTrue(vector.isNull(0)); - assertEquals(str, new String(vector.get(1))); + assertEquals(str, new String(Objects.requireNonNull(vector.get(1)), StandardCharsets.UTF_8)); buf.close(); } @@ -705,7 +706,7 @@ public void testSetNullableLargeVarCharHolderSafe() { String str = "hello world"; ArrowBuf buf = allocator.buffer(16); - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); stringHolder.start = 0; stringHolder.end = str.length(); @@ -715,7 +716,7 @@ public void testSetNullableLargeVarCharHolderSafe() { vector.setSafe(1, nullHolder); // verify results - assertEquals(str, new String(vector.get(0))); + assertEquals(str, new String(Objects.requireNonNull(vector.get(0)), StandardCharsets.UTF_8)); assertTrue(vector.isNull(1)); buf.close(); @@ -743,7 +744,7 @@ public void testLargeVariableWidthVectorNullHashCode() { largeVarChVec.allocateNew(100, 1); largeVarChVec.setValueCount(1); - largeVarChVec.set(0, "abc".getBytes()); + largeVarChVec.set(0, "abc".getBytes(StandardCharsets.UTF_8)); largeVarChVec.setNull(0); assertEquals(0, largeVarChVec.hashCode(0)); @@ -756,7 +757,7 @@ public void testUnloadLargeVariableWidthVector() { largeVarCharVector.allocateNew(5, 2); largeVarCharVector.setValueCount(2); - largeVarCharVector.set(0, "abcd".getBytes()); + largeVarCharVector.set(0, "abcd".getBytes(StandardCharsets.UTF_8)); List bufs = largeVarCharVector.getFieldBuffers(); assertEquals(3, bufs.size()); @@ -821,7 +822,7 @@ public void testGetTransferPairWithField() { try (BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 1000000, 1000000); LargeVarCharVector v1 = new LargeVarCharVector("v1", childAllocator1)) { v1.allocateNew(); - v1.setSafe(4094, "hello world".getBytes(), 0, 11); + v1.setSafe(4094, "hello world".getBytes(StandardCharsets.UTF_8), 0, 11); v1.setValueCount(4001); TransferPair tp = v1.getTransferPair(v1.getField(), allocator); @@ -835,7 +836,7 @@ public void testGetTransferPairWithField() { private void populateLargeVarcharVector(final LargeVarCharVector vector, int valueCount, String[] values) { for (int i = 0; i < valueCount; i += 3) { final String s = String.format("%010d", i); - vector.set(i, s.getBytes()); + vector.set(i, s.getBytes(StandardCharsets.UTF_8)); if (values != null) { values[i] = s; } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 278f497b47991..97f2d9fd6def1 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -107,9 +107,9 @@ public void testCopyFrom() throws Exception { Object result = outVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); - assertEquals(new Long(1), (Long) resultSet.get(0)); - assertEquals(new Long(2), (Long) resultSet.get(1)); - assertEquals(new Long(3), (Long) resultSet.get(2)); + assertEquals(Long.valueOf(1), resultSet.get(0)); + assertEquals(Long.valueOf(2), resultSet.get(1)); + assertEquals(Long.valueOf(3), resultSet.get(2)); /* index 1 */ result = outVector.getObject(1); @@ -148,7 +148,7 @@ public void testSetLastSetUsage() throws Exception { assertEquals(Integer.toString(-1), Integer.toString(listVector.getLastSet())); int index = 0; - int offset = 0; + int offset; /* write [10, 11, 12] to the list vector at index 0 */ BitVectorHelper.setBit(validityBuffer, index); @@ -227,36 +227,36 @@ public void testSetLastSetUsage() throws Exception { assertEquals(Integer.toString(0), Integer.toString(offset)); Long actual = dataVector.getObject(offset); - assertEquals(new Long(10), actual); + assertEquals(Long.valueOf(10), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(11), actual); + assertEquals(Long.valueOf(11), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(12), actual); + assertEquals(Long.valueOf(12), actual); index++; offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(3), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(13), actual); + assertEquals(Long.valueOf(13), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(14), actual); + assertEquals(Long.valueOf(14), actual); index++; offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); assertEquals(Integer.toString(5), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(15), actual); + assertEquals(Long.valueOf(15), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(16), actual); + assertEquals(Long.valueOf(16), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(17), actual); + assertEquals(Long.valueOf(17), actual); index++; offset = offsetBuffer.getInt(index * ListVector.OFFSET_WIDTH); @@ -328,8 +328,8 @@ public void testSplitAndTransfer() throws Exception { /* check the vector output */ int index = 0; - int offset = 0; - Long actual = null; + int offset; + Long actual; /* index 0 */ assertFalse(listVector.isNull(index)); @@ -337,13 +337,13 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(0), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(10), actual); + assertEquals(Long.valueOf(10), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(11), actual); + assertEquals(Long.valueOf(11), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(12), actual); + assertEquals(Long.valueOf(12), actual); /* index 1 */ index++; @@ -352,10 +352,10 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(3), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(13), actual); + assertEquals(Long.valueOf(13), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(14), actual); + assertEquals(Long.valueOf(14), actual); /* index 2 */ index++; @@ -364,16 +364,16 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(5), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(15), actual); + assertEquals(Long.valueOf(15), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(16), actual); + assertEquals(Long.valueOf(16), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(17), actual); + assertEquals(Long.valueOf(17), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(18), actual); + assertEquals(Long.valueOf(18), actual); /* index 3 */ index++; @@ -382,7 +382,7 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(9), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(19), actual); + assertEquals(Long.valueOf(19), actual); /* index 4 */ index++; @@ -391,16 +391,16 @@ public void testSplitAndTransfer() throws Exception { assertEquals(Integer.toString(10), Integer.toString(offset)); actual = dataVector.getObject(offset); - assertEquals(new Long(20), actual); + assertEquals(Long.valueOf(20), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(21), actual); + assertEquals(Long.valueOf(21), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(22), actual); + assertEquals(Long.valueOf(22), actual); offset++; actual = dataVector.getObject(offset); - assertEquals(new Long(23), actual); + assertEquals(Long.valueOf(23), actual); /* index 5 */ index++; @@ -527,15 +527,15 @@ public void testNestedListVector() throws Exception { assertEquals(4, resultSet.get(1).size()); /* size of second inner list */ list = resultSet.get(0); - assertEquals(new Long(50), list.get(0)); - assertEquals(new Long(100), list.get(1)); - assertEquals(new Long(200), list.get(2)); + assertEquals(Long.valueOf(50), list.get(0)); + assertEquals(Long.valueOf(100), list.get(1)); + assertEquals(Long.valueOf(200), list.get(2)); list = resultSet.get(1); - assertEquals(new Long(75), list.get(0)); - assertEquals(new Long(125), list.get(1)); - assertEquals(new Long(150), list.get(2)); - assertEquals(new Long(175), list.get(3)); + assertEquals(Long.valueOf(75), list.get(0)); + assertEquals(Long.valueOf(125), list.get(1)); + assertEquals(Long.valueOf(150), list.get(2)); + assertEquals(Long.valueOf(175), list.get(3)); /* get listVector value at index 1 -- the value itself is a listvector */ result = listVector.getObject(1); @@ -547,16 +547,16 @@ public void testNestedListVector() throws Exception { assertEquals(3, resultSet.get(2).size()); /* size of third inner list */ list = resultSet.get(0); - assertEquals(new Long(10), list.get(0)); + assertEquals(Long.valueOf(10), list.get(0)); list = resultSet.get(1); - assertEquals(new Long(15), list.get(0)); - assertEquals(new Long(20), list.get(1)); + assertEquals(Long.valueOf(15), list.get(0)); + assertEquals(Long.valueOf(20), list.get(1)); list = resultSet.get(2); - assertEquals(new Long(25), list.get(0)); - assertEquals(new Long(30), list.get(1)); - assertEquals(new Long(35), list.get(2)); + assertEquals(Long.valueOf(25), list.get(0)); + assertEquals(Long.valueOf(30), list.get(1)); + assertEquals(Long.valueOf(35), list.get(2)); /* check underlying bitVector */ assertFalse(listVector.isNull(0)); @@ -661,13 +661,13 @@ public void testNestedListVector2() throws Exception { assertEquals(2, resultSet.get(1).size()); /* size of second inner list */ list = resultSet.get(0); - assertEquals(new Long(50), list.get(0)); - assertEquals(new Long(100), list.get(1)); - assertEquals(new Long(200), list.get(2)); + assertEquals(Long.valueOf(50), list.get(0)); + assertEquals(Long.valueOf(100), list.get(1)); + assertEquals(Long.valueOf(200), list.get(2)); list = resultSet.get(1); - assertEquals(new Long(75), list.get(0)); - assertEquals(new Long(125), list.get(1)); + assertEquals(Long.valueOf(75), list.get(0)); + assertEquals(Long.valueOf(125), list.get(1)); /* get listVector value at index 1 -- the value itself is a listvector */ result = listVector.getObject(1); @@ -678,13 +678,13 @@ public void testNestedListVector2() throws Exception { assertEquals(3, resultSet.get(1).size()); /* size of second inner list */ list = resultSet.get(0); - assertEquals(new Long(15), list.get(0)); - assertEquals(new Long(20), list.get(1)); + assertEquals(Long.valueOf(15), list.get(0)); + assertEquals(Long.valueOf(20), list.get(1)); list = resultSet.get(1); - assertEquals(new Long(25), list.get(0)); - assertEquals(new Long(30), list.get(1)); - assertEquals(new Long(35), list.get(2)); + assertEquals(Long.valueOf(25), list.get(0)); + assertEquals(Long.valueOf(30), list.get(1)); + assertEquals(Long.valueOf(35), list.get(2)); /* check underlying bitVector */ assertFalse(listVector.isNull(0)); @@ -728,15 +728,15 @@ public void testGetBufferAddress() throws Exception { Object result = listVector.getObject(0); ArrayList resultSet = (ArrayList) result; assertEquals(3, resultSet.size()); - assertEquals(new Long(50), resultSet.get(0)); - assertEquals(new Long(100), resultSet.get(1)); - assertEquals(new Long(200), resultSet.get(2)); + assertEquals(Long.valueOf(50), resultSet.get(0)); + assertEquals(Long.valueOf(100), resultSet.get(1)); + assertEquals(Long.valueOf(200), resultSet.get(2)); result = listVector.getObject(1); resultSet = (ArrayList) result; assertEquals(2, resultSet.size()); - assertEquals(new Long(250), resultSet.get(0)); - assertEquals(new Long(300), resultSet.get(1)); + assertEquals(Long.valueOf(250), resultSet.get(0)); + assertEquals(Long.valueOf(300), resultSet.get(1)); List buffers = listVector.getFieldBuffers(); @@ -744,7 +744,7 @@ public void testGetBufferAddress() throws Exception { long offsetAddress = listVector.getOffsetBufferAddress(); try { - long dataAddress = listVector.getDataBufferAddress(); + listVector.getDataBufferAddress(); } catch (UnsupportedOperationException ue) { error = true; } finally { @@ -777,7 +777,7 @@ public void testSetInitialCapacity() { try (final ListVector vector = ListVector.empty("", allocator)) { vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); - /** + /* * use the default multiplier of 5, * 512 * 5 => 2560 * 4 => 10240 bytes => 16KB => 4096 value capacity. */ @@ -792,7 +792,7 @@ public void testSetInitialCapacity() { assertEquals(512, vector.getValueCapacity()); assertTrue(vector.getDataVector().getValueCapacity() >= 512 * 4); - /** + /* * inner value capacity we pass to data vector is 512 * 0.1 => 51 * For an int vector this is 204 bytes of memory for data buffer * and 7 bytes for validity buffer. @@ -805,7 +805,7 @@ public void testSetInitialCapacity() { assertEquals(512, vector.getValueCapacity()); assertTrue(vector.getDataVector().getValueCapacity() >= 51); - /** + /* * inner value capacity we pass to data vector is 512 * 0.01 => 5 * For an int vector this is 20 bytes of memory for data buffer * and 1 byte for validity buffer. @@ -818,7 +818,7 @@ public void testSetInitialCapacity() { assertEquals(512, vector.getValueCapacity()); assertTrue(vector.getDataVector().getValueCapacity() >= 5); - /** + /* * inner value capacity we pass to data vector is 5 * 0.1 => 0 * which is then rounded off to 1. So we pass value count as 1 * to the inner int vector. @@ -854,11 +854,11 @@ public void testClearAndReuse() { Object result = vector.getObject(0); ArrayList resultSet = (ArrayList) result; - assertEquals(new Long(7), resultSet.get(0)); + assertEquals(Long.valueOf(7), resultSet.get(0)); result = vector.getObject(1); resultSet = (ArrayList) result; - assertEquals(new Long(8), resultSet.get(0)); + assertEquals(Long.valueOf(8), resultSet.get(0)); // Clear and release the buffers to trigger a realloc when adding next value vector.clear(); @@ -874,11 +874,11 @@ public void testClearAndReuse() { result = vector.getObject(0); resultSet = (ArrayList) result; - assertEquals(new Long(7), resultSet.get(0)); + assertEquals(Long.valueOf(7), resultSet.get(0)); result = vector.getObject(1); resultSet = (ArrayList) result; - assertEquals(new Long(8), resultSet.get(0)); + assertEquals(Long.valueOf(8), resultSet.get(0)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java index 1db55198e4bb3..43f4c3b536fdc 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestMapVector.java @@ -335,8 +335,8 @@ public void testSplitAndTransfer() throws Exception { /* check the vector output */ int index = 0; - int offset = 0; - Map result = null; + int offset; + Map result; /* index 0 */ assertFalse(mapVector.isNull(index)); @@ -571,18 +571,18 @@ public void testMapWithListValue() throws Exception { assertEquals(1L, getResultKey(resultStruct)); ArrayList list = (ArrayList) getResultValue(resultStruct); assertEquals(3, list.size()); // value is a list with 3 elements - assertEquals(new Long(50), list.get(0)); - assertEquals(new Long(100), list.get(1)); - assertEquals(new Long(200), list.get(2)); + assertEquals(Long.valueOf(50), list.get(0)); + assertEquals(Long.valueOf(100), list.get(1)); + assertEquals(Long.valueOf(200), list.get(2)); // Second Map entry resultStruct = (Map) resultSet.get(1); list = (ArrayList) getResultValue(resultStruct); assertEquals(4, list.size()); // value is a list with 4 elements - assertEquals(new Long(75), list.get(0)); - assertEquals(new Long(125), list.get(1)); - assertEquals(new Long(150), list.get(2)); - assertEquals(new Long(175), list.get(3)); + assertEquals(Long.valueOf(75), list.get(0)); + assertEquals(Long.valueOf(125), list.get(1)); + assertEquals(Long.valueOf(150), list.get(2)); + assertEquals(Long.valueOf(175), list.get(3)); // Get mapVector element at index 1 result = mapVector.getObject(1); @@ -593,24 +593,24 @@ public void testMapWithListValue() throws Exception { assertEquals(3L, getResultKey(resultStruct)); list = (ArrayList) getResultValue(resultStruct); assertEquals(1, list.size()); // value is a list with 1 element - assertEquals(new Long(10), list.get(0)); + assertEquals(Long.valueOf(10), list.get(0)); // Second Map entry resultStruct = (Map) resultSet.get(1); assertEquals(4L, getResultKey(resultStruct)); list = (ArrayList) getResultValue(resultStruct); assertEquals(2, list.size()); // value is a list with 1 element - assertEquals(new Long(15), list.get(0)); - assertEquals(new Long(20), list.get(1)); + assertEquals(Long.valueOf(15), list.get(0)); + assertEquals(Long.valueOf(20), list.get(1)); // Third Map entry resultStruct = (Map) resultSet.get(2); assertEquals(5L, getResultKey(resultStruct)); list = (ArrayList) getResultValue(resultStruct); assertEquals(3, list.size()); // value is a list with 1 element - assertEquals(new Long(25), list.get(0)); - assertEquals(new Long(30), list.get(1)); - assertEquals(new Long(35), list.get(2)); + assertEquals(Long.valueOf(25), list.get(0)); + assertEquals(Long.valueOf(30), list.get(1)); + assertEquals(Long.valueOf(35), list.get(2)); /* check underlying bitVector */ assertFalse(mapVector.isNull(0)); @@ -1012,8 +1012,8 @@ public void testMapWithMapKeyAndMapValue() throws Exception { final ArrowBuf offsetBuffer = mapVector.getOffsetBuffer(); /* mapVector has 2 entries at index 0 and 4 entries at index 1 */ - assertEquals(0, offsetBuffer.getInt(0 * MapVector.OFFSET_WIDTH)); - assertEquals(2, offsetBuffer.getInt(1 * MapVector.OFFSET_WIDTH)); + assertEquals(0, offsetBuffer.getInt(0)); + assertEquals(2, offsetBuffer.getInt(MapVector.OFFSET_WIDTH)); assertEquals(6, offsetBuffer.getInt(2 * MapVector.OFFSET_WIDTH)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index 716fa0bde454d..3580a321f01c9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -57,7 +58,7 @@ public void terminate() throws Exception { private void populateVarcharVector(final VarCharVector vector, int valueCount, String[] compareArray) { for (int i = 0; i < valueCount; i += 3) { final String s = String.format("%010d", i); - vector.set(i, s.getBytes()); + vector.set(i, s.getBytes(StandardCharsets.UTF_8)); if (compareArray != null) { compareArray[i] = s; } @@ -86,7 +87,7 @@ public void test() throws Exception { for (int i = 0; i < length; i++) { final boolean expectedSet = ((start + i) % 3) == 0; if (expectedSet) { - final byte[] expectedValue = compareArray[start + i].getBytes(); + final byte[] expectedValue = compareArray[start + i].getBytes(StandardCharsets.UTF_8); assertFalse(newVarCharVector.isNull(i)); assertArrayEquals(expectedValue, newVarCharVector.get(i)); } else { @@ -141,7 +142,7 @@ public void testTransfer() { for (int i = 0; i < valueCount; i++) { final boolean expectedSet = (i % 3) == 0; if (expectedSet) { - final byte[] expectedValue = compareArray[i].getBytes(); + final byte[] expectedValue = compareArray[i].getBytes(StandardCharsets.UTF_8); assertFalse(newVarCharVector.isNull(i)); assertArrayEquals(expectedValue, newVarCharVector.get(i)); } else { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index b53171a597681..1b0387feb73ff 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -404,16 +404,16 @@ public void testGetFieldTypeInfo() throws Exception { assertTrue(vector.getField().equals(field)); // Union has 2 child vectors - assertEquals(vector.size(), 2); + assertEquals(2, vector.size()); // Check child field 0 VectorWithOrdinal intChild = vector.getChildVectorWithOrdinal("int"); - assertEquals(intChild.ordinal, 0); + assertEquals(0, intChild.ordinal); assertEquals(intChild.vector.getField(), children.get(0)); // Check child field 1 VectorWithOrdinal varcharChild = vector.getChildVectorWithOrdinal("varchar"); - assertEquals(varcharChild.ordinal, 1); + assertEquals(1, varcharChild.ordinal); assertEquals(varcharChild.vector.getField(), children.get(1)); } @@ -455,7 +455,7 @@ public void testGetBufferAddress() throws Exception { try { - long offsetAddress = vector.getOffsetBufferAddress(); + vector.getOffsetBufferAddress(); } catch (UnsupportedOperationException ue) { error = true; } finally { @@ -464,7 +464,7 @@ public void testGetBufferAddress() throws Exception { } try { - long dataAddress = vector.getDataBufferAddress(); + vector.getDataBufferAddress(); } catch (UnsupportedOperationException ue) { error = true; } finally { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index fb96870804441..614aff18d4554 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -83,7 +83,7 @@ public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - private static final Charset utf8Charset = Charset.forName("UTF-8"); + private static final Charset utf8Charset = StandardCharsets.UTF_8; private static final byte[] STR1 = "AAAAA1".getBytes(utf8Charset); private static final byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset); private static final byte[] STR3 = "CCCC3".getBytes(utf8Charset); @@ -127,10 +127,9 @@ public void testFixedType1() { try (final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator)) { boolean error = false; - int initialCapacity = 0; vector.allocateNew(1024); - initialCapacity = vector.getValueCapacity(); + int initialCapacity = vector.getValueCapacity(); assertTrue(initialCapacity >= 1024); // Put and set a few values @@ -562,8 +561,6 @@ public void testNullableFixedType1() { assertEquals(103, vector.get(initialCapacity - 2)); assertEquals(104, vector.get(initialCapacity - 1)); - int val = 0; - /* check unset bits/null values */ for (int i = 2, j = 101; i <= 99 || j <= initialCapacity - 3; i++, j++) { if (i <= 99) { @@ -606,8 +603,6 @@ public void testNullableFixedType1() { assertEquals(104, vector.get(initialCapacity - 1)); assertEquals(10000, vector.get(initialCapacity)); - val = 0; - /* check unset bits/null values */ for (int i = 2, j = 101; i < 99 || j < initialCapacity - 3; i++, j++) { if (i <= 99) { @@ -735,7 +730,6 @@ public void testNullableFixedType2() { public void testNullableFixedType3() { // Create a new value vector for 1024 integers try (final IntVector vector = newVector(IntVector.class, EMPTY_SCHEMA_PATH, MinorType.INT, allocator)) { - boolean error = false; int initialCapacity = 1024; /* no memory allocation has happened yet so capacity of underlying buffer should be 0 */ @@ -765,7 +759,6 @@ public void testNullableFixedType3() { } vector.setValueCount(1024); - Field field = vector.getField(); List buffers = vector.getFieldBuffers(); @@ -1105,7 +1098,6 @@ public void testNullableVarType1() { assertEquals(txt, vector.getObject(7)); // Ensure null value throws. - boolean b = false; assertNull(vector.get(8)); } } @@ -1182,18 +1174,18 @@ public void testGetBytesRepeatedly() { final String str = "hello world"; final String str2 = "foo"; - vector.setSafe(0, str.getBytes()); - vector.setSafe(1, str2.getBytes()); + vector.setSafe(0, str.getBytes(StandardCharsets.UTF_8)); + vector.setSafe(1, str2.getBytes(StandardCharsets.UTF_8)); // verify results ReusableByteArray reusableByteArray = new ReusableByteArray(); vector.read(0, reusableByteArray); - assertArrayEquals(str.getBytes(), Arrays.copyOfRange(reusableByteArray.getBuffer(), + assertArrayEquals(str.getBytes(StandardCharsets.UTF_8), Arrays.copyOfRange(reusableByteArray.getBuffer(), 0, (int) reusableByteArray.getLength())); byte[] oldBuffer = reusableByteArray.getBuffer(); vector.read(1, reusableByteArray); - assertArrayEquals(str2.getBytes(), Arrays.copyOfRange(reusableByteArray.getBuffer(), + assertArrayEquals(str2.getBytes(StandardCharsets.UTF_8), Arrays.copyOfRange(reusableByteArray.getBuffer(), 0, (int) reusableByteArray.getLength())); // There should not have been any reallocation since the newer value is smaller in length. @@ -1219,7 +1211,6 @@ public void testGetBytesRepeatedly() { public void testReallocAfterVectorTransfer1() { try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) { int initialCapacity = 4096; - boolean error = false; /* use the default capacity; 4096*8 => 32KB */ vector.setInitialCapacity(initialCapacity); @@ -1259,7 +1250,7 @@ public void testReallocAfterVectorTransfer1() { } /* this should trigger a realloc */ - vector.setSafe(capacityAfterRealloc1, baseValue + (double) (capacityAfterRealloc1)); + vector.setSafe(capacityAfterRealloc1, baseValue + (double) capacityAfterRealloc1); assertTrue(vector.getValueCapacity() >= initialCapacity * 4); int capacityAfterRealloc2 = vector.getValueCapacity(); @@ -1301,7 +1292,6 @@ public void testReallocAfterVectorTransfer1() { public void testReallocAfterVectorTransfer2() { try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) { int initialCapacity = 4096; - boolean error = false; vector.allocateNew(initialCapacity); assertTrue(vector.getValueCapacity() >= initialCapacity); @@ -1338,7 +1328,7 @@ public void testReallocAfterVectorTransfer2() { } /* this should trigger a realloc */ - vector.setSafe(capacityAfterRealloc1, baseValue + (double) (capacityAfterRealloc1)); + vector.setSafe(capacityAfterRealloc1, baseValue + (double) capacityAfterRealloc1); assertTrue(vector.getValueCapacity() >= initialCapacity * 4); int capacityAfterRealloc2 = vector.getValueCapacity(); @@ -1494,7 +1484,6 @@ public void testReallocAfterVectorTransfer4() { assertTrue(valueCapacity >= 4096); /* populate the vector */ - int baseValue = 1000; for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 0) { vector.set(i, 1000 + i); @@ -1649,7 +1638,7 @@ public void testFillEmptiesNotOverfill() { int initialCapacity = vector.getValueCapacity(); assertTrue(initialCapacity >= 4095); - vector.setSafe(4094, "hello".getBytes(), 0, 5); + vector.setSafe(4094, "hello".getBytes(StandardCharsets.UTF_8), 0, 5); /* the above set method should NOT have triggered a realloc */ assertEquals(initialCapacity, vector.getValueCapacity()); @@ -1663,7 +1652,7 @@ public void testFillEmptiesNotOverfill() { @Test public void testSetSafeWithArrowBufNoExcessAllocs() { final int numValues = BaseFixedWidthVector.INITIAL_VALUE_ALLOCATION * 2; - final byte[] valueBytes = "hello world".getBytes(); + final byte[] valueBytes = "hello world".getBytes(StandardCharsets.UTF_8); final int valueBytesLength = valueBytes.length; final int isSet = 1; @@ -1720,7 +1709,7 @@ public void testCopyFromWithNulls() { if (i % 3 == 0) { continue; } - byte[] b = Integer.toString(i).getBytes(); + byte[] b = Integer.toString(i).getBytes(StandardCharsets.UTF_8); vector.setSafe(i, b, 0, b.length); } @@ -1781,7 +1770,7 @@ public void testCopyFromWithNulls1() { if (i % 3 == 0) { continue; } - byte[] b = Integer.toString(i).getBytes(); + byte[] b = Integer.toString(i).getBytes(StandardCharsets.UTF_8); vector.setSafe(i, b, 0, b.length); } @@ -2137,7 +2126,7 @@ public void testGetBufferAddress2() { long dataAddress = vector.getDataBufferAddress(); try { - long offsetAddress = vector.getOffsetBufferAddress(); + vector.getOffsetBufferAddress(); } catch (UnsupportedOperationException ue) { error = true; } finally { @@ -2275,7 +2264,7 @@ public void testSetNullableVarCharHolder() { String str = "hello"; ArrowBuf buf = allocator.buffer(16); - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); stringHolder.start = 0; stringHolder.end = str.length(); @@ -2286,7 +2275,7 @@ public void testSetNullableVarCharHolder() { // verify results assertTrue(vector.isNull(0)); - assertEquals(str, new String(vector.get(1))); + assertEquals(str, new String(vector.get(1), StandardCharsets.UTF_8)); buf.close(); } @@ -2305,7 +2294,7 @@ public void testSetNullableVarCharHolderSafe() { String str = "hello world"; ArrowBuf buf = allocator.buffer(16); - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); stringHolder.start = 0; stringHolder.end = str.length(); @@ -2315,7 +2304,7 @@ public void testSetNullableVarCharHolderSafe() { vector.setSafe(1, nullHolder); // verify results - assertEquals(str, new String(vector.get(0))); + assertEquals(str, new String(vector.get(0), StandardCharsets.UTF_8)); assertTrue(vector.isNull(1)); buf.close(); @@ -2335,7 +2324,7 @@ public void testSetNullableVarBinaryHolder() { String str = "hello"; ArrowBuf buf = allocator.buffer(16); - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); binHolder.start = 0; binHolder.end = str.length(); @@ -2346,7 +2335,7 @@ public void testSetNullableVarBinaryHolder() { // verify results assertTrue(vector.isNull(0)); - assertEquals(str, new String(vector.get(1))); + assertEquals(str, new String(vector.get(1), StandardCharsets.UTF_8)); buf.close(); } @@ -2365,7 +2354,7 @@ public void testSetNullableVarBinaryHolderSafe() { String str = "hello world"; ArrowBuf buf = allocator.buffer(16); - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); binHolder.start = 0; binHolder.end = str.length(); @@ -2375,7 +2364,7 @@ public void testSetNullableVarBinaryHolderSafe() { vector.setSafe(1, nullHolder); // verify results - assertEquals(str, new String(vector.get(0))); + assertEquals(str, new String(vector.get(0), StandardCharsets.UTF_8)); assertTrue(vector.isNull(1)); buf.close(); @@ -2431,8 +2420,8 @@ public void testGetPointerVariableWidth() { for (int i = 0; i < sampleData.length; i++) { String str = sampleData[i]; if (str != null) { - vec1.set(i, sampleData[i].getBytes()); - vec2.set(i, sampleData[i].getBytes()); + vec1.set(i, sampleData[i].getBytes(StandardCharsets.UTF_8)); + vec2.set(i, sampleData[i].getBytes(StandardCharsets.UTF_8)); } else { vec1.setNull(i); vec2.setNull(i); @@ -2827,7 +2816,7 @@ public void testVariableWidthVectorNullHashCode() { varChVec.allocateNew(100, 1); varChVec.setValueCount(1); - varChVec.set(0, "abc".getBytes()); + varChVec.set(0, "abc".getBytes(StandardCharsets.UTF_8)); varChVec.setNull(0); assertEquals(0, varChVec.hashCode(0)); @@ -2945,7 +2934,7 @@ public void testUnloadVariableWidthVector() { varCharVector.allocateNew(5, 2); varCharVector.setValueCount(2); - varCharVector.set(0, "abcd".getBytes()); + varCharVector.set(0, "abcd".getBytes(StandardCharsets.UTF_8)); List bufs = varCharVector.getFieldBuffers(); assertEquals(3, bufs.size()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharListVector.java index a9b155499f773..bfe489fa5af4e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharListVector.java @@ -17,6 +17,8 @@ package org.apache.arrow.vector; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.ListVector; @@ -44,7 +46,7 @@ public void terminate() throws Exception { @Test public void testVarCharListWithNulls() { - byte[] bytes = "a".getBytes(); + byte[] bytes = "a".getBytes(StandardCharsets.UTF_8); try (ListVector vector = new ListVector("VarList", allocator, FieldType.nullable(Types .MinorType.VARCHAR.getType()), null); ArrowBuf tempBuf = allocator.buffer(bytes.length)) { @@ -63,15 +65,15 @@ public void testVarCharListWithNulls() { writer.setPosition(2); writer.startList(); - bytes = "b".getBytes(); + bytes = "b".getBytes(StandardCharsets.UTF_8); tempBuf.setBytes(0, bytes); writer.writeVarChar(0, bytes.length, tempBuf); writer.endList(); writer.setValueCount(2); - Assert.assertTrue(vector.getValueCount() == 2); - Assert.assertTrue(vector.getDataVector().getValueCount() == 2); + Assert.assertEquals(2, vector.getValueCount()); + Assert.assertEquals(2, vector.getDataVector().getValueCount()); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java index dfc75ec8e34cf..b96f6ab6afedd 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java @@ -143,7 +143,7 @@ public void testFixedWidthVectorAllocation() { totalCapacity = vec2.getValidityBuffer().capacity() + vec2.getDataBuffer().capacity(); // the total capacity must be a power of two - assertEquals(totalCapacity & (totalCapacity - 1), 0); + assertEquals(0, totalCapacity & (totalCapacity - 1)); } } @@ -163,7 +163,7 @@ public void testVariableWidthVectorAllocation() { totalCapacity = vec2.getValidityBuffer().capacity() + vec2.getOffsetBuffer().capacity(); // the total capacity must be a power of two - assertEquals(totalCapacity & (totalCapacity - 1), 0); + assertEquals(0, totalCapacity & (totalCapacity - 1)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 7d5701ddb765b..9043bd4f8f2d4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -324,12 +324,12 @@ public void testVariableRepeatedClearAndSet() throws Exception { vector.allocateNewSafe(); // Initial allocation vector.clear(); // clear vector. - vector.setSafe(0, "hello world".getBytes()); + vector.setSafe(0, "hello world".getBytes(StandardCharsets.UTF_8)); int savedValueCapacity = vector.getValueCapacity(); for (int i = 0; i < 1024; ++i) { vector.clear(); // clear vector. - vector.setSafe(0, "hello world".getBytes()); + vector.setSafe(0, "hello world".getBytes(StandardCharsets.UTF_8)); } // should be deterministic, and not cause a run-away increase in capacity. diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java index ce3fb2cdf0ea1..207962eb45b85 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java @@ -61,7 +61,7 @@ public void testResetRowCount() { VectorSchemaRoot vsr = VectorSchemaRoot.of(vec1, vec2); vsr.allocateNew(); - assertEquals(vsr.getRowCount(), 0); + assertEquals(0, vsr.getRowCount()); for (int i = 0; i < size; i++) { vec1.setSafe(i, i % 2); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestTypeEqualsVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestTypeEqualsVisitor.java index c0a3bd89dc18c..62fa0336ea925 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestTypeEqualsVisitor.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestTypeEqualsVisitor.java @@ -20,7 +20,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; @@ -52,11 +51,6 @@ public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - private static final Charset utf8Charset = Charset.forName("UTF-8"); - private static final byte[] STR1 = "AAAAA1".getBytes(utf8Charset); - private static final byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset); - private static final byte[] STR3 = "CCCC3".getBytes(utf8Charset); - @After public void terminate() throws Exception { allocator.close(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index 4c8c96a0d74d3..b7fc681c16118 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -24,6 +24,8 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.Objects; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -413,8 +415,8 @@ public void testPromoteLargeVarCharHelpersOnStruct() throws Exception { writer.end(); final LargeVarCharVector uv = v.getChild("c", LargeVarCharVector.class); - assertEquals("foo", uv.getObject(0).toString()); - assertEquals("foo2", uv.getObject(1).toString()); + assertEquals("foo", Objects.requireNonNull(uv.getObject(0)).toString()); + assertEquals("foo2", Objects.requireNonNull(uv.getObject(1)).toString()); } } @@ -433,8 +435,8 @@ public void testPromoteVarCharHelpersOnStruct() throws Exception { writer.end(); final VarCharVector uv = v.getChild("c", VarCharVector.class); - assertEquals("foo", uv.getObject(0).toString()); - assertEquals("foo2", uv.getObject(1).toString()); + assertEquals("foo", Objects.requireNonNull(uv.getObject(0)).toString()); + assertEquals("foo2", Objects.requireNonNull(uv.getObject(1)).toString()); } } @@ -455,8 +457,8 @@ public void testPromoteVarCharHelpersDirect() throws Exception { // The "test" vector in the parent container should have been replaced with a UnionVector. UnionVector promotedVector = container.getChild("test", UnionVector.class); VarCharVector vector = promotedVector.getVarCharVector(); - assertEquals("foo", vector.getObject(0).toString()); - assertEquals("foo2", vector.getObject(1).toString()); + assertEquals("foo", Objects.requireNonNull(vector.getObject(0)).toString()); + assertEquals("foo2", Objects.requireNonNull(vector.getObject(1)).toString()); } } @@ -477,8 +479,8 @@ public void testPromoteLargeVarCharHelpersDirect() throws Exception { // The "test" vector in the parent container should have been replaced with a UnionVector. UnionVector promotedVector = container.getChild("test", UnionVector.class); LargeVarCharVector vector = promotedVector.getLargeVarCharVector(); - assertEquals("foo", vector.getObject(0).toString()); - assertEquals("foo2", vector.getObject(1).toString()); + assertEquals("foo", Objects.requireNonNull(vector.getObject(0)).toString()); + assertEquals("foo2", Objects.requireNonNull(vector.getObject(1)).toString()); } } @@ -491,20 +493,22 @@ public void testPromoteVarBinaryHelpersOnStruct() throws Exception { writer.start(); writer.setPosition(0); - writer.varBinary("c").writeVarBinary("row1".getBytes()); + writer.varBinary("c").writeVarBinary("row1".getBytes(StandardCharsets.UTF_8)); writer.setPosition(1); - writer.varBinary("c").writeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + writer.varBinary("c").writeVarBinary("row2".getBytes(StandardCharsets.UTF_8), 0, + "row2".getBytes(StandardCharsets.UTF_8).length); writer.setPosition(2); - writer.varBinary("c").writeVarBinary(ByteBuffer.wrap("row3".getBytes())); + writer.varBinary("c").writeVarBinary(ByteBuffer.wrap("row3".getBytes(StandardCharsets.UTF_8))); writer.setPosition(3); - writer.varBinary("c").writeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + writer.varBinary("c").writeVarBinary(ByteBuffer.wrap("row4".getBytes(StandardCharsets.UTF_8)), 0, + "row4".getBytes(StandardCharsets.UTF_8).length); writer.end(); final VarBinaryVector uv = v.getChild("c", VarBinaryVector.class); - assertEquals("row1", new String(uv.get(0))); - assertEquals("row2", new String(uv.get(1))); - assertEquals("row3", new String(uv.get(2))); - assertEquals("row4", new String(uv.get(3))); + assertEquals("row1", new String(Objects.requireNonNull(uv.get(0)), StandardCharsets.UTF_8)); + assertEquals("row2", new String(Objects.requireNonNull(uv.get(1)), StandardCharsets.UTF_8)); + assertEquals("row3", new String(Objects.requireNonNull(uv.get(2)), StandardCharsets.UTF_8)); + assertEquals("row4", new String(Objects.requireNonNull(uv.get(3)), StandardCharsets.UTF_8)); } } @@ -517,22 +521,24 @@ public void testPromoteVarBinaryHelpersDirect() throws Exception { writer.start(); writer.setPosition(0); - writer.writeVarBinary("row1".getBytes()); + writer.writeVarBinary("row1".getBytes(StandardCharsets.UTF_8)); writer.setPosition(1); - writer.writeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + writer.writeVarBinary("row2".getBytes(StandardCharsets.UTF_8), 0, + "row2".getBytes(StandardCharsets.UTF_8).length); writer.setPosition(2); - writer.writeVarBinary(ByteBuffer.wrap("row3".getBytes())); + writer.writeVarBinary(ByteBuffer.wrap("row3".getBytes(StandardCharsets.UTF_8))); writer.setPosition(3); - writer.writeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + writer.writeVarBinary(ByteBuffer.wrap("row4".getBytes(StandardCharsets.UTF_8)), 0, + "row4".getBytes(StandardCharsets.UTF_8).length); writer.end(); // The "test" vector in the parent container should have been replaced with a UnionVector. UnionVector promotedVector = container.getChild("test", UnionVector.class); VarBinaryVector uv = promotedVector.getVarBinaryVector(); - assertEquals("row1", new String(uv.get(0))); - assertEquals("row2", new String(uv.get(1))); - assertEquals("row3", new String(uv.get(2))); - assertEquals("row4", new String(uv.get(3))); + assertEquals("row1", new String(Objects.requireNonNull(uv.get(0)), StandardCharsets.UTF_8)); + assertEquals("row2", new String(Objects.requireNonNull(uv.get(1)), StandardCharsets.UTF_8)); + assertEquals("row3", new String(Objects.requireNonNull(uv.get(2)), StandardCharsets.UTF_8)); + assertEquals("row4", new String(Objects.requireNonNull(uv.get(3)), StandardCharsets.UTF_8)); } } @@ -545,20 +551,22 @@ public void testPromoteLargeVarBinaryHelpersOnStruct() throws Exception { writer.start(); writer.setPosition(0); - writer.largeVarBinary("c").writeLargeVarBinary("row1".getBytes()); + writer.largeVarBinary("c").writeLargeVarBinary("row1".getBytes(StandardCharsets.UTF_8)); writer.setPosition(1); - writer.largeVarBinary("c").writeLargeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + writer.largeVarBinary("c").writeLargeVarBinary("row2".getBytes(StandardCharsets.UTF_8), 0, + "row2".getBytes(StandardCharsets.UTF_8).length); writer.setPosition(2); - writer.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes())); + writer.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes(StandardCharsets.UTF_8))); writer.setPosition(3); - writer.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + writer.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes(StandardCharsets.UTF_8)), 0, + "row4".getBytes(StandardCharsets.UTF_8).length); writer.end(); final LargeVarBinaryVector uv = v.getChild("c", LargeVarBinaryVector.class); - assertEquals("row1", new String(uv.get(0))); - assertEquals("row2", new String(uv.get(1))); - assertEquals("row3", new String(uv.get(2))); - assertEquals("row4", new String(uv.get(3))); + assertEquals("row1", new String(Objects.requireNonNull(uv.get(0)), StandardCharsets.UTF_8)); + assertEquals("row2", new String(Objects.requireNonNull(uv.get(1)), StandardCharsets.UTF_8)); + assertEquals("row3", new String(Objects.requireNonNull(uv.get(2)), StandardCharsets.UTF_8)); + assertEquals("row4", new String(Objects.requireNonNull(uv.get(3)), StandardCharsets.UTF_8)); } } @@ -571,22 +579,24 @@ public void testPromoteLargeVarBinaryHelpersDirect() throws Exception { writer.start(); writer.setPosition(0); - writer.writeLargeVarBinary("row1".getBytes()); + writer.writeLargeVarBinary("row1".getBytes(StandardCharsets.UTF_8)); writer.setPosition(1); - writer.writeLargeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + writer.writeLargeVarBinary("row2".getBytes(StandardCharsets.UTF_8), 0, + "row2".getBytes(StandardCharsets.UTF_8).length); writer.setPosition(2); - writer.writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes())); + writer.writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes(StandardCharsets.UTF_8))); writer.setPosition(3); - writer.writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + writer.writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes(StandardCharsets.UTF_8)), 0, + "row4".getBytes(StandardCharsets.UTF_8).length); writer.end(); // The "test" vector in the parent container should have been replaced with a UnionVector. UnionVector promotedVector = container.getChild("test", UnionVector.class); LargeVarBinaryVector uv = promotedVector.getLargeVarBinaryVector(); - assertEquals("row1", new String(uv.get(0))); - assertEquals("row2", new String(uv.get(1))); - assertEquals("row3", new String(uv.get(2))); - assertEquals("row4", new String(uv.get(3))); + assertEquals("row1", new String(Objects.requireNonNull(uv.get(0)), StandardCharsets.UTF_8)); + assertEquals("row2", new String(Objects.requireNonNull(uv.get(1)), StandardCharsets.UTF_8)); + assertEquals("row3", new String(Objects.requireNonNull(uv.get(2)), StandardCharsets.UTF_8)); + assertEquals("row4", new String(Objects.requireNonNull(uv.get(3)), StandardCharsets.UTF_8)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index e03ce0c056bf1..19f0ea9d4e392 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -21,6 +21,7 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -128,9 +129,7 @@ public void simpleNestedTypes() { @Test public void transferPairSchemaChange() { SchemaChangeCallBack callBack1 = new SchemaChangeCallBack(); - SchemaChangeCallBack callBack2 = new SchemaChangeCallBack(); try (NonNullableStructVector parent = populateStructVector(callBack1)) { - TransferPair tp = parent.getTransferPair("newVector", allocator, callBack2); ComplexWriter writer = new ComplexWriterImpl("newWriter", parent); StructWriter rootWriter = writer.rootAsStruct(); @@ -818,7 +817,7 @@ public void promotableWriter() { for (int i = 100; i < 200; i++) { VarCharWriter varCharWriter = rootWriter.varChar("a"); varCharWriter.setPosition(i); - byte[] bytes = Integer.toString(i).getBytes(); + byte[] bytes = Integer.toString(i).getBytes(StandardCharsets.UTF_8); ArrowBuf tempBuf = allocator.buffer(bytes.length); tempBuf.setBytes(0, bytes); varCharWriter.writeVarChar(0, bytes.length, tempBuf); @@ -1719,21 +1718,23 @@ public void structWriterVarBinaryHelpers() { StructWriter rootWriter = writer.rootAsStruct(); rootWriter.start(); rootWriter.setPosition(0); - rootWriter.varBinary("c").writeVarBinary("row1".getBytes()); + rootWriter.varBinary("c").writeVarBinary("row1".getBytes(StandardCharsets.UTF_8)); rootWriter.setPosition(1); - rootWriter.varBinary("c").writeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + rootWriter.varBinary("c").writeVarBinary("row2".getBytes(StandardCharsets.UTF_8), 0, + "row2".getBytes(StandardCharsets.UTF_8).length); rootWriter.setPosition(2); - rootWriter.varBinary("c").writeVarBinary(ByteBuffer.wrap("row3".getBytes())); + rootWriter.varBinary("c").writeVarBinary(ByteBuffer.wrap("row3".getBytes(StandardCharsets.UTF_8))); rootWriter.setPosition(3); - rootWriter.varBinary("c").writeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + rootWriter.varBinary("c").writeVarBinary(ByteBuffer.wrap( + "row4".getBytes(StandardCharsets.UTF_8)), 0, "row4".getBytes(StandardCharsets.UTF_8).length); rootWriter.end(); VarBinaryVector uv = parent.getChild("root", StructVector.class).getChild("c", VarBinaryVector.class); - assertEquals("row1", new String(uv.get(0))); - assertEquals("row2", new String(uv.get(1))); - assertEquals("row3", new String(uv.get(2))); - assertEquals("row4", new String(uv.get(3))); + assertEquals("row1", new String(uv.get(0), StandardCharsets.UTF_8)); + assertEquals("row2", new String(uv.get(1), StandardCharsets.UTF_8)); + assertEquals("row3", new String(uv.get(2), StandardCharsets.UTF_8)); + assertEquals("row4", new String(uv.get(3), StandardCharsets.UTF_8)); } } @@ -1744,23 +1745,24 @@ public void structWriterLargeVarBinaryHelpers() { StructWriter rootWriter = writer.rootAsStruct(); rootWriter.start(); rootWriter.setPosition(0); - rootWriter.largeVarBinary("c").writeLargeVarBinary("row1".getBytes()); + rootWriter.largeVarBinary("c").writeLargeVarBinary("row1".getBytes(StandardCharsets.UTF_8)); rootWriter.setPosition(1); - rootWriter.largeVarBinary("c").writeLargeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); + rootWriter.largeVarBinary("c").writeLargeVarBinary("row2".getBytes(StandardCharsets.UTF_8), 0, + "row2".getBytes(StandardCharsets.UTF_8).length); rootWriter.setPosition(2); - rootWriter.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes())); + rootWriter.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes(StandardCharsets.UTF_8))); rootWriter.setPosition(3); - rootWriter.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, - "row4".getBytes().length); + rootWriter.largeVarBinary("c").writeLargeVarBinary(ByteBuffer.wrap( + "row4".getBytes(StandardCharsets.UTF_8)), 0, "row4".getBytes(StandardCharsets.UTF_8).length); rootWriter.end(); LargeVarBinaryVector uv = parent.getChild("root", StructVector.class).getChild("c", LargeVarBinaryVector.class); - assertEquals("row1", new String(uv.get(0))); - assertEquals("row2", new String(uv.get(1))); - assertEquals("row3", new String(uv.get(2))); - assertEquals("row4", new String(uv.get(3))); + assertEquals("row1", new String(uv.get(0), StandardCharsets.UTF_8)); + assertEquals("row2", new String(uv.get(1), StandardCharsets.UTF_8)); + assertEquals("row3", new String(uv.get(2), StandardCharsets.UTF_8)); + assertEquals("row4", new String(uv.get(3), StandardCharsets.UTF_8)); } } @@ -1800,16 +1802,18 @@ public void listVarBinaryHelpers() { listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); listWriter.startList(); - listWriter.writeVarBinary("row1".getBytes()); - listWriter.writeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); - listWriter.writeVarBinary(ByteBuffer.wrap("row3".getBytes())); - listWriter.writeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + listWriter.writeVarBinary("row1".getBytes(StandardCharsets.UTF_8)); + listWriter.writeVarBinary("row2".getBytes(StandardCharsets.UTF_8), 0, + "row2".getBytes(StandardCharsets.UTF_8).length); + listWriter.writeVarBinary(ByteBuffer.wrap("row3".getBytes(StandardCharsets.UTF_8))); + listWriter.writeVarBinary(ByteBuffer.wrap( + "row4".getBytes(StandardCharsets.UTF_8)), 0, "row4".getBytes(StandardCharsets.UTF_8).length); listWriter.endList(); listWriter.setValueCount(1); - assertEquals("row1", new String((byte[]) listVector.getObject(0).get(0))); - assertEquals("row2", new String((byte[]) listVector.getObject(0).get(1))); - assertEquals("row3", new String((byte[]) listVector.getObject(0).get(2))); - assertEquals("row4", new String((byte[]) listVector.getObject(0).get(3))); + assertEquals("row1", new String((byte[]) listVector.getObject(0).get(0), StandardCharsets.UTF_8)); + assertEquals("row2", new String((byte[]) listVector.getObject(0).get(1), StandardCharsets.UTF_8)); + assertEquals("row3", new String((byte[]) listVector.getObject(0).get(2), StandardCharsets.UTF_8)); + assertEquals("row4", new String((byte[]) listVector.getObject(0).get(3), StandardCharsets.UTF_8)); } } @@ -1819,16 +1823,18 @@ public void listLargeVarBinaryHelpers() { listVector.allocateNew(); UnionListWriter listWriter = new UnionListWriter(listVector); listWriter.startList(); - listWriter.writeLargeVarBinary("row1".getBytes()); - listWriter.writeLargeVarBinary("row2".getBytes(), 0, "row2".getBytes().length); - listWriter.writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes())); - listWriter.writeLargeVarBinary(ByteBuffer.wrap("row4".getBytes()), 0, "row4".getBytes().length); + listWriter.writeLargeVarBinary("row1".getBytes(StandardCharsets.UTF_8)); + listWriter.writeLargeVarBinary("row2".getBytes(StandardCharsets.UTF_8), 0, + "row2".getBytes(StandardCharsets.UTF_8).length); + listWriter.writeLargeVarBinary(ByteBuffer.wrap("row3".getBytes(StandardCharsets.UTF_8))); + listWriter.writeLargeVarBinary(ByteBuffer.wrap( + "row4".getBytes(StandardCharsets.UTF_8)), 0, "row4".getBytes(StandardCharsets.UTF_8).length); listWriter.endList(); listWriter.setValueCount(1); - assertEquals("row1", new String((byte[]) listVector.getObject(0).get(0))); - assertEquals("row2", new String((byte[]) listVector.getObject(0).get(1))); - assertEquals("row3", new String((byte[]) listVector.getObject(0).get(2))); - assertEquals("row4", new String((byte[]) listVector.getObject(0).get(3))); + assertEquals("row1", new String((byte[]) listVector.getObject(0).get(0), StandardCharsets.UTF_8)); + assertEquals("row2", new String((byte[]) listVector.getObject(0).get(1), StandardCharsets.UTF_8)); + assertEquals("row3", new String((byte[]) listVector.getObject(0).get(2), StandardCharsets.UTF_8)); + assertEquals("row4", new String((byte[]) listVector.getObject(0).get(3), StandardCharsets.UTF_8)); } } @@ -1847,35 +1853,39 @@ public void unionWithVarCharAndBinaryHelpers() throws Exception { unionWriter.setPosition(3); unionWriter.writeLargeVarChar(new Text("row4")); unionWriter.setPosition(4); - unionWriter.writeVarBinary("row5".getBytes()); + unionWriter.writeVarBinary("row5".getBytes(StandardCharsets.UTF_8)); unionWriter.setPosition(5); - unionWriter.writeVarBinary("row6".getBytes(), 0, "row6".getBytes().length); + unionWriter.writeVarBinary("row6".getBytes(StandardCharsets.UTF_8), 0, + "row6".getBytes(StandardCharsets.UTF_8).length); unionWriter.setPosition(6); - unionWriter.writeVarBinary(ByteBuffer.wrap("row7".getBytes())); + unionWriter.writeVarBinary(ByteBuffer.wrap("row7".getBytes(StandardCharsets.UTF_8))); unionWriter.setPosition(7); - unionWriter.writeVarBinary(ByteBuffer.wrap("row8".getBytes()), 0, "row8".getBytes().length); + unionWriter.writeVarBinary(ByteBuffer.wrap("row8".getBytes(StandardCharsets.UTF_8)), 0, + "row8".getBytes(StandardCharsets.UTF_8).length); unionWriter.setPosition(8); - unionWriter.writeLargeVarBinary("row9".getBytes()); + unionWriter.writeLargeVarBinary("row9".getBytes(StandardCharsets.UTF_8)); unionWriter.setPosition(9); - unionWriter.writeLargeVarBinary("row10".getBytes(), 0, "row10".getBytes().length); + unionWriter.writeLargeVarBinary("row10".getBytes(StandardCharsets.UTF_8), 0, + "row10".getBytes(StandardCharsets.UTF_8).length); unionWriter.setPosition(10); - unionWriter.writeLargeVarBinary(ByteBuffer.wrap("row11".getBytes())); + unionWriter.writeLargeVarBinary(ByteBuffer.wrap("row11".getBytes(StandardCharsets.UTF_8))); unionWriter.setPosition(11); - unionWriter.writeLargeVarBinary(ByteBuffer.wrap("row12".getBytes()), 0, "row12".getBytes().length); + unionWriter.writeLargeVarBinary(ByteBuffer.wrap( + "row12".getBytes(StandardCharsets.UTF_8)), 0, "row12".getBytes(StandardCharsets.UTF_8).length); unionWriter.end(); - assertEquals("row1", new String(vector.getVarCharVector().get(0))); - assertEquals("row2", new String(vector.getVarCharVector().get(1))); - assertEquals("row3", new String(vector.getLargeVarCharVector().get(2))); - assertEquals("row4", new String(vector.getLargeVarCharVector().get(3))); - assertEquals("row5", new String(vector.getVarBinaryVector().get(4))); - assertEquals("row6", new String(vector.getVarBinaryVector().get(5))); - assertEquals("row7", new String(vector.getVarBinaryVector().get(6))); - assertEquals("row8", new String(vector.getVarBinaryVector().get(7))); - assertEquals("row9", new String(vector.getLargeVarBinaryVector().get(8))); - assertEquals("row10", new String(vector.getLargeVarBinaryVector().get(9))); - assertEquals("row11", new String(vector.getLargeVarBinaryVector().get(10))); - assertEquals("row12", new String(vector.getLargeVarBinaryVector().get(11))); + assertEquals("row1", new String(vector.getVarCharVector().get(0), StandardCharsets.UTF_8)); + assertEquals("row2", new String(vector.getVarCharVector().get(1), StandardCharsets.UTF_8)); + assertEquals("row3", new String(vector.getLargeVarCharVector().get(2), StandardCharsets.UTF_8)); + assertEquals("row4", new String(vector.getLargeVarCharVector().get(3), StandardCharsets.UTF_8)); + assertEquals("row5", new String(vector.getVarBinaryVector().get(4), StandardCharsets.UTF_8)); + assertEquals("row6", new String(vector.getVarBinaryVector().get(5), StandardCharsets.UTF_8)); + assertEquals("row7", new String(vector.getVarBinaryVector().get(6), StandardCharsets.UTF_8)); + assertEquals("row8", new String(vector.getVarBinaryVector().get(7), StandardCharsets.UTF_8)); + assertEquals("row9", new String(vector.getLargeVarBinaryVector().get(8), StandardCharsets.UTF_8)); + assertEquals("row10", new String(vector.getLargeVarBinaryVector().get(9), StandardCharsets.UTF_8)); + assertEquals("row11", new String(vector.getLargeVarBinaryVector().get(10), StandardCharsets.UTF_8)); + assertEquals("row12", new String(vector.getLargeVarBinaryVector().get(11), StandardCharsets.UTF_8)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index 8663c0c49990d..de9187edb667e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -694,19 +694,19 @@ protected void writeBatchData(ArrowWriter writer, IntVector vector, VectorSchema protected void validateBatchData(ArrowReader reader, IntVector vector) throws IOException { reader.loadNextBatch(); - assertEquals(vector.getValueCount(), 5); + assertEquals(5, vector.getValueCount()); assertTrue(vector.isNull(0)); - assertEquals(vector.get(1), 1); - assertEquals(vector.get(2), 2); + assertEquals(1, vector.get(1)); + assertEquals(2, vector.get(2)); assertTrue(vector.isNull(3)); - assertEquals(vector.get(4), 1); + assertEquals(1, vector.get(4)); reader.loadNextBatch(); - assertEquals(vector.getValueCount(), 3); + assertEquals(3, vector.getValueCount()); assertTrue(vector.isNull(0)); - assertEquals(vector.get(1), 1); - assertEquals(vector.get(2), 2); + assertEquals(1, vector.get(1)); + assertEquals(2, vector.get(2)); } protected VectorSchemaRoot writeMapData(BufferAllocator bufferAllocator) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java index 9348cd3a66708..145bdd588e945 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java @@ -79,8 +79,8 @@ public void testStreamZeroLengthBatch() throws IOException { VectorSchemaRoot root = reader.getVectorSchemaRoot(); IntVector vector = (IntVector) root.getFieldVectors().get(0); reader.loadNextBatch(); - assertEquals(vector.getValueCount(), 0); - assertEquals(root.getRowCount(), 0); + assertEquals(0, vector.getValueCount()); + assertEquals(0, root.getRowCount()); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java index 0aa49d9daa0da..bd5bd4feabbd4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java @@ -476,7 +476,7 @@ public void testRoundtripEmptyVector() throws Exception { assertEquals(schema, readSchema); try (final VectorSchemaRoot data = reader.read()) { assertNotNull(data); - assertEquals(data.getRowCount(), 0); + assertEquals(0, data.getRowCount()); } assertNull(reader.read()); } @@ -496,7 +496,7 @@ public void testRoundtripEmptyVector() throws Exception { assertEquals(schema, readSchema); try (final VectorSchemaRoot data = reader.read()) { assertNotNull(data); - assertEquals(data.getRowCount(), 0); + assertEquals(0, data.getRowCount()); } assertNull(reader.read()); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java index 6aa7a0c6df5c3..ac95121eb73f2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java @@ -27,6 +27,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; import java.util.Map; @@ -138,7 +139,7 @@ private void readData( VarCharVector dictVector = (VarCharVector) dictionary.getVector(); assertEquals(expectedDictItems.length, dictVector.getValueCount()); for (int i = 0; i < dictVector.getValueCount(); i++) { - assertArrayEquals(expectedDictItems[i].getBytes(), dictVector.get(i)); + assertArrayEquals(expectedDictItems[i].getBytes(StandardCharsets.UTF_8), dictVector.get(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/message/TestMessageMetadataResult.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/message/TestMessageMetadataResult.java index ee5361547a0b9..0505a18484b54 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/message/TestMessageMetadataResult.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/message/TestMessageMetadataResult.java @@ -30,7 +30,7 @@ public void getMessageLength_returnsConstructValue() { // This API is used by spark. MessageMetadataResult result = new MessageMetadataResult(1, ByteBuffer.allocate(0), new org.apache.arrow.flatbuf.Message()); - assertEquals(result.getMessageLength(), 1); + assertEquals(1, result.getMessageLength()); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/table/BaseTableTest.java b/java/vector/src/test/java/org/apache/arrow/vector/table/BaseTableTest.java index 78f2ee51b8912..1b7f984992ada 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/table/BaseTableTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/table/BaseTableTest.java @@ -28,8 +28,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -282,8 +284,8 @@ void testDecode() { VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator); dictionaryVector.allocateNew(2); - dictionaryVector.set(0, "one".getBytes()); - dictionaryVector.set(1, "two".getBytes()); + dictionaryVector.set(0, "one".getBytes(StandardCharsets.UTF_8)); + dictionaryVector.set(1, "two".getBytes(StandardCharsets.UTF_8)); dictionaryVector.setValueCount(2); Dictionary dictionary = new Dictionary(dictionaryVector, new DictionaryEncoding(1L, false, null)); @@ -297,8 +299,8 @@ void testDecode() { try (Table t = new Table(vectorList, vectorList.get(0).getValueCount(), provider)) { VarCharVector v = (VarCharVector) t.decode(encoded.getName(), 1L); assertNotNull(v); - assertEquals("one", new String(v.get(0))); - assertEquals("two", new String(v.get(1))); + assertEquals("one", new String(Objects.requireNonNull(v.get(0)), StandardCharsets.UTF_8)); + assertEquals("two", new String(Objects.requireNonNull(v.get(1)), StandardCharsets.UTF_8)); } } @@ -319,8 +321,8 @@ private DictionaryProvider getDictionary() { VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator); dictionaryVector.allocateNew(2); - dictionaryVector.set(0, "one".getBytes()); - dictionaryVector.set(1, "two".getBytes()); + dictionaryVector.set(0, "one".getBytes(StandardCharsets.UTF_8)); + dictionaryVector.set(1, "two".getBytes(StandardCharsets.UTF_8)); dictionaryVector.setValueCount(2); Dictionary dictionary = new Dictionary(dictionaryVector, encoding); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/table/RowTest.java b/java/vector/src/test/java/org/apache/arrow/vector/table/RowTest.java index eb50e866b19f0..3e6a096104d44 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/table/RowTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/table/RowTest.java @@ -650,8 +650,8 @@ void getVarChar() { c.setPosition(1); assertEquals(c.getVarCharObj(1), "two"); assertEquals(c.getVarCharObj(1), c.getVarCharObj(VARCHAR_VECTOR_NAME_1)); - assertArrayEquals("two".getBytes(), c.getVarChar(VARCHAR_VECTOR_NAME_1)); - assertArrayEquals("two".getBytes(), c.getVarChar(1)); + assertArrayEquals("two".getBytes(StandardCharsets.UTF_8), c.getVarChar(VARCHAR_VECTOR_NAME_1)); + assertArrayEquals("two".getBytes(StandardCharsets.UTF_8), c.getVarChar(1)); } } @@ -661,7 +661,7 @@ void getVarBinary() { try (Table t = new Table(vectorList)) { Row c = t.immutableRow(); c.setPosition(1); - assertArrayEquals(c.getVarBinary(1), "two".getBytes()); + assertArrayEquals(c.getVarBinary(1), "two".getBytes(StandardCharsets.UTF_8)); assertArrayEquals(c.getVarBinary(1), c.getVarBinary(VARBINARY_VECTOR_NAME_1)); } } @@ -672,7 +672,7 @@ void getLargeVarBinary() { try (Table t = new Table(vectorList)) { Row c = t.immutableRow(); c.setPosition(1); - assertArrayEquals(c.getLargeVarBinary(1), "two".getBytes()); + assertArrayEquals(c.getLargeVarBinary(1), "two".getBytes(StandardCharsets.UTF_8)); assertArrayEquals(c.getLargeVarBinary(1), c.getLargeVarBinary(VARBINARY_VECTOR_NAME_1)); } } @@ -685,8 +685,8 @@ void getLargeVarChar() { c.setPosition(1); assertEquals(c.getLargeVarCharObj(1), "two"); assertEquals(c.getLargeVarCharObj(1), c.getLargeVarCharObj(VARCHAR_VECTOR_NAME_1)); - assertArrayEquals("two".getBytes(), c.getLargeVarChar(VARCHAR_VECTOR_NAME_1)); - assertArrayEquals("two".getBytes(), c.getLargeVarChar(1)); + assertArrayEquals("two".getBytes(StandardCharsets.UTF_8), c.getLargeVarChar(VARCHAR_VECTOR_NAME_1)); + assertArrayEquals("two".getBytes(StandardCharsets.UTF_8), c.getLargeVarChar(1)); } } @@ -696,7 +696,7 @@ void getFixedBinary() { try (Table t = new Table(vectorList)) { Row c = t.immutableRow(); c.setPosition(1); - assertArrayEquals(c.getFixedSizeBinary(1), "two".getBytes()); + assertArrayEquals(c.getFixedSizeBinary(1), "two".getBytes(StandardCharsets.UTF_8)); assertArrayEquals(c.getFixedSizeBinary(1), c.getFixedSizeBinary(FIXEDBINARY_VECTOR_NAME_1)); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/table/TestUtils.java b/java/vector/src/test/java/org/apache/arrow/vector/table/TestUtils.java index cb0b7b8eb6b87..c0b3bfdf73220 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/table/TestUtils.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/table/TestUtils.java @@ -20,6 +20,7 @@ import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.OFFSET_WIDTH; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -117,8 +118,8 @@ static List intPlusVarcharColumns(BufferAllocator allocator) { IntVector v1 = getSimpleIntVector(allocator); VarCharVector v2 = new VarCharVector(VARCHAR_VECTOR_NAME_1, allocator); v2.allocateNew(2); - v2.set(0, "one".getBytes()); - v2.set(1, "two".getBytes()); + v2.set(0, "one".getBytes(StandardCharsets.UTF_8)); + v2.set(1, "two".getBytes(StandardCharsets.UTF_8)); v2.setValueCount(2); vectorList.add(v1); vectorList.add(v2); @@ -134,8 +135,8 @@ static List intPlusLargeVarcharColumns(BufferAllocator allocator) { IntVector v1 = getSimpleIntVector(allocator); LargeVarCharVector v2 = new LargeVarCharVector(VARCHAR_VECTOR_NAME_1, allocator); v2.allocateNew(2); - v2.set(0, "one".getBytes()); - v2.set(1, "two".getBytes()); + v2.set(0, "one".getBytes(StandardCharsets.UTF_8)); + v2.set(1, "two".getBytes(StandardCharsets.UTF_8)); v2.setValueCount(2); vectorList.add(v1); vectorList.add(v2); @@ -152,8 +153,8 @@ static List intPlusVarBinaryColumns(BufferAllocator allocator) { IntVector v1 = getSimpleIntVector(allocator); VarBinaryVector v2 = new VarBinaryVector(VARBINARY_VECTOR_NAME_1, allocator); v2.allocateNew(2); - v2.set(0, "one".getBytes()); - v2.set(1, "two".getBytes()); + v2.set(0, "one".getBytes(StandardCharsets.UTF_8)); + v2.set(1, "two".getBytes(StandardCharsets.UTF_8)); v2.setValueCount(2); vectorList.add(v1); vectorList.add(v2); @@ -170,8 +171,8 @@ static List intPlusLargeVarBinaryColumns(BufferAllocator allocator) IntVector v1 = getSimpleIntVector(allocator); LargeVarBinaryVector v2 = new LargeVarBinaryVector(VARBINARY_VECTOR_NAME_1, allocator); v2.allocateNew(2); - v2.set(0, "one".getBytes()); - v2.set(1, "two".getBytes()); + v2.set(0, "one".getBytes(StandardCharsets.UTF_8)); + v2.set(1, "two".getBytes(StandardCharsets.UTF_8)); v2.setValueCount(2); vectorList.add(v1); vectorList.add(v2); @@ -188,8 +189,8 @@ static List intPlusFixedBinaryColumns(BufferAllocator allocator) { IntVector v1 = getSimpleIntVector(allocator); FixedSizeBinaryVector v2 = new FixedSizeBinaryVector(FIXEDBINARY_VECTOR_NAME_1, allocator, 3); v2.allocateNew(2); - v2.set(0, "one".getBytes()); - v2.set(1, "two".getBytes()); + v2.set(0, "one".getBytes(StandardCharsets.UTF_8)); + v2.set(1, "two".getBytes(StandardCharsets.UTF_8)); v2.setValueCount(2); vectorList.add(v1); vectorList.add(v2); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java b/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java index 74257c45ca887..3c075c9293079 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java @@ -20,6 +20,8 @@ import static junit.framework.TestCase.assertTrue; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; @@ -204,13 +206,14 @@ public void testPopulateFixedSizeBinaryVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, ("test" + i).getBytes()); + vector1.set(i, ("test" + i).getBytes(StandardCharsets.UTF_8)); } } vector1.setValueCount(10); - setVector(vector2, null, "test1".getBytes(), null, "test3".getBytes(), null, "test5".getBytes(), null, - "test7".getBytes(), null, "test9".getBytes()); + setVector(vector2, null, "test1".getBytes(StandardCharsets.UTF_8), null, + "test3".getBytes(StandardCharsets.UTF_8), null, "test5".getBytes(StandardCharsets.UTF_8), null, + "test7".getBytes(StandardCharsets.UTF_8), null, "test9".getBytes(StandardCharsets.UTF_8)); assertTrue(VectorEqualsVisitor.vectorEquals(vector1, vector2)); } } @@ -571,13 +574,14 @@ public void testPopulateVarBinaryVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, ("test" + i).getBytes()); + vector1.set(i, ("test" + i).getBytes(StandardCharsets.UTF_8)); } } vector1.setValueCount(10); - setVector(vector2, null, "test1".getBytes(), null, "test3".getBytes(), null, "test5".getBytes(), null, - "test7".getBytes(), null, "test9".getBytes()); + setVector(vector2, null, "test1".getBytes(StandardCharsets.UTF_8), null, + "test3".getBytes(StandardCharsets.UTF_8), null, "test5".getBytes(StandardCharsets.UTF_8), null, + "test7".getBytes(StandardCharsets.UTF_8), null, "test9".getBytes(StandardCharsets.UTF_8)); assertTrue(VectorEqualsVisitor.vectorEquals(vector1, vector2)); } } @@ -592,7 +596,7 @@ public void testPopulateVarCharVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, ("test" + i).getBytes()); + vector1.set(i, ("test" + i).getBytes(StandardCharsets.UTF_8)); } } vector1.setValueCount(10); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 084350410a4f5..872b2f3934b07 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -221,7 +221,7 @@ public void roundtripLocation() throws IOException { final ExtensionTypeVector deserialized = (ExtensionTypeVector) readerRoot.getFieldVectors().get(0); Assert.assertTrue(deserialized instanceof LocationVector); - Assert.assertEquals(deserialized.getName(), "location"); + Assert.assertEquals("location", deserialized.getName()); StructVector deserStruct = (StructVector) deserialized.getUnderlyingVector(); Assert.assertNotNull(deserStruct.getChild("Latitude")); Assert.assertNotNull(deserStruct.getChild("Longitude")); @@ -273,7 +273,7 @@ public void testVectorCompare() { // Test out vector appender VectorBatchAppender.batchAppend(a1, a2, bb); - assertEquals(a1.getValueCount(), 6); + assertEquals(6, a1.getValueCount()); validateVisitor.visit(a1, null); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestElementAddressableVectorIterator.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestElementAddressableVectorIterator.java index 419872225e16f..1c8281c85981b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestElementAddressableVectorIterator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestElementAddressableVectorIterator.java @@ -20,6 +20,8 @@ import static junit.framework.TestCase.assertNull; import static org.junit.Assert.assertEquals; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.ArrowBufPointer; @@ -98,7 +100,7 @@ public void testIterateVarCharVector() { if (i == 0) { strVector.setNull(i); } else { - strVector.set(i, String.valueOf(i).getBytes()); + strVector.set(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); } } @@ -125,7 +127,7 @@ public void testIterateVarCharVector() { assertEquals(expected.length(), pt.getLength()); pt.getBuf().getBytes(pt.getOffset(), actual); - assertEquals(expected, new String(actual)); + assertEquals(expected, new String(actual, StandardCharsets.UTF_8)); } index += 1; } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestReusableByteArray.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestReusableByteArray.java index b11aa5638d651..f562e63b4bf8d 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestReusableByteArray.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestReusableByteArray.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; @@ -54,25 +55,27 @@ public void testSetByteArrayRepeatedly() { ReusableByteArray byteArray = new ReusableByteArray(); try (ArrowBuf workingBuf = allocator.buffer(100)) { final String str = "test"; - workingBuf.setBytes(0, str.getBytes()); - byteArray.set(workingBuf, 0, str.getBytes().length); - assertEquals(str.getBytes().length, byteArray.getLength()); - assertArrayEquals(str.getBytes(), Arrays.copyOfRange(byteArray.getBuffer(), 0, (int) byteArray.getLength())); - assertEquals(Base64.getEncoder().encodeToString(str.getBytes()), byteArray.toString()); - assertEquals(new ReusableByteArray(str.getBytes()), byteArray); - assertEquals(new ReusableByteArray(str.getBytes()).hashCode(), byteArray.hashCode()); + workingBuf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); + byteArray.set(workingBuf, 0, str.getBytes(StandardCharsets.UTF_8).length); + assertEquals(str.getBytes(StandardCharsets.UTF_8).length, byteArray.getLength()); + assertArrayEquals(str.getBytes(StandardCharsets.UTF_8), Arrays.copyOfRange(byteArray.getBuffer(), 0, + (int) byteArray.getLength())); + assertEquals(Base64.getEncoder().encodeToString(str.getBytes(StandardCharsets.UTF_8)), byteArray.toString()); + assertEquals(new ReusableByteArray(str.getBytes(StandardCharsets.UTF_8)), byteArray); + assertEquals(new ReusableByteArray(str.getBytes(StandardCharsets.UTF_8)).hashCode(), byteArray.hashCode()); // Test a longer string. Should require reallocation. final String str2 = "test_longer"; byte[] oldBuffer = byteArray.getBuffer(); workingBuf.clear(); - workingBuf.setBytes(0, str2.getBytes()); - byteArray.set(workingBuf, 0, str2.getBytes().length); - assertEquals(str2.getBytes().length, byteArray.getLength()); - assertArrayEquals(str2.getBytes(), Arrays.copyOfRange(byteArray.getBuffer(), 0, (int) byteArray.getLength())); - assertEquals(Base64.getEncoder().encodeToString(str2.getBytes()), byteArray.toString()); - assertEquals(new ReusableByteArray(str2.getBytes()), byteArray); - assertEquals(new ReusableByteArray(str2.getBytes()).hashCode(), byteArray.hashCode()); + workingBuf.setBytes(0, str2.getBytes(StandardCharsets.UTF_8)); + byteArray.set(workingBuf, 0, str2.getBytes(StandardCharsets.UTF_8).length); + assertEquals(str2.getBytes(StandardCharsets.UTF_8).length, byteArray.getLength()); + assertArrayEquals(str2.getBytes(StandardCharsets.UTF_8), Arrays.copyOfRange(byteArray.getBuffer(), 0, + (int) byteArray.getLength())); + assertEquals(Base64.getEncoder().encodeToString(str2.getBytes(StandardCharsets.UTF_8)), byteArray.toString()); + assertEquals(new ReusableByteArray(str2.getBytes(StandardCharsets.UTF_8)), byteArray); + assertEquals(new ReusableByteArray(str2.getBytes(StandardCharsets.UTF_8)).hashCode(), byteArray.hashCode()); // Verify reallocation needed. assertNotSame(oldBuffer, byteArray.getBuffer()); @@ -82,13 +85,14 @@ public void testSetByteArrayRepeatedly() { final String str3 = "short"; oldBuffer = byteArray.getBuffer(); workingBuf.clear(); - workingBuf.setBytes(0, str3.getBytes()); - byteArray.set(workingBuf, 0, str3.getBytes().length); - assertEquals(str3.getBytes().length, byteArray.getLength()); - assertArrayEquals(str3.getBytes(), Arrays.copyOfRange(byteArray.getBuffer(), 0, (int) byteArray.getLength())); - assertEquals(Base64.getEncoder().encodeToString(str3.getBytes()), byteArray.toString()); - assertEquals(new ReusableByteArray(str3.getBytes()), byteArray); - assertEquals(new ReusableByteArray(str3.getBytes()).hashCode(), byteArray.hashCode()); + workingBuf.setBytes(0, str3.getBytes(StandardCharsets.UTF_8)); + byteArray.set(workingBuf, 0, str3.getBytes(StandardCharsets.UTF_8).length); + assertEquals(str3.getBytes(StandardCharsets.UTF_8).length, byteArray.getLength()); + assertArrayEquals(str3.getBytes(StandardCharsets.UTF_8), Arrays.copyOfRange(byteArray.getBuffer(), 0, + (int) byteArray.getLength())); + assertEquals(Base64.getEncoder().encodeToString(str3.getBytes(StandardCharsets.UTF_8)), byteArray.toString()); + assertEquals(new ReusableByteArray(str3.getBytes(StandardCharsets.UTF_8)), byteArray); + assertEquals(new ReusableByteArray(str3.getBytes(StandardCharsets.UTF_8)).hashCode(), byteArray.hashCode()); // Verify reallocation was not needed. assertSame(oldBuffer, byteArray.getBuffer()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index ab36ea2fd2129..93e7535947536 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -437,8 +437,6 @@ public void testAppendStructVector() { delta.accept(appender, null); assertEquals(length1 + length2, target.getValueCount()); - IntVector child1 = (IntVector) target.getVectorById(0); - VarCharVector child2 = (VarCharVector) target.getVectorById(1); try (IntVector expected1 = new IntVector("expected1", allocator); VarCharVector expected2 = new VarCharVector("expected2", allocator)) { From 3fe598ae4dfd7805ab05452dd5ed4b0d6c97d8d5 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 23 Jan 2024 18:31:40 +0100 Subject: [PATCH 73/92] GH-39666: [C++] Ensure CSV and JSON benchmarks present a bytes/s or items/s metric (#39764) ### Rationale for this change Some of our microbenchmarks only present an iteration time in (nano,micro...)seconds. That is usually tedious to read and difficult to interpret. ### What changes are included in this PR? Ensure that benchmarks present a items/seconds and/or a bytes/seconds metric where that makes sense. ### Are these changes tested? Manually. ### Are there any user-facing changes? No. * Closes: #39666 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/csv/writer_benchmark.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/csv/writer_benchmark.cc b/cpp/src/arrow/csv/writer_benchmark.cc index 54c0f50613754..9baa00d48a6d2 100644 --- a/cpp/src/arrow/csv/writer_benchmark.cc +++ b/cpp/src/arrow/csv/writer_benchmark.cc @@ -97,7 +97,7 @@ void BenchmarkWriteCsv(benchmark::State& state, const WriteOptions& options, const RecordBatch& batch) { int64_t total_size = 0; - while (state.KeepRunning()) { + for (auto _ : state) { auto out = io::BufferOutputStream::Create().ValueOrDie(); ABORT_NOT_OK(WriteCSV(batch, options, out.get())); auto buffer = out->Finish().ValueOrDie(); @@ -106,6 +106,7 @@ void BenchmarkWriteCsv(benchmark::State& state, const WriteOptions& options, // byte size of the generated csv dataset state.SetBytesProcessed(total_size); + state.SetItemsProcessed(state.iterations() * batch.num_columns() * batch.num_rows()); state.counters["null_percent"] = static_cast(state.range(0)); } From df83e50cdbc956846476a1dbcd5f09ef7058ed58 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 24 Jan 2024 11:22:51 +0100 Subject: [PATCH 74/92] GH-39667: [C++] Ensure dataset benchmarks present a bytes/s or items/s metric (#39766) ### Rationale for this change Some of our microbenchmarks only present an iteration time in (nano,micro...)seconds. That is usually tedious to read and difficult to interpret. ### What changes are included in this PR? Ensure that benchmarks present a items/seconds and/or a bytes/seconds metric where that makes sense. ### Are these changes tested? Manually. ### Are there any user-facing changes? No. * Closes: #39667 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/dataset/file_benchmark.cc | 27 +++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/dataset/file_benchmark.cc b/cpp/src/arrow/dataset/file_benchmark.cc index 8953cbd110643..f687392d13601 100644 --- a/cpp/src/arrow/dataset/file_benchmark.cc +++ b/cpp/src/arrow/dataset/file_benchmark.cc @@ -30,7 +30,12 @@ namespace arrow { namespace dataset { -static std::shared_ptr GetDataset() { +struct SampleDataset { + std::shared_ptr dataset; + int64_t num_fragments; +}; + +static SampleDataset GetDataset() { std::vector files; std::vector paths; for (int a = 0; a < 100; a++) { @@ -50,25 +55,35 @@ static std::shared_ptr GetDataset() { FinishOptions finish_options; finish_options.inspect_options.fragments = 0; EXPECT_OK_AND_ASSIGN(auto dataset, factory->Finish(finish_options)); - return dataset; + return {dataset, static_cast(paths.size())}; } // A benchmark of filtering fragments in a dataset. static void GetAllFragments(benchmark::State& state) { auto dataset = GetDataset(); for (auto _ : state) { - ASSERT_OK_AND_ASSIGN(auto fragments, dataset->GetFragments()); + ASSERT_OK_AND_ASSIGN(auto fragments, dataset.dataset->GetFragments()); ABORT_NOT_OK(fragments.Visit([](std::shared_ptr) { return Status::OK(); })); } + state.SetItemsProcessed(state.iterations() * dataset.num_fragments); + state.counters["num_fragments"] = dataset.num_fragments; } static void GetFilteredFragments(benchmark::State& state, compute::Expression filter) { auto dataset = GetDataset(); - ASSERT_OK_AND_ASSIGN(filter, filter.Bind(*dataset->schema())); + ASSERT_OK_AND_ASSIGN(filter, filter.Bind(*dataset.dataset->schema())); + int64_t num_filtered_fragments = 0; for (auto _ : state) { - ASSERT_OK_AND_ASSIGN(auto fragments, dataset->GetFragments(filter)); - ABORT_NOT_OK(fragments.Visit([](std::shared_ptr) { return Status::OK(); })); + num_filtered_fragments = 0; + ASSERT_OK_AND_ASSIGN(auto fragments, dataset.dataset->GetFragments(filter)); + ABORT_NOT_OK(fragments.Visit([&](std::shared_ptr) { + ++num_filtered_fragments; + return Status::OK(); + })); } + state.SetItemsProcessed(state.iterations() * dataset.num_fragments); + state.counters["num_fragments"] = dataset.num_fragments; + state.counters["num_filtered_fragments"] = num_filtered_fragments; } using compute::field_ref; From 2e8bd8d0b53560a561656337021abc3e4aa73f8c Mon Sep 17 00:00:00 2001 From: Gabriel Tomitsuka Date: Wed, 24 Jan 2024 15:31:51 +0100 Subject: [PATCH 75/92] GH-39761: [Docs] Link to Go documentation references outdated documentation from 2018 (#39750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change The godoc format shows docs in a versioned fashion, making the ["basic" link](https://pkg.go.dev/github.com/apache/arrow/go) reference v0.0.0 from 2018, and godoc doesn't warn you when docs are outdated. Accordingly, for people to find "real" documentation, they have to manually search for the latest docs. Otherwise, it gives the impression that the package is abandoned / makes devs waste considerable time — e.g., I work with Go professionally and wasted 15 minutes on the 2018 docs until I realized they were wrong. ### What changes are included in this PR? * Reference the latest version (v16) in the list of supported implementations of the Arrow website & R package. * Add step to release process that automatically updates the version on release ### Are these changes tested? Yes, I've added tests for this to `dev/release/post-11-bump-versions-test.rb`. ### Are there any user-facing changes? Yes, the link to Implementations > Go on the website will change. * Closes: #39761 Authored-by: Gabriel Tomitsuka Signed-off-by: Sutou Kouhei --- dev/release/post-11-bump-versions-test.rb | 18 ++++++++++++++++++ dev/release/utils-prepare.sh | 17 +++++++++++++++++ docs/source/index.rst | 2 +- r/_pkgdown.yml | 2 +- 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/dev/release/post-11-bump-versions-test.rb b/dev/release/post-11-bump-versions-test.rb index 4b6933d6102a9..78d9320bfb312 100644 --- a/dev/release/post-11-bump-versions-test.rb +++ b/dev/release/post-11-bump-versions-test.rb @@ -197,6 +197,15 @@ def test_version_post_tag ] if release_type == :major expected_changes += [ + { + path: "docs/source/index.rst", + hunks: [ + [ + "- Go ", + "+ Go ", + ], + ], + }, { path: "r/pkgdown/assets/versions.json", hunks: [ @@ -212,6 +221,15 @@ def test_version_post_tag ], ], }, + { + path: "r/_pkgdown.yml", + hunks: [ + [ + "- [Go](https://pkg.go.dev/github.com/apache/arrow/go/v#{@snapshot_major_version})
", + "+ [Go](https://pkg.go.dev/github.com/apache/arrow/go/v#{@next_major_version})
", + ], + ], + }, ] else expected_changes += [ diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index 8e4c8a84ae8fd..51367087228a4 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -127,6 +127,7 @@ update_versions() { DESCRIPTION rm -f DESCRIPTION.bak git add DESCRIPTION + # Replace dev version with release version sed -i.bak -E -e \ "/^