From 4e34a20a31fae2546f9cfbaa520d7561b80563c7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 1 Jul 2024 11:18:25 -0500 Subject: [PATCH 01/53] Backport: Fix segfault in conditional join (#16094) (#16100) Backports #16094 to 24.06 for inclusion in a hotfix release. --- cpp/src/join/conditional_join.cu | 13 +--- cpp/tests/join/conditional_join_tests.cu | 92 +++++++++++++++++------- 2 files changed, 70 insertions(+), 35 deletions(-) diff --git a/cpp/src/join/conditional_join.cu b/cpp/src/join/conditional_join.cu index f02dee5f7f5..97a06d5a923 100644 --- a/cpp/src/join/conditional_join.cu +++ b/cpp/src/join/conditional_join.cu @@ -48,8 +48,7 @@ std::unique_ptr> conditional_join_anti_semi( { if (right.num_rows() == 0) { switch (join_type) { - case join_kind::LEFT_ANTI_JOIN: - return std::make_unique>(left.num_rows(), stream, mr); + case join_kind::LEFT_ANTI_JOIN: return get_trivial_left_join_indices(left, stream, mr).first; case join_kind::LEFT_SEMI_JOIN: return std::make_unique>(0, stream, mr); default: CUDF_FAIL("Invalid join kind."); break; @@ -96,10 +95,6 @@ std::unique_ptr> conditional_join_anti_semi( join_size = size.value(stream); } - if (left.num_rows() == 0) { - return std::make_unique>(0, stream, mr); - } - rmm::device_scalar write_index(0, stream); auto left_indices = std::make_unique>(join_size, stream, mr); @@ -149,8 +144,7 @@ conditional_join(table_view const& left, // with a corresponding NULL from the right. case join_kind::LEFT_JOIN: case join_kind::LEFT_ANTI_JOIN: - case join_kind::FULL_JOIN: - return get_trivial_left_join_indices(left, stream, rmm::mr::get_current_device_resource()); + case join_kind::FULL_JOIN: return get_trivial_left_join_indices(left, stream, mr); // Inner and left semi joins return empty output because no matches can exist. case join_kind::INNER_JOIN: case join_kind::LEFT_SEMI_JOIN: @@ -169,8 +163,7 @@ conditional_join(table_view const& left, std::make_unique>(0, stream, mr)); // Full joins need to return the trivial complement. case join_kind::FULL_JOIN: { - auto ret_flipped = - get_trivial_left_join_indices(right, stream, rmm::mr::get_current_device_resource()); + auto ret_flipped = get_trivial_left_join_indices(right, stream, mr); return std::pair(std::move(ret_flipped.second), std::move(ret_flipped.first)); } default: CUDF_FAIL("Invalid join kind."); break; diff --git a/cpp/tests/join/conditional_join_tests.cu b/cpp/tests/join/conditional_join_tests.cu index 79968bcd7f4..7ab4a2ea465 100644 --- a/cpp/tests/join/conditional_join_tests.cu +++ b/cpp/tests/join/conditional_join_tests.cu @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -222,21 +223,25 @@ struct ConditionalJoinPairReturnTest : public ConditionalJoinTest { std::vector> expected_outputs) { auto result_size = this->join_size(left, right, predicate); - EXPECT_TRUE(result_size == expected_outputs.size()); - - auto result = this->join(left, right, predicate); - std::vector> result_pairs; - for (size_t i = 0; i < result.first->size(); ++i) { - // Note: Not trying to be terribly efficient here since these tests are - // small, otherwise a batch copy to host before constructing the tuples - // would be important. - result_pairs.push_back({result.first->element(i, cudf::get_default_stream()), - result.second->element(i, cudf::get_default_stream())}); - } + EXPECT_EQ(result_size, expected_outputs.size()); + + auto result = this->join(left, right, predicate); + auto lhs_result = cudf::detail::make_std_vector_sync(*result.first, cudf::get_default_stream()); + auto rhs_result = + cudf::detail::make_std_vector_sync(*result.second, cudf::get_default_stream()); + std::vector> result_pairs(lhs_result.size()); + std::transform(lhs_result.begin(), + lhs_result.end(), + rhs_result.begin(), + result_pairs.begin(), + [](cudf::size_type lhs, cudf::size_type rhs) { + return std::pair{lhs, rhs}; + }); std::sort(result_pairs.begin(), result_pairs.end()); std::sort(expected_outputs.begin(), expected_outputs.end()); - EXPECT_TRUE(std::equal(expected_outputs.begin(), expected_outputs.end(), result_pairs.begin())); + EXPECT_TRUE(std::equal( + expected_outputs.begin(), expected_outputs.end(), result_pairs.begin(), result_pairs.end())); } /* @@ -411,6 +416,11 @@ TYPED_TEST(ConditionalInnerJoinTest, TestOneColumnLeftEmpty) this->test({{}}, {{3, 4, 5}}, left_zero_eq_right_zero, {}); }; +TYPED_TEST(ConditionalInnerJoinTest, TestOneColumnRightEmpty) +{ + this->test({{3, 4, 5}}, {{}}, left_zero_eq_right_zero, {}); +}; + TYPED_TEST(ConditionalInnerJoinTest, TestOneColumnTwoRowAllEqual) { this->test({{0, 1}}, {{0, 0}}, left_zero_eq_right_zero, {{0, 0}, {0, 1}}); @@ -600,6 +610,14 @@ TYPED_TEST(ConditionalLeftJoinTest, TestOneColumnLeftEmpty) this->test({{}}, {{3, 4, 5}}, left_zero_eq_right_zero, {}); }; +TYPED_TEST(ConditionalLeftJoinTest, TestOneColumnRightEmpty) +{ + this->test({{3, 4, 5}}, + {{}}, + left_zero_eq_right_zero, + {{0, JoinNoneValue}, {1, JoinNoneValue}, {2, JoinNoneValue}}); +}; + TYPED_TEST(ConditionalLeftJoinTest, TestCompareRandomToHash) { auto [left, right] = gen_random_repeated_columns(); @@ -666,6 +684,14 @@ TYPED_TEST(ConditionalFullJoinTest, TestOneColumnLeftEmpty) {{JoinNoneValue, 0}, {JoinNoneValue, 1}, {JoinNoneValue, 2}}); }; +TYPED_TEST(ConditionalFullJoinTest, TestOneColumnRightEmpty) +{ + this->test({{3, 4, 5}}, + {{}}, + left_zero_eq_right_zero, + {{0, JoinNoneValue}, {1, JoinNoneValue}, {2, JoinNoneValue}}); +}; + TYPED_TEST(ConditionalFullJoinTest, TestTwoColumnThreeRowSomeEqual) { this->test({{0, 1, 2}, {10, 20, 30}}, @@ -705,20 +731,16 @@ struct ConditionalJoinSingleReturnTest : public ConditionalJoinTest { auto [left_wrappers, right_wrappers, left_columns, right_columns, left, right] = this->parse_input(left_data, right_data); auto result_size = this->join_size(left, right, predicate); - EXPECT_TRUE(result_size == expected_outputs.size()); - - auto result = this->join(left, right, predicate); - std::vector resulting_indices; - for (size_t i = 0; i < result->size(); ++i) { - // Note: Not trying to be terribly efficient here since these tests are - // small, otherwise a batch copy to host before constructing the tuples - // would be important. - resulting_indices.push_back(result->element(i, cudf::get_default_stream())); - } - std::sort(resulting_indices.begin(), resulting_indices.end()); + EXPECT_EQ(result_size, expected_outputs.size()); + + auto result = this->join(left, right, predicate); + auto result_indices = cudf::detail::make_std_vector_sync(*result, cudf::get_default_stream()); + std::sort(result_indices.begin(), result_indices.end()); std::sort(expected_outputs.begin(), expected_outputs.end()); - EXPECT_TRUE( - std::equal(resulting_indices.begin(), resulting_indices.end(), expected_outputs.begin())); + EXPECT_TRUE(std::equal(result_indices.begin(), + result_indices.end(), + expected_outputs.begin(), + expected_outputs.end())); } void _compare_to_hash_join(std::unique_ptr> const& result, @@ -826,6 +848,16 @@ struct ConditionalLeftSemiJoinTest : public ConditionalJoinSingleReturnTest { TYPED_TEST_SUITE(ConditionalLeftSemiJoinTest, cudf::test::IntegralTypesNotBool); +TYPED_TEST(ConditionalLeftSemiJoinTest, TestOneColumnLeftEmpty) +{ + this->test({{}}, {{3, 4, 5}}, left_zero_eq_right_zero, {}); +}; + +TYPED_TEST(ConditionalLeftSemiJoinTest, TestOneColumnRightEmpty) +{ + this->test({{3, 4, 5}}, {{}}, left_zero_eq_right_zero, {}); +}; + TYPED_TEST(ConditionalLeftSemiJoinTest, TestTwoColumnThreeRowSomeEqual) { this->test({{0, 1, 2}, {10, 20, 30}}, {{0, 1, 3}, {30, 40, 50}}, left_zero_eq_right_zero, {0, 1}); @@ -873,6 +905,16 @@ struct ConditionalLeftAntiJoinTest : public ConditionalJoinSingleReturnTest { TYPED_TEST_SUITE(ConditionalLeftAntiJoinTest, cudf::test::IntegralTypesNotBool); +TYPED_TEST(ConditionalLeftAntiJoinTest, TestOneColumnLeftEmpty) +{ + this->test({{}}, {{3, 4, 5}}, left_zero_eq_right_zero, {}); +}; + +TYPED_TEST(ConditionalLeftAntiJoinTest, TestOneColumnRightEmpty) +{ + this->test({{3, 4, 5}}, {{}}, left_zero_eq_right_zero, {0, 1, 2}); +}; + TYPED_TEST(ConditionalLeftAntiJoinTest, TestTwoColumnThreeRowSomeEqual) { this->test({{0, 1, 2}, {10, 20, 30}}, {{0, 1, 3}, {30, 40, 50}}, left_zero_eq_right_zero, {2}); From e41242094092f9ed31fd4d04f8a30107c1ffb2ff Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Jul 2024 11:24:52 -0700 Subject: [PATCH 02/53] Backport #16038 to 24.06 (#16101) Backporting #16038 for a patch release. --------- Co-authored-by: Paul Mattione <156858817+pmattione-nvidia@users.noreply.github.com> --- cpp/include/cudf/ast/detail/operators.hpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cpp/include/cudf/ast/detail/operators.hpp b/cpp/include/cudf/ast/detail/operators.hpp index b618f33a6e5..c483d459833 100644 --- a/cpp/include/cudf/ast/detail/operators.hpp +++ b/cpp/include/cudf/ast/detail/operators.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -819,7 +820,17 @@ struct operator_functor { template struct cast { static constexpr auto arity{1}; - template + template ()>* = nullptr> + __device__ inline auto operator()(From f) -> To + { + if constexpr (cuda::std::is_floating_point_v) { + return convert_fixed_to_floating(f); + } else { + return static_cast(f); + } + } + + template ()>* = nullptr> __device__ inline auto operator()(From f) -> decltype(static_cast(f)) { return static_cast(f); From dfab1b589e5907b324dc1688f6dab862d194012c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 1 Jul 2024 15:33:42 -0500 Subject: [PATCH 03/53] Backport: Use size_t to allow large conditional joins (#16127) (#16133) Backports #16127 to 24.06 for inclusion in a hotfix release. --------- Co-authored-by: Vyas Ramasubramani --- cpp/src/join/conditional_join.cu | 5 +- cpp/src/join/conditional_join_kernels.cuh | 124 ++++++++++++++++++++-- cpp/src/join/join_common_utils.cuh | 95 ----------------- 3 files changed, 117 insertions(+), 107 deletions(-) diff --git a/cpp/src/join/conditional_join.cu b/cpp/src/join/conditional_join.cu index 97a06d5a923..d4ef2747c9d 100644 --- a/cpp/src/join/conditional_join.cu +++ b/cpp/src/join/conditional_join.cu @@ -95,7 +95,7 @@ std::unique_ptr> conditional_join_anti_semi( join_size = size.value(stream); } - rmm::device_scalar write_index(0, stream); + rmm::device_scalar write_index(0, stream); auto left_indices = std::make_unique>(join_size, stream, mr); @@ -232,13 +232,14 @@ conditional_join(table_view const& left, std::make_unique>(0, stream, mr)); } - rmm::device_scalar write_index(0, stream); + rmm::device_scalar write_index(0, stream); auto left_indices = std::make_unique>(join_size, stream, mr); auto right_indices = std::make_unique>(join_size, stream, mr); auto const& join_output_l = left_indices->data(); auto const& join_output_r = right_indices->data(); + if (has_nulls) { conditional_join <<>>( diff --git a/cpp/src/join/conditional_join_kernels.cuh b/cpp/src/join/conditional_join_kernels.cuh index 1e16c451f5a..62769862f54 100644 --- a/cpp/src/join/conditional_join_kernels.cuh +++ b/cpp/src/join/conditional_join_kernels.cuh @@ -29,6 +29,110 @@ namespace cudf { namespace detail { +/** + * @brief Adds a pair of indices to the shared memory cache + * + * @param[in] first The first index in the pair + * @param[in] second The second index in the pair + * @param[in,out] current_idx_shared Pointer to shared index that determines + * where in the shared memory cache the pair will be written + * @param[in] warp_id The ID of the warp of the calling the thread + * @param[out] joined_shared_l Pointer to the shared memory cache for left indices + * @param[out] joined_shared_r Pointer to the shared memory cache for right indices + */ +__inline__ __device__ void add_pair_to_cache(size_type const first, + size_type const second, + std::size_t* current_idx_shared, + int const warp_id, + size_type* joined_shared_l, + size_type* joined_shared_r) +{ + cuda::atomic_ref ref{*(current_idx_shared + warp_id)}; + std::size_t my_current_idx = ref.fetch_add(1, cuda::memory_order_relaxed); + // It's guaranteed to fit into the shared cache + joined_shared_l[my_current_idx] = first; + joined_shared_r[my_current_idx] = second; +} + +__inline__ __device__ void add_left_to_cache(size_type const first, + std::size_t* current_idx_shared, + int const warp_id, + size_type* joined_shared_l) +{ + cuda::atomic_ref ref{*(current_idx_shared + warp_id)}; + std::size_t my_current_idx = ref.fetch_add(1, cuda::memory_order_relaxed); + joined_shared_l[my_current_idx] = first; +} + +template +__device__ void flush_output_cache(unsigned int const activemask, + std::size_t const max_size, + int const warp_id, + int const lane_id, + std::size_t* current_idx, + std::size_t current_idx_shared[num_warps], + size_type join_shared_l[num_warps][output_cache_size], + size_type join_shared_r[num_warps][output_cache_size], + size_type* join_output_l, + size_type* join_output_r) +{ + // count how many active threads participating here which could be less than warp_size + int const num_threads = __popc(activemask); + std::size_t output_offset = 0; + + if (0 == lane_id) { + cuda::atomic_ref ref{*current_idx}; + output_offset = ref.fetch_add(current_idx_shared[warp_id], cuda::memory_order_relaxed); + } + + // No warp sync is necessary here because we are assuming that ShuffleIndex + // is internally using post-CUDA 9.0 synchronization-safe primitives + // (__shfl_sync instead of __shfl). __shfl is technically not guaranteed to + // be safe by the compiler because it is not required by the standard to + // converge divergent branches before executing. + output_offset = cub::ShuffleIndex(output_offset, 0, activemask); + + for (std::size_t shared_out_idx = static_cast(lane_id); + shared_out_idx < current_idx_shared[warp_id]; + shared_out_idx += num_threads) { + std::size_t thread_offset = output_offset + shared_out_idx; + if (thread_offset < max_size) { + join_output_l[thread_offset] = join_shared_l[warp_id][shared_out_idx]; + join_output_r[thread_offset] = join_shared_r[warp_id][shared_out_idx]; + } + } +} + +template +__device__ void flush_output_cache(unsigned int const activemask, + std::size_t const max_size, + int const warp_id, + int const lane_id, + std::size_t* current_idx, + std::size_t current_idx_shared[num_warps], + size_type join_shared_l[num_warps][output_cache_size], + size_type* join_output_l) +{ + int const num_threads = __popc(activemask); + std::size_t output_offset = 0; + + if (0 == lane_id) { + cuda::atomic_ref ref{*current_idx}; + output_offset = ref.fetch_add(current_idx_shared[warp_id], cuda::memory_order_relaxed); + } + + output_offset = cub::ShuffleIndex(output_offset, 0, activemask); + + for (std::size_t shared_out_idx = static_cast(lane_id); + shared_out_idx < current_idx_shared[warp_id]; + shared_out_idx += num_threads) { + std::size_t thread_offset = output_offset + shared_out_idx; + if (thread_offset < max_size) { + join_output_l[thread_offset] = join_shared_l[warp_id][shared_out_idx]; + } + } +} + /** * @brief Computes the output size of joining the left table to the right table. * @@ -103,14 +207,14 @@ CUDF_KERNEL void compute_conditional_join_output_size( } } - using BlockReduce = cub::BlockReduce; + using BlockReduce = cub::BlockReduce; __shared__ typename BlockReduce::TempStorage temp_storage; std::size_t block_counter = BlockReduce(temp_storage).Sum(thread_counter); // Add block counter to global counter if (threadIdx.x == 0) { cuda::atomic_ref ref{*output_size}; - ref.fetch_add(block_counter, cuda::std::memory_order_relaxed); + ref.fetch_add(block_counter, cuda::memory_order_relaxed); } } @@ -143,13 +247,13 @@ CUDF_KERNEL void conditional_join(table_device_view left_table, join_kind join_type, cudf::size_type* join_output_l, cudf::size_type* join_output_r, - cudf::size_type* current_idx, + std::size_t* current_idx, cudf::ast::detail::expression_device_view device_expression_data, - cudf::size_type const max_size, + std::size_t const max_size, bool const swap_tables) { constexpr int num_warps = block_size / detail::warp_size; - __shared__ cudf::size_type current_idx_shared[num_warps]; + __shared__ std::size_t current_idx_shared[num_warps]; __shared__ cudf::size_type join_shared_l[num_warps][output_cache_size]; __shared__ cudf::size_type join_shared_r[num_warps][output_cache_size]; @@ -183,7 +287,7 @@ CUDF_KERNEL void conditional_join(table_device_view left_table, if (outer_row_index < outer_num_rows) { bool found_match = false; - for (thread_index_type inner_row_index(0); inner_row_index < inner_num_rows; + for (cudf::thread_index_type inner_row_index(0); inner_row_index < inner_num_rows; ++inner_row_index) { auto output_dest = cudf::ast::detail::value_expression_result(); auto const left_row_index = swap_tables ? inner_row_index : outer_row_index; @@ -277,12 +381,12 @@ CUDF_KERNEL void conditional_join_anti_semi( table_device_view right_table, join_kind join_type, cudf::size_type* join_output_l, - cudf::size_type* current_idx, + std::size_t* current_idx, cudf::ast::detail::expression_device_view device_expression_data, - cudf::size_type const max_size) + std::size_t const max_size) { constexpr int num_warps = block_size / detail::warp_size; - __shared__ cudf::size_type current_idx_shared[num_warps]; + __shared__ std::size_t current_idx_shared[num_warps]; __shared__ cudf::size_type join_shared_l[num_warps][output_cache_size]; extern __shared__ char raw_intermediate_storage[]; @@ -310,7 +414,7 @@ CUDF_KERNEL void conditional_join_anti_semi( for (cudf::thread_index_type outer_row_index = start_idx; outer_row_index < outer_num_rows; outer_row_index += stride) { bool found_match = false; - for (thread_index_type inner_row_index(0); inner_row_index < inner_num_rows; + for (cudf::thread_index_type inner_row_index(0); inner_row_index < inner_num_rows; ++inner_row_index) { auto output_dest = cudf::ast::detail::value_expression_result(); diff --git a/cpp/src/join/join_common_utils.cuh b/cpp/src/join/join_common_utils.cuh index 31f267d5cfb..3d0f3e4340d 100644 --- a/cpp/src/join/join_common_utils.cuh +++ b/cpp/src/join/join_common_utils.cuh @@ -262,101 +262,6 @@ struct valid_range { } }; -/** - * @brief Adds a pair of indices to the shared memory cache - * - * @param[in] first The first index in the pair - * @param[in] second The second index in the pair - * @param[in,out] current_idx_shared Pointer to shared index that determines - * where in the shared memory cache the pair will be written - * @param[in] warp_id The ID of the warp of the calling the thread - * @param[out] joined_shared_l Pointer to the shared memory cache for left indices - * @param[out] joined_shared_r Pointer to the shared memory cache for right indices - */ -__inline__ __device__ void add_pair_to_cache(size_type const first, - size_type const second, - size_type* current_idx_shared, - int const warp_id, - size_type* joined_shared_l, - size_type* joined_shared_r) -{ - size_type my_current_idx{atomicAdd(current_idx_shared + warp_id, size_type(1))}; - // its guaranteed to fit into the shared cache - joined_shared_l[my_current_idx] = first; - joined_shared_r[my_current_idx] = second; -} - -__inline__ __device__ void add_left_to_cache(size_type const first, - size_type* current_idx_shared, - int const warp_id, - size_type* joined_shared_l) -{ - size_type my_current_idx{atomicAdd(current_idx_shared + warp_id, size_type(1))}; - - joined_shared_l[my_current_idx] = first; -} - -template -__device__ void flush_output_cache(unsigned int const activemask, - cudf::size_type const max_size, - int const warp_id, - int const lane_id, - cudf::size_type* current_idx, - cudf::size_type current_idx_shared[num_warps], - size_type join_shared_l[num_warps][output_cache_size], - size_type join_shared_r[num_warps][output_cache_size], - size_type* join_output_l, - size_type* join_output_r) -{ - // count how many active threads participating here which could be less than warp_size - int const num_threads = __popc(activemask); - cudf::size_type output_offset = 0; - - if (0 == lane_id) { output_offset = atomicAdd(current_idx, current_idx_shared[warp_id]); } - - // No warp sync is necessary here because we are assuming that ShuffleIndex - // is internally using post-CUDA 9.0 synchronization-safe primitives - // (__shfl_sync instead of __shfl). __shfl is technically not guaranteed to - // be safe by the compiler because it is not required by the standard to - // converge divergent branches before executing. - output_offset = cub::ShuffleIndex(output_offset, 0, activemask); - - for (int shared_out_idx = lane_id; shared_out_idx < current_idx_shared[warp_id]; - shared_out_idx += num_threads) { - cudf::size_type thread_offset = output_offset + shared_out_idx; - if (thread_offset < max_size) { - join_output_l[thread_offset] = join_shared_l[warp_id][shared_out_idx]; - join_output_r[thread_offset] = join_shared_r[warp_id][shared_out_idx]; - } - } -} - -template -__device__ void flush_output_cache(unsigned int const activemask, - cudf::size_type const max_size, - int const warp_id, - int const lane_id, - cudf::size_type* current_idx, - cudf::size_type current_idx_shared[num_warps], - size_type join_shared_l[num_warps][output_cache_size], - size_type* join_output_l) -{ - int const num_threads = __popc(activemask); - cudf::size_type output_offset = 0; - - if (0 == lane_id) { output_offset = atomicAdd(current_idx, current_idx_shared[warp_id]); } - - output_offset = cub::ShuffleIndex(output_offset, 0, activemask); - - for (int shared_out_idx = lane_id; shared_out_idx < current_idx_shared[warp_id]; - shared_out_idx += num_threads) { - cudf::size_type thread_offset = output_offset + shared_out_idx; - if (thread_offset < max_size) { - join_output_l[thread_offset] = join_shared_l[warp_id][shared_out_idx]; - } - } -} - } // namespace detail } // namespace cudf From 781794bb52448f617351ed96441a8e2fdb765dd7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 1 Jul 2024 14:59:04 -0700 Subject: [PATCH 04/53] Backport #16045 to 24.06 (#16102) Backporting #16045 for a patch release. --------- Co-authored-by: Paul Mattione <156858817+pmattione-nvidia@users.noreply.github.com> --- cpp/tests/ast/transform_tests.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cpp/tests/ast/transform_tests.cpp b/cpp/tests/ast/transform_tests.cpp index ef1d09e5652..6b350c137d0 100644 --- a/cpp/tests/ast/transform_tests.cpp +++ b/cpp/tests/ast/transform_tests.cpp @@ -65,6 +65,22 @@ TEST_F(TransformTest, ColumnReference) CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); } +TEST_F(TransformTest, BasicAdditionDoubleCast) +{ + auto c_0 = column_wrapper{3, 20, 1, 50}; + std::vector<__int128_t> data1{10, 7, 20, 0}; + auto c_1 = cudf::test::fixed_point_column_wrapper<__int128_t>( + data1.begin(), data1.end(), numeric::scale_type{0}); + auto table = cudf::table_view{{c_0, c_1}}; + auto col_ref_0 = cudf::ast::column_reference(0); + auto col_ref_1 = cudf::ast::column_reference(1); + auto cast = cudf::ast::operation(cudf::ast::ast_operator::CAST_TO_FLOAT64, col_ref_1); + auto expression = cudf::ast::operation(cudf::ast::ast_operator::ADD, col_ref_0, cast); + auto expected = column_wrapper{13, 27, 21, 50}; + auto result = cudf::compute_column(table, expression); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view(), verbosity); +} + TEST_F(TransformTest, Literal) { auto c_0 = column_wrapper{3, 20, 1, 50}; From 1889c7c0f517c95143016a6e391275144a034f7a Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 15 Jul 2024 20:32:15 +0200 Subject: [PATCH 05/53] MAINT: Adapt to NumPy 2 promotion changes (#16141) Splitting out the non API changes from gh-15897, the Scalar API change is required for the tests to pass with NumPy 2, but almost all changes should be relatively straight forward here on their own. (I will add inline comments.) --- This PR does not fix integer comparisons, there are currently no tests that run into these. xref: https://github.com/rapidsai/build-planning/issues/38 Authors: - Sebastian Berg (https://github.com/seberg) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16141 --- python/cudf/cudf/core/_internals/where.py | 24 +++++++++++------- python/cudf/cudf/core/column/categorical.py | 4 ++- python/cudf/cudf/core/column/numerical.py | 27 ++++++++++++++++----- python/cudf/cudf/tests/test_binops.py | 21 +++++++++++++--- python/cudf/cudf/tests/test_doctests.py | 13 +++++++++- python/cudf/cudf/tests/test_dtypes.py | 1 - 6 files changed, 69 insertions(+), 21 deletions(-) diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 44ce0ddef25..f3183e6029d 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -54,13 +54,17 @@ def _check_and_cast_columns_with_other( other_is_scalar = is_scalar(other) if other_is_scalar: - if (isinstance(other, float) and not np.isnan(other)) and ( - source_dtype.type(other) != other - ): - raise TypeError( - f"Cannot safely cast non-equivalent " - f"{type(other).__name__} to {source_dtype.name}" - ) + if isinstance(other, float) and not np.isnan(other): + try: + is_safe = source_dtype.type(other) == other + except OverflowError: + is_safe = False + + if not is_safe: + raise TypeError( + f"Cannot safely cast non-equivalent " + f"{type(other).__name__} to {source_dtype.name}" + ) if cudf.utils.utils.is_na_like(other): return _normalize_categorical( @@ -84,8 +88,10 @@ def _check_and_cast_columns_with_other( ) return _normalize_categorical(source_col, other.astype(source_dtype)) - if _is_non_decimal_numeric_dtype(source_dtype) and _can_cast( - other, source_dtype + if ( + _is_non_decimal_numeric_dtype(source_dtype) + and not other_is_scalar # can-cast fails for Python scalars + and _can_cast(other, source_dtype) ): common_dtype = source_dtype elif ( diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index f763d3b4b0c..9aaccca349d 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -47,7 +47,9 @@ ) -_DEFAULT_CATEGORICAL_VALUE = -1 +# Using np.int8(-1) to allow silent wrap-around when casting to uint +# it may make sense to make this dtype specific or a function. +_DEFAULT_CATEGORICAL_VALUE = np.int8(-1) class CategoricalAccessor(ColumnMethods): diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index a0550bff72b..b8fa00e9643 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -301,15 +301,28 @@ def normalize_binop_value( if isinstance(other, cudf.Scalar): if self.dtype == other.dtype: return other + # expensive device-host transfer just to # adjust the dtype other = other.value + + # NumPy 2 needs a Python scalar to do weak promotion, but + # pandas forces weak promotion always + # TODO: We could use 0, 0.0, and 0j for promotion to avoid copies. + if other.dtype.kind in "ifc": + other = other.item() + elif not isinstance(other, (int, float, complex)): + # Go via NumPy to get the value + other = np.array(other) + if other.dtype.kind in "ifc": + other = other.item() + # Try and match pandas and hence numpy. Deduce the common - # dtype via the _value_ of other, and the dtype of self. TODO: - # When NEP50 is accepted, this might want changed or - # simplified. - # This is not at all simple: - # np.result_type(np.int64(0), np.uint8) + # dtype via the _value_ of other, and the dtype of self on NumPy 1.x + # with NumPy 2, we force weak promotion even for our/NumPy scalars + # to match pandas 2.2. + # Weak promotion is not at all simple: + # np.result_type(0, np.uint8) # => np.uint8 # np.result_type(np.asarray([0], dtype=np.int64), np.uint8) # => np.int64 @@ -626,7 +639,9 @@ def can_cast_safely(self, to_dtype: DtypeObj) -> bool: min_, max_ = iinfo.min, iinfo.max # best we can do is hope to catch it here and avoid compare - if (self.min() >= min_) and (self.max() <= max_): + # Use Python floats, which have precise comparison for float64. + # NOTE(seberg): it would make sense to limit to the mantissa range. + if (float(self.min()) >= min_) and (float(self.max()) <= max_): filled = self.fillna(0) return (cudf.Series(filled) % 1 == 0).all() else: diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 7d8c3b53115..5265278db4c 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -539,7 +539,14 @@ def test_series_reflected_ops_scalar(func, dtype, obj_class): if obj_class == "Index": gs = Index(gs) - gs_result = func(gs) + try: + gs_result = func(gs) + except OverflowError: + # An error is fine, if pandas raises the same error: + with pytest.raises(OverflowError): + func(random_series) + + return # class typing if obj_class == "Index": @@ -589,7 +596,14 @@ def test_series_reflected_ops_cudf_scalar(funcs, dtype, obj_class): if obj_class == "Index": gs = Index(gs) - gs_result = gpu_func(gs) + try: + gs_result = gpu_func(gs) + except OverflowError: + # An error is fine, if pandas raises the same error: + with pytest.raises(OverflowError): + cpu_func(random_series) + + return # class typing if obj_class == "Index": @@ -770,7 +784,8 @@ def test_operator_func_series_and_scalar( fill_value=fill_value, ) pdf_series_result = getattr(pdf_series, func)( - scalar, fill_value=fill_value + np.array(scalar)[()] if use_cudf_scalar else scalar, + fill_value=fill_value, ) assert_eq(pdf_series_result, gdf_series_result) diff --git a/python/cudf/cudf/tests/test_doctests.py b/python/cudf/cudf/tests/test_doctests.py index 0da5c6b04d6..794660cffcb 100644 --- a/python/cudf/cudf/tests/test_doctests.py +++ b/python/cudf/cudf/tests/test_doctests.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. import contextlib import doctest import inspect @@ -8,6 +8,7 @@ import numpy as np import pytest +from packaging import version import cudf @@ -80,6 +81,16 @@ def chdir_to_tmp_path(cls, tmp_path): yield os.chdir(original_directory) + @pytest.fixture(autouse=True) + def prinoptions(cls): + # TODO: NumPy now prints scalars as `np.int8(1)`, etc. this should + # be adapted evantually. + if version.parse(np.__version__) >= version.parse("2.0"): + with np.printoptions(legacy="1.25"): + yield + else: + yield + @pytest.mark.parametrize( "docstring", itertools.chain(*[_find_doctests_in_obj(mod) for mod in tests]), diff --git a/python/cudf/cudf/tests/test_dtypes.py b/python/cudf/cudf/tests/test_dtypes.py index edb534a3618..c62b5889fdd 100644 --- a/python/cudf/cudf/tests/test_dtypes.py +++ b/python/cudf/cudf/tests/test_dtypes.py @@ -341,7 +341,6 @@ def test_dtype(in_dtype, expect): np.complex128, complex, "S", - "a", "V", "float16", np.float16, From 128f0c917bbc3342f9eca12ca2bf714c88206256 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 15 Jul 2024 20:34:14 +0200 Subject: [PATCH 06/53] API: Check for integer overflows when creating scalar form python int (#16140) This aligns with NumPy, which deprecated this since a while and raises an error now on NumPy 2, for example for `Scalar(-1, dtype=np.uint8)`. Since it aligns with NumPy, the DeprecationWarning of earlier NumPy versions is inherited for those. This (or similar handling) is required to be compatible with NumPy 2/pandas, since the default needs to be to reject operation when values are out of bounds for e.g. `uint8_series + 1000`, the 1000 should not be silently cast to a `uint8`. --- Split from gh-15897 xref: https://github.com/rapidsai/build-planning/issues/38 Authors: - Sebastian Berg (https://github.com/seberg) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16140 --- python/cudf/cudf/tests/test_scalar.py | 17 +++++++++++++++++ python/cudf/cudf/tests/test_unaops.py | 5 ++++- python/cudf/cudf/utils/dtypes.py | 14 ++++++++------ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/tests/test_scalar.py b/python/cudf/cudf/tests/test_scalar.py index 05a91a8fea3..195231e9960 100644 --- a/python/cudf/cudf/tests/test_scalar.py +++ b/python/cudf/cudf/tests/test_scalar.py @@ -8,6 +8,7 @@ import pandas as pd import pyarrow as pa import pytest +from packaging import version import rmm @@ -253,6 +254,22 @@ def test_generic_null_scalar_construction_fails(value): cudf.Scalar(value) +@pytest.mark.parametrize( + "value, dtype", [(1000, "uint8"), (2**30, "int16"), (-1, "uint16")] +) +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +def test_scalar_out_of_bounds_pyint_fails(value, dtype): + # Test that we align with NumPy on scalar creation behavior from + # Python integers. + if version.parse(np.__version__) >= version.parse("2.0"): + with pytest.raises(OverflowError): + cudf.Scalar(value, dtype) + else: + # NumPy allowed this, but it gives a DeprecationWarning on newer + # versions (which cudf did not used to do). + assert cudf.Scalar(value, dtype).value == np.dtype(dtype).type(value) + + @pytest.mark.parametrize( "dtype", NUMERIC_TYPES + DATETIME_TYPES + TIMEDELTA_TYPES + ["object"] ) diff --git a/python/cudf/cudf/tests/test_unaops.py b/python/cudf/cudf/tests/test_unaops.py index dbbf4fba3a6..5f5d79c1dce 100644 --- a/python/cudf/cudf/tests/test_unaops.py +++ b/python/cudf/cudf/tests/test_unaops.py @@ -81,7 +81,10 @@ def generate_valid_scalar_unaop_combos(): @pytest.mark.parametrize("slr,dtype,op", generate_valid_scalar_unaop_combos()) def test_scalar_unary_operations(slr, dtype, op): slr_host = np.array([slr])[0].astype(cudf.dtype(dtype)) - slr_device = cudf.Scalar(slr, dtype=dtype) + # The scalar may be out of bounds, so go via array force-cast + # NOTE: This is a change in behavior + slr = np.array(slr).astype(dtype)[()] + slr_device = cudf.Scalar(slr) expect = op(slr_host) got = op(slr_device) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 2aa3129ab30..0dec857ea96 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -253,16 +253,18 @@ def to_cudf_compatible_scalar(val, dtype=None): elif isinstance(val, datetime.timedelta): val = np.timedelta64(val) - val = _maybe_convert_to_default_type( - cudf.api.types.pandas_dtype(type(val)) - ).type(val) - if dtype is not None: - if isinstance(val, str) and np.dtype(dtype).kind == "M": + dtype = np.dtype(dtype) + if isinstance(val, str) and dtype.kind == "M": # pd.Timestamp can handle str, but not np.str_ val = pd.Timestamp(str(val)).to_datetime64().astype(dtype) else: - val = val.astype(dtype) + # At least datetimes cannot be converted to scalar via dtype.type: + val = np.array(val, dtype)[()] + else: + val = _maybe_convert_to_default_type( + cudf.api.types.pandas_dtype(type(val)) + ).type(val) if val.dtype.type is np.datetime64: time_unit, _ = np.datetime_data(val.dtype) From ceb73d91c090882ec69642a78b7d791a1bf220fe Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Mon, 15 Jul 2024 12:45:51 -0700 Subject: [PATCH 07/53] Make nvcomp adapter compatible with new version macros (#16245) New nvcomp version changed the names of the version macros. This PR adds "aliasing" to the old names so rest of the code is not affected. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - MithunR (https://github.com/mythrocks) - Muhammad Haseeb (https://github.com/mhaseeb123) URL: https://github.com/rapidsai/cudf/pull/16245 --- cpp/src/io/comp/nvcomp_adapter.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index 0e34c96debd..5d0c6a8c83b 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -37,6 +37,13 @@ #include NVCOMP_ZSTD_HEADER #endif +// When building with nvcomp 4.0 or newer, map the new version macros to the old ones +#ifndef NVCOMP_MAJOR_VERSION +#define NVCOMP_MAJOR_VERSION NVCOMP_VER_MAJOR +#define NVCOMP_MINOR_VERSION NVCOMP_VER_MINOR +#define NVCOMP_PATCH_VERSION NVCOMP_VER_PATCH +#endif + #define NVCOMP_HAS_ZSTD_DECOMP(MAJOR, MINOR, PATCH) (MAJOR > 2 or (MAJOR == 2 and MINOR >= 3)) #define NVCOMP_HAS_ZSTD_COMP(MAJOR, MINOR, PATCH) (MAJOR > 2 or (MAJOR == 2 and MINOR >= 4)) From 04330f2e9e73ac71a86666c55d0fe7248eaf8db6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:23:07 -1000 Subject: [PATCH 08/53] Fix convert_dtypes with convert_integer=False/convert_floating=True (#15964) If `convert_integer=False`, there should be no attempt to convert to integer Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/15964 --- python/cudf/cudf/core/indexed_frame.py | 34 +++++++++++-------- .../cudf/cudf/tests/series/test_conversion.py | 13 +++++++ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 63fa96d0db0..30b68574960 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -6235,13 +6235,13 @@ def rank( def convert_dtypes( self, - infer_objects=True, - convert_string=True, - convert_integer=True, - convert_boolean=True, - convert_floating=True, + infer_objects: bool = True, + convert_string: bool = True, + convert_integer: bool = True, + convert_boolean: bool = True, + convert_floating: bool = True, dtype_backend=None, - ): + ) -> Self: """ Convert columns to the best possible nullable dtypes. @@ -6252,17 +6252,21 @@ def convert_dtypes( All other dtypes are always returned as-is as all dtypes in cudf are nullable. """ - result = self.copy() - - if convert_floating: - # cast any floating columns to int64 if - # they are all integer data: - for name, col in result._data.items(): + if not (convert_floating and convert_integer): + return self.copy() + else: + cols = [] + for col in self._columns: if col.dtype.kind == "f": col = col.fillna(0) - if cp.allclose(col, col.astype("int64")): - result._data[name] = col.astype("int64") - return result + as_int = col.astype("int64") + if cp.allclose(col, as_int): + cols.append(as_int) + continue + cols.append(col) + return self._from_data_like_self( + self._data._from_columns_like_self(cols, verify=False) + ) @_warn_no_dask_cudf def __dask_tokenize__(self): diff --git a/python/cudf/cudf/tests/series/test_conversion.py b/python/cudf/cudf/tests/series/test_conversion.py index e1dd359e1ba..1d680d7860d 100644 --- a/python/cudf/cudf/tests/series/test_conversion.py +++ b/python/cudf/cudf/tests/series/test_conversion.py @@ -31,5 +31,18 @@ def test_convert_dtypes(data, dtype): assert_eq(expect, got) +def test_convert_integer_false_convert_floating_true(): + data = [1.000000000000000000000000001, 1] + expected = pd.Series(data).convert_dtypes( + convert_integer=False, convert_floating=True + ) + result = ( + cudf.Series(data) + .convert_dtypes(convert_integer=False, convert_floating=True) + .to_pandas(nullable=True) + ) + assert_eq(result, expected) + + # Now write the same test, but construct a DataFrame # as input instead of parametrizing: From dba46e7a8957b8389b69e820485e319a1d314017 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:21:50 -1000 Subject: [PATCH 09/53] Replace is_datetime/timedelta_dtype checks with .kind checks (#16262) It appears this was called when we already had a dtype object so can instead just simply check the .kind attribute Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16262 --- python/cudf/cudf/_fuzz_testing/utils.py | 3 +- python/cudf/cudf/core/column/datetime.py | 7 +-- python/cudf/cudf/core/column/timedelta.py | 4 +- python/cudf/cudf/core/dataframe.py | 7 ++- python/cudf/cudf/core/scalar.py | 8 +--- python/cudf/cudf/core/tools/numeric.py | 9 +--- python/cudf/cudf/tests/test_binops.py | 7 +-- python/cudf/cudf/tests/test_dataframe.py | 4 +- python/cudf/cudf/tests/test_list.py | 7 +-- python/cudf/cudf/tests/test_scalar.py | 11 +---- python/cudf/cudf/utils/dtypes.py | 56 ++++++++--------------- 11 files changed, 37 insertions(+), 86 deletions(-) diff --git a/python/cudf/cudf/_fuzz_testing/utils.py b/python/cudf/cudf/_fuzz_testing/utils.py index e6dfe2eae62..8ce92e1c0f6 100644 --- a/python/cudf/cudf/_fuzz_testing/utils.py +++ b/python/cudf/cudf/_fuzz_testing/utils.py @@ -192,8 +192,7 @@ def convert_nulls_to_none(records, df): col for col in df.columns if df[col].dtype in pandas_dtypes_to_np_dtypes - or pd.api.types.is_datetime64_dtype(df[col].dtype) - or pd.api.types.is_timedelta64_dtype(df[col].dtype) + or df[col].dtype.kind in "mM" ] for record in records: diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 214e84028d2..409c44f6eee 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -18,7 +18,6 @@ from cudf import _lib as libcudf from cudf._lib.labeling import label_bins from cudf._lib.search import search_sorted -from cudf.api.types import is_datetime64_dtype, is_timedelta64_dtype from cudf.core._compat import PANDAS_GE_220 from cudf.core._internals.timezones import ( check_ambiguous_and_nonexistent, @@ -565,10 +564,8 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: # We check this on `other` before reflection since we already know the # dtype of `self`. - other_is_timedelta = is_timedelta64_dtype(other.dtype) - other_is_datetime64 = not other_is_timedelta and is_datetime64_dtype( - other.dtype - ) + other_is_timedelta = other.dtype.kind == "m" + other_is_datetime64 = other.dtype.kind == "M" lhs, rhs = (other, self) if reflect else (self, other) out_dtype = None diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index 2cbed9212de..36d7d9f9614 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -12,7 +12,7 @@ import cudf from cudf import _lib as libcudf -from cudf.api.types import is_scalar, is_timedelta64_dtype +from cudf.api.types import is_scalar from cudf.core.buffer import Buffer, acquire_spill_lock from cudf.core.column import ColumnBase, column, string from cudf.utils.dtypes import np_to_pa_dtype @@ -153,7 +153,7 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: this: ColumnBinaryOperand = self out_dtype = None - if is_timedelta64_dtype(other.dtype): + if other.dtype.kind == "m": # TODO: pandas will allow these operators to work but return false # when comparing to non-timedelta dtypes. We should do the same. if op in { diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index f110b788789..2aa1b95e2d1 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -33,7 +33,6 @@ from cudf.api.types import ( _is_scalar_or_zero_d_array, is_bool_dtype, - is_datetime_dtype, is_dict_like, is_dtype_equal, is_list_like, @@ -6113,7 +6112,7 @@ def _prepare_for_rowwise_op(self, method, skipna, numeric_only): else: filtered = self.copy(deep=False) - is_pure_dt = all(is_datetime_dtype(dt) for dt in filtered.dtypes) + is_pure_dt = all(dt.kind == "M" for dt in filtered.dtypes) common_dtype = find_common_type(filtered.dtypes) if ( @@ -6510,7 +6509,7 @@ def _apply_cupy_method_axis_1(self, method, *args, **kwargs): cudf.utils.dtypes.get_min_float_dtype( prepared._data[col] ) - if not is_datetime_dtype(common_dtype) + if common_dtype.kind != "M" else cudf.dtype("float64") ) .fillna(np.nan) @@ -6537,7 +6536,7 @@ def _apply_cupy_method_axis_1(self, method, *args, **kwargs): result_dtype = ( common_dtype if method in type_coerced_methods - or is_datetime_dtype(common_dtype) + or (common_dtype is not None and common_dtype.kind == "M") else None ) result = column.as_column(result, dtype=result_dtype) diff --git a/python/cudf/cudf/core/scalar.py b/python/cudf/cudf/core/scalar.py index 29460d8c67e..f6331aa1f49 100644 --- a/python/cudf/cudf/core/scalar.py +++ b/python/cudf/cudf/core/scalar.py @@ -8,7 +8,7 @@ import pyarrow as pa import cudf -from cudf.api.types import is_datetime64_dtype, is_scalar, is_timedelta64_dtype +from cudf.api.types import is_scalar from cudf.core.dtypes import ListDtype, StructDtype from cudf.core.missing import NA, NaT from cudf.core.mixins import BinaryOperand @@ -245,11 +245,7 @@ def _preprocess_host_value(self, value, dtype): dtype = cudf.dtype(dtype) if not valid: - value = ( - NaT - if is_datetime64_dtype(dtype) or is_timedelta64_dtype(dtype) - else NA - ) + value = NaT if dtype.kind in "mM" else NA return value, dtype diff --git a/python/cudf/cudf/core/tools/numeric.py b/python/cudf/cudf/core/tools/numeric.py index ef6b86a04a7..466d46f7dca 100644 --- a/python/cudf/cudf/core/tools/numeric.py +++ b/python/cudf/cudf/core/tools/numeric.py @@ -8,12 +8,7 @@ import cudf from cudf import _lib as libcudf from cudf._lib import strings as libstrings -from cudf.api.types import ( - _is_non_decimal_numeric_dtype, - is_datetime_dtype, - is_string_dtype, - is_timedelta_dtype, -) +from cudf.api.types import _is_non_decimal_numeric_dtype, is_string_dtype from cudf.core.column import as_column from cudf.core.dtypes import CategoricalDtype from cudf.utils.dtypes import can_convert_to_column @@ -114,7 +109,7 @@ def to_numeric(arg, errors="raise", downcast=None): col = as_column(arg) dtype = col.dtype - if is_datetime_dtype(dtype) or is_timedelta_dtype(dtype): + if dtype.kind in "mM": col = col.astype(cudf.dtype("int64")) elif isinstance(dtype, CategoricalDtype): cat_dtype = col.dtype.type diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index 5265278db4c..503b1a975b4 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -1694,12 +1694,7 @@ def test_scalar_null_binops(op, dtype_l, dtype_r): rhs = cudf.Scalar(cudf.NA, dtype=dtype_r) result = op(lhs, rhs) - assert result.value is ( - cudf.NaT - if cudf.api.types.is_datetime64_dtype(result.dtype) - or cudf.api.types.is_timedelta64_dtype(result.dtype) - else cudf.NA - ) + assert result.value is (cudf.NaT if result.dtype.kind in "mM" else cudf.NA) # make sure dtype is the same as had there been a valid scalar valid_lhs = cudf.Scalar(1, dtype=dtype_l) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index f40106a30f4..7ccf83e424c 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -5457,9 +5457,7 @@ def test_rowwise_ops_datetime_dtypes(data, op, skipna, numeric_only): gdf = cudf.DataFrame(data) pdf = gdf.to_pandas() - if not numeric_only and not all( - cudf.api.types.is_datetime64_dtype(dt) for dt in gdf.dtypes - ): + if not numeric_only and not all(dt.kind == "M" for dt in gdf.dtypes): with pytest.raises(TypeError): got = getattr(gdf, op)( axis=1, skipna=skipna, numeric_only=numeric_only diff --git a/python/cudf/cudf/tests/test_list.py b/python/cudf/cudf/tests/test_list.py index ec9d7995b05..36bcaa66d7d 100644 --- a/python/cudf/cudf/tests/test_list.py +++ b/python/cudf/cudf/tests/test_list.py @@ -694,12 +694,7 @@ def test_list_scalar_host_construction_null(elem_type, nesting_level): dtype = cudf.ListDtype(dtype) slr = cudf.Scalar(None, dtype=dtype) - assert slr.value is ( - cudf.NaT - if cudf.api.types.is_datetime64_dtype(slr.dtype) - or cudf.api.types.is_timedelta64_dtype(slr.dtype) - else cudf.NA - ) + assert slr.value is (cudf.NaT if slr.dtype.kind in "mM" else cudf.NA) @pytest.mark.parametrize( diff --git a/python/cudf/cudf/tests/test_scalar.py b/python/cudf/cudf/tests/test_scalar.py index 195231e9960..f2faf4343b6 100644 --- a/python/cudf/cudf/tests/test_scalar.py +++ b/python/cudf/cudf/tests/test_scalar.py @@ -212,9 +212,7 @@ def test_scalar_roundtrip(value): ) def test_null_scalar(dtype): s = cudf.Scalar(None, dtype=dtype) - if cudf.api.types.is_datetime64_dtype( - dtype - ) or cudf.api.types.is_timedelta64_dtype(dtype): + if s.dtype.kind in "mM": assert s.value is cudf.NaT else: assert s.value is cudf.NA @@ -369,12 +367,7 @@ def test_scalar_implicit_int_conversion(value): @pytest.mark.parametrize("dtype", sorted(set(ALL_TYPES) - {"category"})) def test_scalar_invalid_implicit_conversion(cls, dtype): try: - cls( - pd.NaT - if cudf.api.types.is_datetime64_dtype(dtype) - or cudf.api.types.is_timedelta64_dtype(dtype) - else pd.NA - ) + cls(pd.NaT if cudf.dtype(dtype).kind in "mM" else pd.NA) except TypeError as e: with pytest.raises(TypeError, match=re.escape(str(e))): slr = cudf.Scalar(None, dtype=dtype) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 0dec857ea96..59e5ec1df04 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -424,9 +424,7 @@ def get_time_unit(obj): def _get_nan_for_dtype(dtype): dtype = cudf.dtype(dtype) - if pd.api.types.is_datetime64_dtype( - dtype - ) or pd.api.types.is_timedelta64_dtype(dtype): + if dtype.kind in "mM": time_unit, _ = np.datetime_data(dtype) return dtype.type("nat", time_unit) elif dtype.kind == "f": @@ -527,16 +525,14 @@ def find_common_type(dtypes): return cudf.dtype("O") # Aggregate same types - dtypes = set(dtypes) + dtypes = {cudf.dtype(dtype) for dtype in dtypes} + if len(dtypes) == 1: + return dtypes.pop() if any( isinstance(dtype, cudf.core.dtypes.DecimalDtype) for dtype in dtypes ): - if all( - cudf.api.types.is_decimal_dtype(dtype) - or cudf.api.types.is_numeric_dtype(dtype) - for dtype in dtypes - ): + if all(cudf.api.types.is_numeric_dtype(dtype) for dtype in dtypes): return _find_common_type_decimal( [ dtype @@ -546,40 +542,28 @@ def find_common_type(dtypes): ) else: return cudf.dtype("O") - if any(isinstance(dtype, cudf.ListDtype) for dtype in dtypes): - if len(dtypes) == 1: - return dtypes.get(0) - else: - # TODO: As list dtypes allow casting - # to identical types, improve this logic of returning a - # common dtype, for example: - # ListDtype(int64) & ListDtype(int32) common - # dtype could be ListDtype(int64). - raise NotImplementedError( - "Finding a common type for `ListDtype` is currently " - "not supported" - ) - if any(isinstance(dtype, cudf.StructDtype) for dtype in dtypes): - if len(dtypes) == 1: - return dtypes.get(0) - else: - raise NotImplementedError( - "Finding a common type for `StructDtype` is currently " - "not supported" - ) + elif any( + isinstance(dtype, (cudf.ListDtype, cudf.StructDtype)) + for dtype in dtypes + ): + # TODO: As list dtypes allow casting + # to identical types, improve this logic of returning a + # common dtype, for example: + # ListDtype(int64) & ListDtype(int32) common + # dtype could be ListDtype(int64). + raise NotImplementedError( + "Finding a common type for `ListDtype` or `StructDtype` is currently " + "not supported" + ) # Corner case 1: # Resort to np.result_type to handle "M" and "m" types separately - dt_dtypes = set( - filter(lambda t: cudf.api.types.is_datetime_dtype(t), dtypes) - ) + dt_dtypes = set(filter(lambda t: t.kind == "M", dtypes)) if len(dt_dtypes) > 0: dtypes = dtypes - dt_dtypes dtypes.add(np.result_type(*dt_dtypes)) - td_dtypes = set( - filter(lambda t: pd.api.types.is_timedelta64_dtype(t), dtypes) - ) + td_dtypes = set(filter(lambda t: t.kind == "m", dtypes)) if len(td_dtypes) > 0: dtypes = dtypes - td_dtypes dtypes.add(np.result_type(*td_dtypes)) From 47a0a87db454cc767ab5f74beb2198a480d6f2c0 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:13:29 -1000 Subject: [PATCH 10/53] Type & reduce cupy usage (#16277) There are some cupy usages that don't seem _strictly_ necessary (generating starting data, array type conversion) in some APIs. IMO we should prefer using CPU data/the existing data structure/Column ops over cupy when possible closes https://github.com/rapidsai/cudf/issues/12133 Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16277 --- python/cudf/cudf/core/_base_index.py | 4 ++- python/cudf/cudf/core/column/column.py | 8 +++--- python/cudf/cudf/core/column/datetime.py | 6 ++-- python/cudf/cudf/core/column/numerical.py | 10 ++----- python/cudf/cudf/core/cut.py | 6 ++-- python/cudf/cudf/core/dataframe.py | 18 +++++++----- python/cudf/cudf/core/frame.py | 6 ++-- python/cudf/cudf/core/groupby/groupby.py | 23 ++++++++------- python/cudf/cudf/core/index.py | 34 ++++++++++++----------- python/cudf/cudf/core/multiindex.py | 13 +++++---- python/cudf/cudf/core/tools/datetimes.py | 9 +++--- python/cudf/cudf/tests/test_datetime.py | 15 ++-------- 12 files changed, 74 insertions(+), 78 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index e160fa697ee..9ba2d161619 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -38,6 +38,8 @@ if TYPE_CHECKING: from collections.abc import Generator + import cupy + from cudf.core.column_accessor import ColumnAccessor @@ -2001,7 +2003,7 @@ def drop_duplicates( self._column_names, ) - def duplicated(self, keep="first"): + def duplicated(self, keep="first") -> cupy.ndarray: """ Indicate duplicate index values. diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index f633d527681..fd3664ecac4 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -721,7 +721,7 @@ def notnull(self) -> ColumnBase: return result def indices_of( - self, value: ScalarLike | Self + self, value: ScalarLike ) -> cudf.core.column.NumericalColumn: """ Find locations of value in the column @@ -735,10 +735,10 @@ def indices_of( ------- Column of indices that match value """ - if not isinstance(value, ColumnBase): - value = as_column([value], dtype=self.dtype) + if not is_scalar(value): + raise ValueError("value must be a scalar") else: - assert len(value) == 1 + value = as_column(value, dtype=self.dtype, length=1) mask = libcudf.search.contains(value, self) return apply_boolean_mask( [as_column(range(0, len(self)), dtype=size_type_dtype)], mask diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 409c44f6eee..004a059af95 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -629,9 +629,9 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: def indices_of( self, value: ScalarLike ) -> cudf.core.column.NumericalColumn: - value = column.as_column( - pd.to_datetime(value), dtype=self.dtype - ).astype("int64") + value = ( + pd.to_datetime(value).to_numpy().astype(self.dtype).astype("int64") + ) return self.astype("int64").indices_of(value) @property diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index b8fa00e9643..7f05a5f91a1 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -5,7 +5,6 @@ import functools from typing import TYPE_CHECKING, Any, Callable, Sequence, cast -import cupy as cp import numpy as np import pandas as pd from typing_extensions import Self @@ -13,7 +12,6 @@ import cudf from cudf import _lib as libcudf from cudf._lib import pylibcudf -from cudf._lib.types import size_type_dtype from cudf.api.types import ( is_bool_dtype, is_float_dtype, @@ -131,12 +129,8 @@ def indices_of(self, value: ScalarLike) -> NumericalColumn: and self.dtype.kind in {"c", "f"} and np.isnan(value) ): - return column.as_column( - cp.argwhere( - cp.isnan(self.data_array_view(mode="read")) - ).flatten(), - dtype=size_type_dtype, - ) + nan_col = libcudf.unary.is_nan(self) + return nan_col.indices_of(True) else: return super().indices_of(value) diff --git a/python/cudf/cudf/core/cut.py b/python/cudf/cudf/core/cut.py index d9f62f51f92..197f46ee9fe 100644 --- a/python/cudf/cudf/core/cut.py +++ b/python/cudf/cudf/core/cut.py @@ -188,9 +188,6 @@ def cut( # adjust bin edges decimal precision int_label_bins = np.around(bins, precision) - # the inputs is a column of the values in the array x - input_arr = as_column(x) - # checking for the correct inclusivity values if right: closed = "right" @@ -242,6 +239,9 @@ def cut( labels if len(set(labels)) == len(labels) else None ) + # the inputs is a column of the values in the array x + input_arr = as_column(x) + if isinstance(bins, pd.IntervalIndex): # get the left and right edges of the bins as columns # we cannot typecast an IntervalIndex, so we need to diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 2aa1b95e2d1..2121e623c1c 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -429,7 +429,7 @@ def _setitem_tuple_arg(self, key, value): else: value = cupy.asarray(value) - if cupy.ndim(value) == 2: + if value.ndim == 2: # If the inner dimension is 1, it's broadcastable to # all columns of the dataframe. indexed_shape = columns_df.loc[key[0]].shape @@ -566,7 +566,7 @@ def _setitem_tuple_arg(self, key, value): # TODO: consolidate code path with identical counterpart # in `_DataFrameLocIndexer._setitem_tuple_arg` value = cupy.asarray(value) - if cupy.ndim(value) == 2: + if value.ndim == 2: indexed_shape = columns_df.iloc[key[0]].shape if value.shape[1] == 1: if value.shape[0] != indexed_shape[0]: @@ -2199,8 +2199,8 @@ def from_dict( orient = orient.lower() if orient == "index": - if len(data) > 0 and isinstance( - next(iter(data.values())), (cudf.Series, cupy.ndarray) + if isinstance( + next(iter(data.values()), None), (cudf.Series, cupy.ndarray) ): result = cls(data).T result.columns = ( @@ -5698,7 +5698,13 @@ def from_records(cls, data, index=None, columns=None, nan_as_null=False): @classmethod @_performance_tracking - def _from_arrays(cls, data, index=None, columns=None, nan_as_null=False): + def _from_arrays( + cls, + data: np.ndarray | cupy.ndarray, + index=None, + columns=None, + nan_as_null=False, + ): """Convert a numpy/cupy array to DataFrame. Parameters @@ -5716,8 +5722,6 @@ def _from_arrays(cls, data, index=None, columns=None, nan_as_null=False): ------- DataFrame """ - - data = cupy.asarray(data) if data.ndim != 1 and data.ndim != 2: raise ValueError( f"records dimension expected 1 or 2 but found: {data.ndim}" diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 253d200f7d4..802751e47ad 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1189,7 +1189,7 @@ def searchsorted( side: Literal["left", "right"] = "left", ascending: bool = True, na_position: Literal["first", "last"] = "last", - ): + ) -> ScalarLike | cupy.ndarray: """Find indices where elements should be inserted to maintain order Parameters @@ -1527,7 +1527,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): @acquire_spill_lock() def _apply_cupy_ufunc_to_operands( self, ufunc, cupy_func, operands, **kwargs - ): + ) -> list[dict[Any, ColumnBase]]: # Note: There are some operations that may be supported by libcudf but # are not supported by pandas APIs. In particular, libcudf binary # operations support logical and/or operations as well as @@ -1538,7 +1538,7 @@ def _apply_cupy_ufunc_to_operands( # without cupy. mask = None - data = [{} for _ in range(ufunc.nout)] + data: list[dict[Any, ColumnBase]] = [{} for _ in range(ufunc.nout)] for name, (left, right, _, _) in operands.items(): cupy_inputs = [] for inp in (left, right) if ufunc.nin == 2 else (left,): diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index eccb3acabf6..8659d7c2392 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -35,7 +35,12 @@ from cudf.utils.utils import GetAttrGetItemMixin if TYPE_CHECKING: - from cudf._typing import AggType, DataFrameOrSeries, MultiColumnAggType + from cudf._typing import ( + AggType, + DataFrameOrSeries, + MultiColumnAggType, + ScalarLike, + ) def _deprecate_collect(): @@ -357,7 +362,7 @@ def groups(self): ) @cached_property - def indices(self): + def indices(self) -> dict[ScalarLike, cp.ndarray]: """ Dict {group name -> group indices}. @@ -1015,18 +1020,16 @@ def ngroup(self, ascending=True): if ascending: # Count ascending from 0 to num_groups - 1 - group_ids = cudf.Series._from_data({None: cp.arange(num_groups)}) + groups = range(num_groups) elif has_null_group: # Count descending from num_groups - 1 to 0, but subtract one more # for the null group making it num_groups - 2 to -1. - group_ids = cudf.Series._from_data( - {None: cp.arange(num_groups - 2, -2, -1)} - ) + groups = range(num_groups - 2, -2, -1) else: # Count descending from num_groups - 1 to 0 - group_ids = cudf.Series._from_data( - {None: cp.arange(num_groups - 1, -1, -1)} - ) + groups = range(num_groups - 1, -1, -1) + + group_ids = cudf.Series._from_data({None: as_column(groups)}) if has_null_group: group_ids.iloc[-1] = cudf.NA @@ -1713,7 +1716,7 @@ def rolling_avg(val, avg): return grouped_values.apply_chunks(function, **kwargs) @_performance_tracking - def _broadcast(self, values): + def _broadcast(self, values: cudf.Series) -> cudf.Series: """ Broadcast the results of an aggregation to the group diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index b398ee2343e..4164f981fca 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -103,7 +103,7 @@ def __subclasscheck__(self, subclass): def _lexsorted_equal_range( idx: Index | cudf.MultiIndex, - key_as_table: Frame, + keys: list[ColumnBase], is_sorted: bool, ) -> tuple[int, int, ColumnBase | None]: """Get equal range for key in lexicographically sorted index. If index @@ -118,13 +118,13 @@ def _lexsorted_equal_range( sort_vals = idx lower_bound = search_sorted( [*sort_vals._data.columns], - [*key_as_table._columns], + keys, side="left", ascending=sort_vals.is_monotonic_increasing, ).element_indexing(0) upper_bound = search_sorted( [*sort_vals._data.columns], - [*key_as_table._columns], + keys, side="right", ascending=sort_vals.is_monotonic_increasing, ).element_indexing(0) @@ -260,7 +260,9 @@ def searchsorted( ), "Invalid ascending flag" return search_range(value, self._range, side=side) - def factorize(self, sort: bool = False, use_na_sentinel: bool = True): + def factorize( + self, sort: bool = False, use_na_sentinel: bool = True + ) -> tuple[cupy.ndarray, Self]: if sort and self.step < 0: codes = cupy.arange(len(self) - 1, -1, -1) uniques = self[::-1] @@ -753,15 +755,16 @@ def difference(self, other, sort=None): super().difference(other, sort=sort) ) - def _try_reconstruct_range_index(self, index): - if isinstance(index, RangeIndex) or index.dtype.kind == "f": + def _try_reconstruct_range_index( + self, index: BaseIndex + ) -> Self | BaseIndex: + if isinstance(index, RangeIndex) or index.dtype.kind not in "iu": return index # Evenly spaced values can return a # RangeIndex instead of a materialized Index. - if not index._column.has_nulls(): + if not index._column.has_nulls(): # type: ignore[attr-defined] uniques = cupy.unique(cupy.diff(index.values)) - if len(uniques) == 1 and uniques[0].get() != 0: - diff = uniques[0].get() + if len(uniques) == 1 and (diff := uniques[0].get()) != 0: new_range = range(index[0], index[-1] + diff, diff) return type(self)(new_range, name=index.name) return index @@ -1309,7 +1312,7 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None): return _return_get_indexer_result(result_series.to_cupy()) @_performance_tracking - def get_loc(self, key): + def get_loc(self, key) -> int | slice | cupy.ndarray: if not is_scalar(key): raise TypeError("Should be a scalar-like") @@ -1317,9 +1320,8 @@ def get_loc(self, key): self.is_monotonic_increasing or self.is_monotonic_decreasing ) - target_as_table = cudf.core.frame.Frame({"None": as_column([key])}) lower_bound, upper_bound, sort_inds = _lexsorted_equal_range( - self, target_as_table, is_sorted + self, [as_column([key])], is_sorted ) if lower_bound == upper_bound: @@ -1330,7 +1332,7 @@ def get_loc(self, key): return ( lower_bound if is_sorted - else sort_inds.element_indexing(lower_bound) + else sort_inds.element_indexing(lower_bound) # type: ignore[union-attr] ) if is_sorted: @@ -1339,8 +1341,8 @@ def get_loc(self, key): return slice(lower_bound, upper_bound) # Not sorted and not unique. Return a boolean mask - mask = cupy.full(self._data.nrows, False) - true_inds = sort_inds.slice(lower_bound, upper_bound).values + mask = cupy.full(len(self), False) + true_inds = sort_inds.slice(lower_bound, upper_bound).values # type: ignore[union-attr] mask[true_inds] = True return mask @@ -2076,7 +2078,7 @@ def day_of_year(self): @property # type: ignore @_performance_tracking - def is_leap_year(self): + def is_leap_year(self) -> cupy.ndarray: """ Boolean indicator if the date belongs to a leap year. diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 6503dae6ff5..3ed72ff812a 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1926,17 +1926,18 @@ def get_loc(self, key): # Handle partial key search. If length of `key` is less than `nlevels`, # Only search levels up to `len(key)` level. - key_as_table = cudf.core.frame.Frame( - {i: column.as_column(k, length=1) for i, k in enumerate(key)} - ) partial_index = self.__class__._from_data( - data=self._data.select_by_index(slice(key_as_table._num_columns)) + data=self._data.select_by_index(slice(len(key))) ) ( lower_bound, upper_bound, sort_inds, - ) = _lexsorted_equal_range(partial_index, key_as_table, is_sorted) + ) = _lexsorted_equal_range( + partial_index, + [column.as_column(k, length=1) for k in key], + is_sorted, + ) if lower_bound == upper_bound: raise KeyError(key) @@ -1961,7 +1962,7 @@ def get_loc(self, key): return true_inds # Not sorted and not unique. Return a boolean mask - mask = cp.full(self._data.nrows, False) + mask = cp.full(len(self), False) mask[true_inds] = True return mask diff --git a/python/cudf/cudf/core/tools/datetimes.py b/python/cudf/cudf/core/tools/datetimes.py index 064e8fc667d..c6e2b5d10e1 100644 --- a/python/cudf/cudf/core/tools/datetimes.py +++ b/python/cudf/cudf/core/tools/datetimes.py @@ -6,7 +6,6 @@ import warnings from typing import Literal, Sequence -import cupy as cp import numpy as np import pandas as pd import pandas.tseries.offsets as pd_offset @@ -894,7 +893,7 @@ def date_range( # integers and divide the number range evenly with `periods` elements. start = cudf.Scalar(start, dtype=dtype).value.astype("int64") end = cudf.Scalar(end, dtype=dtype).value.astype("int64") - arr = cp.linspace(start=start, stop=end, num=periods) + arr = np.linspace(start=start, stop=end, num=periods) result = cudf.core.column.as_column(arr).astype("datetime64[ns]") return cudf.DatetimeIndex._from_data({name: result}).tz_localize(tz) @@ -991,8 +990,10 @@ def date_range( stop = end_estim.astype("int64") start = start.value.astype("int64") step = _offset_to_nanoseconds_lower_bound(offset) - arr = cp.arange(start=start, stop=stop, step=step, dtype="int64") - res = cudf.core.column.as_column(arr).astype("datetime64[ns]") + arr = range(int(start), int(stop), step) + res = cudf.core.column.as_column(arr, dtype="int64").astype( + "datetime64[ns]" + ) return cudf.DatetimeIndex._from_data({name: res}, freq=freq).tz_localize( tz diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 092e9790c63..7ab9ff2ef23 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -1534,18 +1534,7 @@ def test_date_range_start_end_periods(start, end, periods): ) -def test_date_range_start_end_freq(request, start, end, freq): - request.applymarker( - pytest.mark.xfail( - condition=( - start == "1831-05-08 15:23:21" - and end == "1996-11-21 04:05:30" - and freq == "110546789ms" - ), - reason="https://github.com/rapidsai/cudf/issues/12133", - ) - ) - +def test_date_range_start_end_freq(start, end, freq): if isinstance(freq, str): _gfreq = _pfreq = freq else: @@ -1561,7 +1550,7 @@ def test_date_range_start_end_freq(request, start, end, freq): ) -def test_date_range_start_freq_periods(request, start, freq, periods): +def test_date_range_start_freq_periods(start, freq, periods): if isinstance(freq, str): _gfreq = _pfreq = freq else: From beda22ed28030bbed2faaa5a49509255f11976aa Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:29:05 -1000 Subject: [PATCH 11/53] Replace is_bool_type with checking .dtype.kind (#16255) It appears this was called when we already had a dtype object so can instead just simply check the .kind attribute Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16255 --- python/cudf/cudf/core/_base_index.py | 9 +++------ python/cudf/cudf/core/_internals/where.py | 8 ++------ python/cudf/cudf/core/column/column.py | 7 +++---- python/cudf/cudf/core/column/numerical.py | 5 ++--- python/cudf/cudf/core/dataframe.py | 13 ++++++------- python/cudf/cudf/core/groupby/groupby.py | 4 ++-- python/cudf/cudf/core/indexing_utils.py | 3 +-- python/cudf/cudf/core/multiindex.py | 4 ---- python/cudf/cudf/core/series.py | 11 +++++------ python/cudf/cudf/core/single_column_frame.py | 3 +-- python/cudf/cudf/tests/test_dataframe.py | 2 +- python/cudf/cudf/tests/test_index.py | 5 ++--- 12 files changed, 28 insertions(+), 46 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 9ba2d161619..479f87bb78b 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -20,7 +20,6 @@ from cudf._lib.types import size_type_dtype from cudf.api.extensions import no_default from cudf.api.types import ( - is_bool_dtype, is_integer, is_integer_dtype, is_list_like, @@ -610,10 +609,8 @@ def union(self, other, sort=None): ) if cudf.get_option("mode.pandas_compatible"): - if ( - is_bool_dtype(self.dtype) and not is_bool_dtype(other.dtype) - ) or ( - not is_bool_dtype(self.dtype) and is_bool_dtype(other.dtype) + if (self.dtype.kind == "b" and other.dtype.kind != "b") or ( + self.dtype.kind != "b" and other.dtype.kind == "b" ): # Bools + other types will result in mixed type. # This is not yet consistent in pandas and specific to APIs. @@ -2154,7 +2151,7 @@ def _apply_boolean_mask(self, boolean_mask): Rows corresponding to `False` is dropped. """ boolean_mask = cudf.core.column.as_column(boolean_mask) - if not is_bool_dtype(boolean_mask.dtype): + if boolean_mask.dtype.kind != "b": raise ValueError("boolean_mask is not boolean type.") return self._from_columns_like_self( diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index f3183e6029d..4a36be76b6d 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -7,11 +7,7 @@ import numpy as np import cudf -from cudf.api.types import ( - _is_non_decimal_numeric_dtype, - is_bool_dtype, - is_scalar, -) +from cudf.api.types import _is_non_decimal_numeric_dtype, is_scalar from cudf.core.dtypes import CategoricalDtype from cudf.utils.dtypes import ( _can_cast, @@ -112,7 +108,7 @@ def _check_and_cast_columns_with_other( other = cudf.Scalar(other) if is_mixed_with_object_dtype(other, source_col) or ( - is_bool_dtype(source_dtype) and not is_bool_dtype(common_dtype) + source_dtype.kind == "b" and common_dtype.kind != "b" ): raise TypeError(mixed_err) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index fd3664ecac4..dbdf501e022 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -41,7 +41,6 @@ _is_non_decimal_numeric_dtype, _is_pandas_nullable_extension_dtype, infer_dtype, - is_bool_dtype, is_dtype_equal, is_scalar, is_string_dtype, @@ -619,7 +618,7 @@ def _scatter_by_column( key: cudf.core.column.NumericalColumn, value: cudf.core.scalar.Scalar | ColumnBase, ) -> Self: - if is_bool_dtype(key.dtype): + if key.dtype.kind == "b": # `key` is boolean mask if len(key) != len(self): raise ValueError( @@ -644,7 +643,7 @@ def _scatter_by_column( self._check_scatter_key_length(num_keys, value) - if is_bool_dtype(key.dtype): + if key.dtype.kind == "b": return libcudf.copying.boolean_mask_scatter([value], [self], key)[ 0 ]._with_type_metadata(self.dtype) @@ -1083,7 +1082,7 @@ def as_decimal_column( def apply_boolean_mask(self, mask) -> ColumnBase: mask = as_column(mask) - if not is_bool_dtype(mask.dtype): + if mask.dtype.kind != "b": raise ValueError("boolean_mask is not boolean type.") return apply_boolean_mask([self], mask)[0]._with_type_metadata( diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 7f05a5f91a1..cea68c88c90 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -13,7 +13,6 @@ from cudf import _lib as libcudf from cudf._lib import pylibcudf from cudf.api.types import ( - is_bool_dtype, is_float_dtype, is_integer, is_integer_dtype, @@ -159,7 +158,7 @@ def __setitem__(self, key: Any, value: Any): else as_column(value) ) - if not is_bool_dtype(self.dtype) and is_bool_dtype(device_value.dtype): + if self.dtype.kind != "b" and device_value.dtype.kind == "b": raise TypeError(f"Invalid value {value} for dtype {self.dtype}") else: device_value = device_value.astype(self.dtype) @@ -264,7 +263,7 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: f"{self.dtype.type.__name__} and " f"{other.dtype.type.__name__}" ) - if is_bool_dtype(self.dtype) or is_bool_dtype(other.dtype): + if self.dtype.kind == "b" or other.dtype.kind == "b": out_dtype = "bool" if ( diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 2121e623c1c..b3d938829c9 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -32,7 +32,6 @@ from cudf.api.extensions import no_default from cudf.api.types import ( _is_scalar_or_zero_d_array, - is_bool_dtype, is_dict_like, is_dtype_equal, is_list_like, @@ -171,7 +170,7 @@ def _can_downcast_to_series(self, df, arg): ): return False else: - if is_bool_dtype(as_column(arg[0]).dtype) and not isinstance( + if as_column(arg[0]).dtype.kind == "b" and not isinstance( arg[1], slice ): return True @@ -320,7 +319,7 @@ def _getitem_tuple_arg(self, arg): tmp_arg[1], ) - if is_bool_dtype(tmp_arg[0].dtype): + if tmp_arg[0].dtype.kind == "b": df = columns_df._apply_boolean_mask( BooleanMask(tmp_arg[0], len(columns_df)) ) @@ -3678,8 +3677,8 @@ def agg(self, aggs, axis=None): """ dtypes = [self[col].dtype for col in self._column_names] common_dtype = find_common_type(dtypes) - if not is_bool_dtype(common_dtype) and any( - is_bool_dtype(dtype) for dtype in dtypes + if common_dtype.kind != "b" and any( + dtype.kind == "b" for dtype in dtypes ): raise MixedTypeError("Cannot create a column with mixed types") @@ -6305,8 +6304,8 @@ def _reduce( and any( not is_object_dtype(dtype) for dtype in source_dtypes ) - or not is_bool_dtype(common_dtype) - and any(is_bool_dtype(dtype) for dtype in source_dtypes) + or common_dtype.kind != "b" + and any(dtype.kind == "b" for dtype in source_dtypes) ): raise TypeError( "Columns must all have the same dtype to " diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 8659d7c2392..d2c75715be2 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -22,7 +22,7 @@ from cudf._lib.sort import segmented_sort_by_key from cudf._lib.types import size_type_dtype from cudf.api.extensions import no_default -from cudf.api.types import is_bool_dtype, is_list_like, is_numeric_dtype +from cudf.api.types import is_list_like, is_numeric_dtype from cudf.core._compat import PANDAS_LT_300 from cudf.core.abc import Serializable from cudf.core.column.column import ColumnBase, StructDtype, as_column @@ -1534,7 +1534,7 @@ def mult(df): # For `sum` & `product`, boolean types # will need to result in `int64` type. for name, col in res._data.items(): - if is_bool_dtype(col.dtype): + if col.dtype.kind == "b": res._data[name] = col.astype("int") return res diff --git a/python/cudf/cudf/core/indexing_utils.py b/python/cudf/cudf/core/indexing_utils.py index a5fed02cbed..9c81b0eb607 100644 --- a/python/cudf/cudf/core/indexing_utils.py +++ b/python/cudf/cudf/core/indexing_utils.py @@ -10,7 +10,6 @@ import cudf from cudf.api.types import ( _is_scalar_or_zero_d_array, - is_bool_dtype, is_integer, is_integer_dtype, ) @@ -230,7 +229,7 @@ def parse_row_iloc_indexer(key: Any, n: int) -> IndexingSpec: key = cudf.core.column.as_column(key) if isinstance(key, cudf.core.column.CategoricalColumn): key = key.astype(key.codes.dtype) - if is_bool_dtype(key.dtype): + if key.dtype.kind == "b": return MaskIndexer(BooleanMask(key, n)) elif len(key) == 0: return EmptyIndexer() diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 3ed72ff812a..ff4b06c6334 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -841,10 +841,6 @@ def _get_row_major( | tuple[Any, ...] | list[tuple[Any, ...]], ) -> DataFrameOrSeries: - if pd.api.types.is_bool_dtype( - list(row_tuple) if isinstance(row_tuple, tuple) else row_tuple - ): - return df[row_tuple] if isinstance(row_tuple, slice): if row_tuple.start is None: row_tuple = slice(self[0], row_tuple.stop, row_tuple.step) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 8c8fa75918c..e12cc3d52fb 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -22,7 +22,6 @@ from cudf.api.types import ( _is_non_decimal_numeric_dtype, _is_scalar_or_zero_d_array, - is_bool_dtype, is_dict_like, is_integer, is_integer_dtype, @@ -221,10 +220,10 @@ def __setitem__(self, key, value): f"Cannot assign {value=} to " f"non-float dtype={self._frame.dtype}" ) - elif ( - self._frame.dtype.kind == "b" - and not is_bool_dtype(value) - and value not in {None, cudf.NA} + elif self._frame.dtype.kind == "b" and not ( + value in {None, cudf.NA} + or isinstance(value, (np.bool_, bool)) + or (isinstance(value, cudf.Scalar) and value.dtype.kind == "b") ): raise MixedTypeError( f"Cannot assign {value=} to " @@ -3221,7 +3220,7 @@ def describe( percentiles = np.array([0.25, 0.5, 0.75]) dtype = "str" - if is_bool_dtype(self.dtype): + if self.dtype.kind == "b": data = _describe_categorical(self, percentiles) elif isinstance(self._column, cudf.core.column.NumericalColumn): data = _describe_numeric(self, percentiles) diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index f9555aee6a2..04c7db7a53c 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -11,7 +11,6 @@ from cudf.api.extensions import no_default from cudf.api.types import ( _is_scalar_or_zero_d_array, - is_bool_dtype, is_integer, is_integer_dtype, is_numeric_dtype, @@ -361,7 +360,7 @@ def _get_elements_from_column(self, arg) -> ScalarLike | ColumnBase: arg = cudf.core.column.column_empty(0, dtype="int32") if is_integer_dtype(arg.dtype): return self._column.take(arg) - if is_bool_dtype(arg.dtype): + if arg.dtype.kind == "b": if (bn := len(arg)) != (n := len(self)): raise IndexError( f"Boolean mask has wrong length: {bn} not {n}" diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 7ccf83e424c..2009fc49ce5 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -5234,7 +5234,7 @@ def test_rowwise_ops(data, op, skipna, numeric_only): else (pdf[column].notna().count() == 0) ) or cudf.api.types.is_numeric_dtype(pdf[column].dtype) - or cudf.api.types.is_bool_dtype(pdf[column].dtype) + or pdf[column].dtype.kind == "b" for column in pdf ): with pytest.raises(TypeError): diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 05dcd85df6a..9eba6122d26 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -16,7 +16,6 @@ import cudf from cudf.api.extensions import no_default -from cudf.api.types import is_bool_dtype from cudf.core.index import CategoricalIndex, DatetimeIndex, Index, RangeIndex from cudf.testing import assert_eq from cudf.testing._utils import ( @@ -2397,8 +2396,8 @@ def test_intersection_index(idx1, idx2, sort, pandas_compatible): expected, actual, exact=False - if (is_bool_dtype(idx1.dtype) and not is_bool_dtype(idx2.dtype)) - or (not is_bool_dtype(idx1.dtype) or is_bool_dtype(idx2.dtype)) + if (idx1.dtype.kind == "b" and idx2.dtype.kind != "b") + or (idx1.dtype.kind != "b" or idx2.dtype.kind == "b") else True, ) From 669db3ea4a0c24a343c5619dd00904ad22ea215b Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Tue, 16 Jul 2024 14:24:58 +0100 Subject: [PATCH 12/53] Fix logic in to_arrow for empty list column (#16279) An empty list column need not have empty children, it just needs to have zero length. In this case, the offsets array will have zero length, and we need to create a temporary buffer. Now that this branch runs, fix two errors in the construction of the arrow array: 1. The element type, if there are children, should be taken from the child array; 2. If the child arrays are empty, we must make an empty null array, rather than passing a null pointer as the values array, otherwise we hit a segfault inside arrow. The previous fix in #16201 correctly handled the empty children case (except for point two), but not the first case, which we do here. Since we we're previously going down this code path (child_arrays was never empty), we never hit the latent segfault from point two. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - David Wendt (https://github.com/davidwendt) - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cudf/pull/16279 --- cpp/src/interop/to_arrow.cu | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cpp/src/interop/to_arrow.cu b/cpp/src/interop/to_arrow.cu index 8c4be1b50a5..622a3aba4bb 100644 --- a/cpp/src/interop/to_arrow.cu +++ b/cpp/src/interop/to_arrow.cu @@ -378,13 +378,11 @@ std::shared_ptr dispatch_to_arrow::operator()( auto children_meta = metadata.children_meta.empty() ? std::vector{{}, {}} : metadata.children_meta; auto child_arrays = fetch_child_array(input_view, children_meta, ar_mr, stream); - if (child_arrays.empty()) { - // Empty list will have only one value in offset of 4 bytes - auto tmp_offset_buffer = allocate_arrow_buffer(sizeof(int32_t), ar_mr); - memset(tmp_offset_buffer->mutable_data(), 0, sizeof(int32_t)); - - return std::make_shared( - arrow::list(arrow::null()), 0, std::move(tmp_offset_buffer), nullptr); + if (child_arrays.empty() || child_arrays[0]->data()->length == 0) { + auto element_type = child_arrays.empty() ? arrow::null() : child_arrays[1]->type(); + auto result = arrow::MakeEmptyArray(arrow::list(element_type), ar_mr); + CUDF_EXPECTS(result.ok(), "Failed to construct empty arrow list array\n"); + return result.ValueUnsafe(); } auto offset_buffer = child_arrays[0]->data()->buffers[1]; From a6de6cc23702ed71b80625f461a90e910a33642f Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Tue, 16 Jul 2024 15:42:12 +0100 Subject: [PATCH 13/53] Introduce version file so we can conditionally handle things in tests (#16280) We decided we would attempt to support a range of versions back to 1.0. We'll test with oldest and newest versions we support. To facilitate, introduce some versioning constants. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - Thomas Li (https://github.com/lithomas1) URL: https://github.com/rapidsai/cudf/pull/16280 --- python/cudf_polars/cudf_polars/dsl/ir.py | 7 ++++- .../cudf_polars/cudf_polars/utils/versions.py | 28 +++++++++++++++++++ python/cudf_polars/tests/test_scan.py | 5 ++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 python/cudf_polars/cudf_polars/utils/versions.py diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 5e6544ef77c..cce0c4a3d94 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -313,7 +313,12 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: raise NotImplementedError( f"Unhandled scan type: {self.typ}" ) # pragma: no cover; post init trips first - if row_index is not None: + if ( + row_index is not None + # TODO: remove condition when dropping support for polars 1.0 + # https://github.com/pola-rs/polars/pull/17363 + and row_index[0] in self.schema + ): name, offset = row_index dtype = self.schema[name] step = plc.interop.from_arrow( diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py new file mode 100644 index 00000000000..a9ac14c25aa --- /dev/null +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +"""Version utilities so that cudf_polars supports a range of polars versions.""" + +# ruff: noqa: SIM300 +from __future__ import annotations + +from packaging.version import parse + +from polars import __version__ + +POLARS_VERSION = parse(__version__) + +POLARS_VERSION_GE_10 = POLARS_VERSION >= parse("1.0") +POLARS_VERSION_GE_11 = POLARS_VERSION >= parse("1.1") +POLARS_VERSION_GE_12 = POLARS_VERSION >= parse("1.2") +POLARS_VERSION_GT_10 = POLARS_VERSION > parse("1.0") +POLARS_VERSION_GT_11 = POLARS_VERSION > parse("1.1") +POLARS_VERSION_GT_12 = POLARS_VERSION > parse("1.2") + +POLARS_VERSION_LE_12 = POLARS_VERSION <= parse("1.2") +POLARS_VERSION_LE_11 = POLARS_VERSION <= parse("1.1") +POLARS_VERSION_LT_12 = POLARS_VERSION < parse("1.2") +POLARS_VERSION_LT_11 = POLARS_VERSION < parse("1.1") + +if POLARS_VERSION < parse("1.0"): # pragma: no cover + raise ImportError("cudf_polars requires py-polars v1.0 or greater.") diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index c41a94da14b..d0c41090433 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -10,6 +10,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils import versions @pytest.fixture( @@ -97,6 +98,10 @@ def test_scan_unsupported_raises(tmp_path): assert_ir_translation_raises(q, NotImplementedError) +@pytest.mark.xfail( + versions.POLARS_VERSION_LT_11, + reason="https://github.com/pola-rs/polars/issues/15730", +) def test_scan_row_index_projected_out(tmp_path): df = pl.DataFrame({"a": [1, 2, 3]}) From 3418f915d1a1ff82a72918d978924dfad2645a5a Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 16 Jul 2024 12:38:47 -0500 Subject: [PATCH 14/53] Introduce dedicated options for low memory readers (#16289) This PR disables low memory readers by default in `cudf.pandas` and instead gives a provision to enable them with dedicated options. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16289 --- python/cudf/cudf/_lib/json.pyx | 2 +- python/cudf/cudf/io/parquet.py | 2 +- python/cudf/cudf/options.py | 26 ++++++++++++++++++++++++++ python/cudf/cudf/tests/test_json.py | 2 +- python/cudf/cudf/tests/test_parquet.py | 2 +- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/_lib/json.pyx b/python/cudf/cudf/_lib/json.pyx index 853dd431099..03bf9ed8b75 100644 --- a/python/cudf/cudf/_lib/json.pyx +++ b/python/cudf/cudf/_lib/json.pyx @@ -99,7 +99,7 @@ cpdef read_json(object filepaths_or_buffers, else: raise TypeError("`dtype` must be 'list like' or 'dict'") - if cudf.get_option("mode.pandas_compatible") and lines: + if cudf.get_option("io.json.low_memory") and lines: res_cols, res_col_names, res_child_names = plc.io.json.chunked_read_json( plc.io.SourceInfo(filepaths_or_buffers), processed_dtypes, diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index fd0792b5edb..02b26ea1c01 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -916,7 +916,7 @@ def _read_parquet( "cudf engine doesn't support the " f"following positional arguments: {list(args)}" ) - if cudf.get_option("mode.pandas_compatible"): + if cudf.get_option("io.parquet.low_memory"): return libparquet.ParquetReader( filepaths_or_buffers, columns=columns, diff --git a/python/cudf/cudf/options.py b/python/cudf/cudf/options.py index 1f539e7f266..94e73021cec 100644 --- a/python/cudf/cudf/options.py +++ b/python/cudf/cudf/options.py @@ -325,6 +325,32 @@ def _integer_and_none_validator(val): _make_contains_validator([False, True]), ) +_register_option( + "io.parquet.low_memory", + False, + textwrap.dedent( + """ + If set to `False`, reads entire parquet in one go. + If set to `True`, reads parquet file in chunks. + \tValid values are True or False. Default is False. + """ + ), + _make_contains_validator([False, True]), +) + +_register_option( + "io.json.low_memory", + False, + textwrap.dedent( + """ + If set to `False`, reads entire json in one go. + If set to `True`, reads json file in chunks. + \tValid values are True or False. Default is False. + """ + ), + _make_contains_validator([False, True]), +) + class option_context(ContextDecorator): """ diff --git a/python/cudf/cudf/tests/test_json.py b/python/cudf/cudf/tests/test_json.py index 7771afd692f..c81c2d1d94b 100644 --- a/python/cudf/cudf/tests/test_json.py +++ b/python/cudf/cudf/tests/test_json.py @@ -1441,6 +1441,6 @@ def test_chunked_json_reader(): df.to_json(buf, lines=True, orient="records", engine="cudf") buf.seek(0) df = df.to_pandas() - with cudf.option_context("mode.pandas_compatible", True): + with cudf.option_context("io.json.low_memory", True): gdf = cudf.read_json(buf, lines=True) assert_eq(df, gdf) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index ff0c9040737..ecb7fd44422 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -3772,6 +3772,6 @@ def test_parquet_reader_pandas_compatibility(): ) buffer = BytesIO() df.to_parquet(buffer) - with cudf.option_context("mode.pandas_compatible", True): + with cudf.option_context("io.parquet.low_memory", True): expected = cudf.read_parquet(buffer) assert_eq(expected, df) From e2b7e4370c8513811e9c72b30f499a5614b49f7c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 16 Jul 2024 14:20:00 -0400 Subject: [PATCH 15/53] Build and test with CUDA 12.5.1 (#16259) This PR updates the latest CUDA build/test version 12.2.2 to 12.5.1. Contributes to https://github.com/rapidsai/build-planning/issues/73 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) - https://github.com/jakirkham URL: https://github.com/rapidsai/cudf/pull/16259 --- .../cuda12.2-conda/devcontainer.json | 8 ++-- .devcontainer/cuda12.2-pip/devcontainer.json | 10 ++-- .github/workflows/build.yaml | 20 ++++---- .github/workflows/pandas-tests.yaml | 4 +- .github/workflows/pr.yaml | 48 +++++++++---------- .../workflows/pr_issue_status_automation.yml | 6 +-- .github/workflows/test.yaml | 22 ++++----- CONTRIBUTING.md | 2 +- README.md | 2 +- ..._64.yaml => all_cuda-125_arch-x86_64.yaml} | 4 +- dependencies.yaml | 6 ++- 11 files changed, 68 insertions(+), 64 deletions(-) rename conda/environments/{all_cuda-122_arch-x86_64.yaml => all_cuda-125_arch-x86_64.yaml} (97%) diff --git a/.devcontainer/cuda12.2-conda/devcontainer.json b/.devcontainer/cuda12.2-conda/devcontainer.json index 05bf9173d25..fadce01d060 100644 --- a/.devcontainer/cuda12.2-conda/devcontainer.json +++ b/.devcontainer/cuda12.2-conda/devcontainer.json @@ -3,7 +3,7 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "12.2", + "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "conda", "BASE": "rapidsai/devcontainers:24.08-cpp-mambaforge-ubuntu22.04" } @@ -11,7 +11,7 @@ "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.2-conda" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-conda" ], "hostRequirements": {"gpu": "optional"}, "features": { @@ -20,7 +20,7 @@ "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.2-envs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.5-envs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent", @@ -29,7 +29,7 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.5-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.2-pip/devcontainer.json b/.devcontainer/cuda12.2-pip/devcontainer.json index 74420214726..026eb540952 100644 --- a/.devcontainer/cuda12.2-pip/devcontainer.json +++ b/.devcontainer/cuda12.2-pip/devcontainer.json @@ -3,15 +3,15 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "12.2", + "CUDA": "12.5", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.2-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.08-cpp-cuda12.5-ubuntu22.04" } }, "runArgs": [ "--rm", "--name", - "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.2-pip" + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-24.08-cuda12.5-pip" ], "hostRequirements": {"gpu": "optional"}, "features": { @@ -20,7 +20,7 @@ "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.5-venvs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent", @@ -28,7 +28,7 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.5-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2e5959338b0..937080572ad 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -101,7 +101,7 @@ jobs: wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -111,7 +111,7 @@ jobs: wheel-build-cudf-polars: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -123,7 +123,7 @@ jobs: wheel-publish-cudf-polars: needs: wheel-build-cudf-polars secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pandas-tests.yaml b/.github/workflows/pandas-tests.yaml index a8643923a4d..1516cb09449 100644 --- a/.github/workflows/pandas-tests.yaml +++ b/.github/workflows/pandas-tests.yaml @@ -17,9 +17,9 @@ jobs: pandas-tests: # run the Pandas unit tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and .CUDA_VER == "12.2.2" )) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and (.CUDA_VER | startswith("12.5.")) )) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ceee9074b93..1fe64e7f318 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,41 +34,41 @@ jobs: - pandas-tests - pandas-tests-diff secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-12.5.1 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-12.5.1 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda-12.5.1 with: build_type: pull-request enable_check_symbols: true conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda-12.5.1 with: build_type: pull-request conda-python-cudf-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -76,14 +76,14 @@ jobs: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: pull-request script: "ci/test_python_other.sh" conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -93,7 +93,7 @@ jobs: static-configure: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: pull-request # Use the wheel container so we can skip conda solves and since our @@ -103,7 +103,7 @@ jobs: conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -113,7 +113,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -123,21 +123,21 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: "ci/build_wheel_cudf.sh" wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-cudf-polars: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -146,7 +146,7 @@ jobs: wheel-tests-cudf-polars: needs: wheel-build-cudf-polars secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -157,7 +157,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -166,7 +166,7 @@ jobs: wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -174,10 +174,10 @@ jobs: script: ci/test_wheel_dask_cudf.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@cuda-12.5.1 with: arch: '["amd64"]' - cuda: '["12.2"]' + cuda: '["12.5"]' build_command: | sccache -z; build-all -DBUILD_BENCHMARKS=ON --verbose; @@ -185,7 +185,7 @@ jobs: unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request @@ -194,9 +194,9 @@ jobs: # run the Pandas unit tests using PR branch needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and .CUDA_VER == "12.2.2" )) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and (.CUDA_VER | startswith("12.5.")) )) build_type: pull-request script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. @@ -204,7 +204,7 @@ jobs: pandas-tests-diff: # diff the results of running the Pandas unit tests and publish a job summary needs: pandas-tests - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: node_type: cpu4 build_type: pull-request diff --git a/.github/workflows/pr_issue_status_automation.yml b/.github/workflows/pr_issue_status_automation.yml index 8ca971dc28d..2a8ebd30993 100644 --- a/.github/workflows/pr_issue_status_automation.yml +++ b/.github/workflows/pr_issue_status_automation.yml @@ -23,7 +23,7 @@ on: jobs: get-project-id: - uses: rapidsai/shared-workflows/.github/workflows/project-get-item-id.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-item-id.yaml@cuda-12.5.1 if: github.event.pull_request.state == 'open' secrets: inherit permissions: @@ -34,7 +34,7 @@ jobs: update-status: # This job sets the PR and its linked issues to "In Progress" status - uses: rapidsai/shared-workflows/.github/workflows/project-get-set-single-select-field.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-set-single-select-field.yaml@cuda-12.5.1 if: ${{ github.event.pull_request.state == 'open' && needs.get-project-id.outputs.ITEM_PROJECT_ID != '' }} needs: get-project-id with: @@ -50,7 +50,7 @@ jobs: update-sprint: # This job sets the PR and its linked issues to the current "Weekly Sprint" - uses: rapidsai/shared-workflows/.github/workflows/project-get-set-iteration-field.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/project-get-set-iteration-field.yaml@cuda-12.5.1 if: ${{ github.event.pull_request.state == 'open' && needs.get-project-id.outputs.ITEM_PROJECT_ID != '' }} needs: get-project-id with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 36c9088d93c..73f8d726e77 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -25,7 +25,7 @@ jobs: enable_check_symbols: true conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -33,7 +33,7 @@ jobs: sha: ${{ inputs.sha }} conda-cpp-memcheck-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -45,7 +45,7 @@ jobs: run_script: "ci/test_cpp_memcheck.sh" static-configure: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: pull-request # Use the wheel container so we can skip conda solves and since our @@ -54,7 +54,7 @@ jobs: run_script: "ci/configure_cpp_static.sh" conda-python-cudf-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -64,7 +64,7 @@ jobs: conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -73,7 +73,7 @@ jobs: script: "ci/test_python_other.sh" conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -85,7 +85,7 @@ jobs: run_script: "ci/test_java.sh" conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -97,7 +97,7 @@ jobs: run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -106,7 +106,7 @@ jobs: script: ci/test_wheel_cudf.sh wheel-tests-dask-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -117,7 +117,7 @@ jobs: script: ci/test_wheel_dask_cudf.sh unit-tests-cudf-pandas: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4fbc28fa6e1..f9cdde7c2b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,7 +104,7 @@ Instructions for a minimal build environment without conda are included below. # create the conda environment (assuming in base `cudf` directory) # note: RAPIDS currently doesn't support `channel_priority: strict`; # use `channel_priority: flexible` instead -conda env create --name cudf_dev --file conda/environments/all_cuda-122_arch-x86_64.yaml +conda env create --name cudf_dev --file conda/environments/all_cuda-125_arch-x86_64.yaml # activate the environment conda activate cudf_dev ``` diff --git a/README.md b/README.md index 17d2df9a936..1ab6a2d7457 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ cuDF can be installed with conda (via [miniconda](https://docs.conda.io/projects ```bash conda install -c rapidsai -c conda-forge -c nvidia \ - cudf=24.08 python=3.11 cuda-version=12.2 + cudf=24.08 python=3.11 cuda-version=12.5 ``` We also provide [nightly Conda packages](https://anaconda.org/rapidsai-nightly) built from the HEAD diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml similarity index 97% rename from conda/environments/all_cuda-122_arch-x86_64.yaml rename to conda/environments/all_cuda-125_arch-x86_64.yaml index c32d21c5d36..3f5fae49cbb 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -23,7 +23,7 @@ dependencies: - cuda-nvtx-dev - cuda-python>=12.0,<13.0a0 - cuda-sanitizer-api -- cuda-version=12.2 +- cuda-version=12.5 - cupy>=12.0.0 - cxx-compiler - cython>=3.0.3 @@ -96,4 +96,4 @@ dependencies: - zlib>=1.2.13 - pip: - git+https://github.com/python-streamz/streamz.git@master -name: all_cuda-122_arch-x86_64 +name: all_cuda-125_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 27621ff9a3f..67ed3773b44 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["11.8", "12.2"] + cuda: ["11.8", "12.5"] arch: [x86_64] includes: - build_base @@ -402,6 +402,10 @@ dependencies: cuda: "12.2" packages: - cuda-version=12.2 + - matrix: + cuda: "12.5" + packages: + - cuda-version=12.5 cuda: specific: - output_types: conda From 05ea7c9cf6a0fd39384e2044b4c9b46f543d4ad0 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:51:20 -0700 Subject: [PATCH 16/53] Fix tests for polars 1.2 (#16292) Authors: - Thomas Li (https://github.com/lithomas1) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/16292 --- python/cudf_polars/tests/test_groupby.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index 50adca01950..b07d8e38217 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -12,6 +12,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils import versions @pytest.fixture @@ -100,7 +101,7 @@ def test_groupby_sorted_keys(df: pl.LazyFrame, keys, exprs): with pytest.raises(AssertionError): # https://github.com/pola-rs/polars/issues/17556 assert_gpu_result_equal(q, check_exact=False) - if schema[sort_keys[1]] == pl.Boolean(): + if versions.POLARS_VERSION_LT_12 and schema[sort_keys[1]] == pl.Boolean(): # https://github.com/pola-rs/polars/issues/17557 with pytest.raises(AssertionError): assert_gpu_result_equal(qsorted, check_exact=False) From 62191103032706371d76ce83c6ec59d13376b231 Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:23:49 -0400 Subject: [PATCH 17/53] [BUG] Make name attr of Index fast slow attrs (#16270) Debugging the spike in failures from #16234 Authors: - Matthew Murray (https://github.com/Matt711) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16270 --- python/cudf/cudf/pandas/_wrappers/pandas.py | 36 ++++++++------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index d3a3488081a..59a243dd7c4 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -260,18 +260,14 @@ def Index__new__(cls, *args, **kwargs): return self -def name(self): - return self._fsproxy_wrapped._name - - def Index__setattr__(self, name, value): if name.startswith("_"): object.__setattr__(self, name, value) return if name == "name": - setattr(self._fsproxy_wrapped, "_name", value) + setattr(self._fsproxy_wrapped, "name", value) if name == "names": - setattr(self._fsproxy_wrapped, "_names", value) + setattr(self._fsproxy_wrapped, "names", value) return _FastSlowAttribute("__setattr__").__get__(self, type(self))( name, value ) @@ -300,7 +296,7 @@ def Index__setattr__(self, name, value): "_accessors": set(), "_data": _FastSlowAttribute("_data", private=True), "_mask": _FastSlowAttribute("_mask", private=True), - "name": property(name), + "name": _FastSlowAttribute("name"), }, ) @@ -314,7 +310,7 @@ def Index__setattr__(self, name, value): additional_attributes={ "__init__": _DELETE, "__setattr__": Index__setattr__, - "name": property(name), + "name": _FastSlowAttribute("name"), }, ) @@ -345,7 +341,7 @@ def Index__setattr__(self, name, value): additional_attributes={ "__init__": _DELETE, "__setattr__": Index__setattr__, - "name": property(name), + "name": _FastSlowAttribute("name"), }, ) @@ -375,10 +371,10 @@ def Index__setattr__(self, name, value): bases=(Index,), additional_attributes={ "__init__": _DELETE, + "__setattr__": Index__setattr__, "_data": _FastSlowAttribute("_data", private=True), "_mask": _FastSlowAttribute("_mask", private=True), - "__setattr__": Index__setattr__, - "name": property(name), + "name": _FastSlowAttribute("name"), }, ) @@ -412,10 +408,10 @@ def Index__setattr__(self, name, value): bases=(Index,), additional_attributes={ "__init__": _DELETE, + "__setattr__": Index__setattr__, "_data": _FastSlowAttribute("_data", private=True), "_mask": _FastSlowAttribute("_mask", private=True), - "__setattr__": Index__setattr__, - "name": property(name), + "name": _FastSlowAttribute("name"), }, ) @@ -470,10 +466,10 @@ def Index__setattr__(self, name, value): bases=(Index,), additional_attributes={ "__init__": _DELETE, + "__setattr__": Index__setattr__, "_data": _FastSlowAttribute("_data", private=True), "_mask": _FastSlowAttribute("_mask", private=True), - "__setattr__": Index__setattr__, - "name": property(name), + "name": _FastSlowAttribute("name"), }, ) @@ -508,10 +504,6 @@ def Index__setattr__(self, name, value): ) -def names(self): - return self._fsproxy_wrapped._names - - MultiIndex = make_final_proxy_type( "MultiIndex", cudf.MultiIndex, @@ -522,7 +514,7 @@ def names(self): additional_attributes={ "__init__": _DELETE, "__setattr__": Index__setattr__, - "name": property(names), + "names": _FastSlowAttribute("names"), }, ) @@ -709,10 +701,10 @@ def names(self): bases=(Index,), additional_attributes={ "__init__": _DELETE, + "__setattr__": Index__setattr__, "_data": _FastSlowAttribute("_data", private=True), "_mask": _FastSlowAttribute("_mask", private=True), - "__setattr__": Index__setattr__, - "name": property(name), + "name": _FastSlowAttribute("name"), }, ) From 6a954e299d97f69a62fd184529fa7d5f29c0e09f Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:02:20 -0700 Subject: [PATCH 18/53] Migrate expressions to pylibcudf (#16056) xref #15162 Migrates expresions to use pylibcudf. Authors: - Thomas Li (https://github.com/lithomas1) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16056 --- .../api_docs/pylibcudf/datetime.rst | 6 +- .../api_docs/pylibcudf/expressions.rst | 6 + .../user_guide/api_docs/pylibcudf/index.rst | 1 + python/cudf/cudf/_lib/CMakeLists.txt | 1 - python/cudf/cudf/_lib/__init__.py | 3 +- python/cudf/cudf/_lib/expressions.pyx | 156 -------------- python/cudf/cudf/_lib/parquet.pyx | 2 +- .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 1 + python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 1 + python/cudf/cudf/_lib/pylibcudf/__init__.py | 1 + .../cudf/_lib/{ => pylibcudf}/expressions.pxd | 29 ++- .../cudf/cudf/_lib/pylibcudf/expressions.pyx | 195 ++++++++++++++++++ .../_lib/pylibcudf/libcudf/CMakeLists.txt | 4 +- .../_lib/pylibcudf/libcudf/expressions.pxd | 103 ++++----- .../_lib/pylibcudf/libcudf/expressions.pyx | 0 python/cudf/cudf/_lib/transform.pyx | 2 +- .../cudf/cudf/core/_internals/expressions.py | 11 +- .../cudf/pylibcudf_tests/test_expressions.py | 50 +++++ 18 files changed, 335 insertions(+), 237 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/expressions.rst delete mode 100644 python/cudf/cudf/_lib/expressions.pyx rename python/cudf/cudf/_lib/{ => pylibcudf}/expressions.pxd (50%) create mode 100644 python/cudf/cudf/_lib/pylibcudf/expressions.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pyx create mode 100644 python/cudf/cudf/pylibcudf_tests/test_expressions.py diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/datetime.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/datetime.rst index ebf5fab3052..558268ea495 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/datetime.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/datetime.rst @@ -1,6 +1,6 @@ -======= -copying -======= +======== +datetime +======== .. automodule:: cudf._lib.pylibcudf.datetime :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/expressions.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/expressions.rst new file mode 100644 index 00000000000..03f769ee861 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/expressions.rst @@ -0,0 +1,6 @@ +=========== +expressions +=========== + +.. automodule:: cudf._lib.pylibcudf.expressions + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst index 5899d272160..505765bba0f 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/index.rst @@ -15,6 +15,7 @@ This page provides API documentation for pylibcudf. concatenate copying datetime + expressions filling gpumemoryview groupby diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 5a067e84f56..38b7e9ebe04 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -21,7 +21,6 @@ set(cython_sources copying.pyx csv.pyx datetime.pyx - expressions.pyx filling.pyx groupby.pyx hash.pyx diff --git a/python/cudf/cudf/_lib/__init__.py b/python/cudf/cudf/_lib/__init__.py index 18b95f5f2e1..34c0e29d0b1 100644 --- a/python/cudf/cudf/_lib/__init__.py +++ b/python/cudf/cudf/_lib/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. import numpy as np from . import ( @@ -8,7 +8,6 @@ copying, csv, datetime, - expressions, filling, groupby, hash, diff --git a/python/cudf/cudf/_lib/expressions.pyx b/python/cudf/cudf/_lib/expressions.pyx deleted file mode 100644 index 3fb29279ed7..00000000000 --- a/python/cudf/cudf/_lib/expressions.pyx +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -from enum import Enum - -import numpy as np - -from cython.operator cimport dereference -from libc.stdint cimport int64_t -from libcpp.memory cimport make_unique, unique_ptr -from libcpp.string cimport string -from libcpp.utility cimport move - -from cudf._lib.pylibcudf.libcudf cimport expressions as libcudf_exp -from cudf._lib.pylibcudf.libcudf.types cimport size_type -from cudf._lib.pylibcudf.libcudf.wrappers.timestamps cimport ( - timestamp_ms, - timestamp_us, -) - -# Necessary for proper casting, see below. -ctypedef int32_t underlying_type_ast_operator - - -# Aliases for simplicity -ctypedef unique_ptr[libcudf_exp.expression] expression_ptr - - -class ASTOperator(Enum): - ADD = libcudf_exp.ast_operator.ADD - SUB = libcudf_exp.ast_operator.SUB - MUL = libcudf_exp.ast_operator.MUL - DIV = libcudf_exp.ast_operator.DIV - TRUE_DIV = libcudf_exp.ast_operator.TRUE_DIV - FLOOR_DIV = libcudf_exp.ast_operator.FLOOR_DIV - MOD = libcudf_exp.ast_operator.MOD - PYMOD = libcudf_exp.ast_operator.PYMOD - POW = libcudf_exp.ast_operator.POW - EQUAL = libcudf_exp.ast_operator.EQUAL - NULL_EQUAL = libcudf_exp.ast_operator.NULL_EQUAL - NOT_EQUAL = libcudf_exp.ast_operator.NOT_EQUAL - LESS = libcudf_exp.ast_operator.LESS - GREATER = libcudf_exp.ast_operator.GREATER - LESS_EQUAL = libcudf_exp.ast_operator.LESS_EQUAL - GREATER_EQUAL = libcudf_exp.ast_operator.GREATER_EQUAL - BITWISE_AND = libcudf_exp.ast_operator.BITWISE_AND - BITWISE_OR = libcudf_exp.ast_operator.BITWISE_OR - BITWISE_XOR = libcudf_exp.ast_operator.BITWISE_XOR - LOGICAL_AND = libcudf_exp.ast_operator.LOGICAL_AND - NULL_LOGICAL_AND = libcudf_exp.ast_operator.NULL_LOGICAL_AND - LOGICAL_OR = libcudf_exp.ast_operator.LOGICAL_OR - NULL_LOGICAL_OR = libcudf_exp.ast_operator.NULL_LOGICAL_OR - # Unary operators - IDENTITY = libcudf_exp.ast_operator.IDENTITY - IS_NULL = libcudf_exp.ast_operator.IS_NULL - SIN = libcudf_exp.ast_operator.SIN - COS = libcudf_exp.ast_operator.COS - TAN = libcudf_exp.ast_operator.TAN - ARCSIN = libcudf_exp.ast_operator.ARCSIN - ARCCOS = libcudf_exp.ast_operator.ARCCOS - ARCTAN = libcudf_exp.ast_operator.ARCTAN - SINH = libcudf_exp.ast_operator.SINH - COSH = libcudf_exp.ast_operator.COSH - TANH = libcudf_exp.ast_operator.TANH - ARCSINH = libcudf_exp.ast_operator.ARCSINH - ARCCOSH = libcudf_exp.ast_operator.ARCCOSH - ARCTANH = libcudf_exp.ast_operator.ARCTANH - EXP = libcudf_exp.ast_operator.EXP - LOG = libcudf_exp.ast_operator.LOG - SQRT = libcudf_exp.ast_operator.SQRT - CBRT = libcudf_exp.ast_operator.CBRT - CEIL = libcudf_exp.ast_operator.CEIL - FLOOR = libcudf_exp.ast_operator.FLOOR - ABS = libcudf_exp.ast_operator.ABS - RINT = libcudf_exp.ast_operator.RINT - BIT_INVERT = libcudf_exp.ast_operator.BIT_INVERT - NOT = libcudf_exp.ast_operator.NOT - - -class TableReference(Enum): - LEFT = libcudf_exp.table_reference.LEFT - RIGHT = libcudf_exp.table_reference.RIGHT - - -# Note that this function only currently supports numeric literals. libcudf -# expressions don't really support other types yet though, so this isn't -# restrictive at the moment. -cdef class Literal(Expression): - def __cinit__(self, value): - if isinstance(value, int): - self.c_scalar.reset(new numeric_scalar[int64_t](value, True)) - self.c_obj = move(make_unique[libcudf_exp.literal]( - dereference(self.c_scalar) - )) - elif isinstance(value, float): - self.c_scalar.reset(new numeric_scalar[double](value, True)) - self.c_obj = move(make_unique[libcudf_exp.literal]( - dereference(self.c_scalar) - )) - elif isinstance(value, str): - self.c_scalar.reset(new string_scalar(value.encode(), True)) - self.c_obj = move(make_unique[libcudf_exp.literal]( - dereference(self.c_scalar) - )) - elif isinstance(value, np.datetime64): - scale, _ = np.datetime_data(value.dtype) - int_value = value.astype(np.int64) - if scale == "ms": - self.c_scalar.reset(new timestamp_scalar[timestamp_ms]( - int_value, True) - ) - self.c_obj = move(make_unique[libcudf_exp.literal]( - dereference(self.c_scalar) - )) - elif scale == "us": - self.c_scalar.reset(new timestamp_scalar[timestamp_us]( - int_value, True) - ) - self.c_obj = move(make_unique[libcudf_exp.literal]( - dereference(self.c_scalar) - )) - else: - raise NotImplementedError( - f"Unhandled datetime scale {scale=}" - ) - else: - raise NotImplementedError( - f"Don't know how to make literal with type {type(value)}" - ) - - -cdef class ColumnReference(Expression): - def __cinit__(self, size_type index): - self.c_obj = move(make_unique[libcudf_exp.column_reference]( - index - )) - - -cdef class Operation(Expression): - def __cinit__(self, op, Expression left, Expression right=None): - cdef libcudf_exp.ast_operator op_value = ( - op.value - ) - - if right is None: - self.c_obj = move(make_unique[libcudf_exp.operation]( - op_value, dereference(left.c_obj) - )) - else: - self.c_obj = move(make_unique[libcudf_exp.operation]( - op_value, dereference(left.c_obj), dereference(right.c_obj) - )) - -cdef class ColumnNameReference(Expression): - def __cinit__(self, string name): - self.c_obj = \ - move(make_unique[libcudf_exp.column_name_reference](name)) diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index 158fb6051c3..e7959d21e01 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -37,12 +37,12 @@ cimport cudf._lib.pylibcudf.libcudf.io.data_sink as cudf_io_data_sink cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types cimport cudf._lib.pylibcudf.libcudf.types as cudf_types from cudf._lib.column cimport Column -from cudf._lib.expressions cimport Expression from cudf._lib.io.utils cimport ( make_sinks_info, make_source_info, update_struct_field_names, ) +from cudf._lib.pylibcudf.expressions cimport Expression from cudf._lib.pylibcudf.io.datasource cimport NativeFileDatasource from cudf._lib.pylibcudf.libcudf.expressions cimport expression from cudf._lib.pylibcudf.libcudf.io.parquet cimport ( diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index a2d11bbea6e..0800fa18e94 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -20,6 +20,7 @@ set(cython_sources concatenate.pyx copying.pyx datetime.pyx + expressions.pyx filling.pyx gpumemoryview.pyx groupby.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index da2b7806203..26e89b818d3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -8,6 +8,7 @@ from . cimport ( concatenate, copying, datetime, + expressions, filling, groupby, join, diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index acbc84d7177..e89a5ed9f96 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -7,6 +7,7 @@ concatenate, copying, datetime, + expressions, filling, groupby, interop, diff --git a/python/cudf/cudf/_lib/expressions.pxd b/python/cudf/cudf/_lib/pylibcudf/expressions.pxd similarity index 50% rename from python/cudf/cudf/_lib/expressions.pxd rename to python/cudf/cudf/_lib/pylibcudf/expressions.pxd index 4a20c5fc545..64825b89d9f 100644 --- a/python/cudf/cudf/_lib/expressions.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/expressions.pxd @@ -1,36 +1,31 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -from libc.stdint cimport int32_t, int64_t +# Copyright (c) 2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr +from libcpp.string cimport string from cudf._lib.pylibcudf.libcudf.expressions cimport ( - column_reference, + ast_operator, expression, - literal, - operation, -) -from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport ( - numeric_scalar, - scalar, - string_scalar, - timestamp_scalar, + table_reference, ) +from .scalar cimport Scalar + cdef class Expression: cdef unique_ptr[expression] c_obj - cdef class Literal(Expression): - cdef unique_ptr[scalar] c_scalar - + # Hold on to input scalar so it doesn't get gc'ed + cdef Scalar scalar cdef class ColumnReference(Expression): pass - cdef class Operation(Expression): - pass + # Hold on to the input expressions so + # they don't get gc'ed + cdef Expression right + cdef Expression left cdef class ColumnNameReference(Expression): pass diff --git a/python/cudf/cudf/_lib/pylibcudf/expressions.pyx b/python/cudf/cudf/_lib/pylibcudf/expressions.pyx new file mode 100644 index 00000000000..38de11406ad --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/expressions.pyx @@ -0,0 +1,195 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +from cudf._lib.pylibcudf.libcudf.expressions import \ + ast_operator as ASTOperator # no-cython-lint +from cudf._lib.pylibcudf.libcudf.expressions import \ + table_reference as TableReference # no-cython-lint + +from cython.operator cimport dereference +from libc.stdint cimport int32_t, int64_t +from libcpp.memory cimport make_unique, unique_ptr +from libcpp.string cimport string +from libcpp.utility cimport move + +from cudf._lib.pylibcudf.libcudf cimport expressions as libcudf_exp +from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport ( + duration_scalar, + numeric_scalar, + string_scalar, + timestamp_scalar, +) +from cudf._lib.pylibcudf.libcudf.types cimport size_type, type_id +from cudf._lib.pylibcudf.libcudf.wrappers.durations cimport ( + duration_ms, + duration_ns, + duration_s, + duration_us, +) +from cudf._lib.pylibcudf.libcudf.wrappers.timestamps cimport ( + timestamp_ms, + timestamp_ns, + timestamp_s, + timestamp_us, +) + +from .scalar cimport Scalar +from .traits cimport is_chrono, is_numeric +from .types cimport DataType + +# Aliases for simplicity +ctypedef unique_ptr[libcudf_exp.expression] expression_ptr + +cdef class Literal(Expression): + """ + A literal value used in an abstract syntax tree. + + For details, see :cpp:class:`cudf::ast::literal`. + + Parameters + ---------- + value : Scalar + The Scalar value of the Literal. + Must be either numeric, string, or a timestamp/duration scalar. + """ + def __cinit__(self, Scalar value): + self.scalar = value + cdef DataType typ = value.type() + cdef type_id tid = value.type().id() + if not (is_numeric(typ) or is_chrono(typ) or tid == type_id.STRING): + raise ValueError( + "Only numeric, string, or timestamp/duration scalars are accepted" + ) + # TODO: Accept type-erased scalar in AST C++ code + # Then a lot of this code can be deleted + if tid == type_id.INT64: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.INT32: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.FLOAT64: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.FLOAT32: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.STRING: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.TIMESTAMP_NANOSECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.TIMESTAMP_MICROSECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.TIMESTAMP_MILLISECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.TIMESTAMP_MILLISECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.TIMESTAMP_SECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.DURATION_NANOSECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.DURATION_MICROSECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.DURATION_MILLISECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.DURATION_MILLISECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + elif tid == type_id.DURATION_SECONDS: + self.c_obj = move(make_unique[libcudf_exp.literal]( + dereference(self.scalar.c_obj) + )) + else: + raise NotImplementedError( + f"Don't know how to make literal with type id {tid}" + ) + +cdef class ColumnReference(Expression): + """ + An expression referring to data from a column in a table. + + For details, see :cpp:class:`cudf::ast::column_reference`. + + Parameters + ---------- + index : size_type + The index of this column in the table + (provided when the expression is evaluated). + table_source : TableReference, default TableReferenece.LEFT + Which table to use in cases with two tables (e.g. joins) + """ + def __cinit__( + self, + size_type index, + table_reference table_source=table_reference.LEFT + ): + self.c_obj = move(make_unique[libcudf_exp.column_reference]( + index, table_source + )) + + +cdef class Operation(Expression): + """ + An operation expression holds an operator and zero or more operands. + + For details, see :cpp:class:`cudf::ast::operation`. + + Parameters + ---------- + op : Operator + left : Expression + Left input expression (left operand) + right: Expression, default None + Right input expression (right operand). + You should only pass this if the input expression is a binary operation. + """ + def __cinit__(self, ast_operator op, Expression left, Expression right=None): + self.left = left + self.right = right + if right is None: + self.c_obj = move(make_unique[libcudf_exp.operation]( + op, dereference(left.c_obj) + )) + else: + self.c_obj = move(make_unique[libcudf_exp.operation]( + op, dereference(left.c_obj), dereference(right.c_obj) + )) + +cdef class ColumnNameReference(Expression): + """ + An expression referring to data from a column in a table. + + For details, see :cpp:class:`cudf::ast::column_name_reference`. + + Parameters + ---------- + column_name : str + Name of this column in the table metadata + (provided when the expression is evaluated). + """ + def __cinit__(self, str name): + self.c_obj = \ + move(make_unique[libcudf_exp.column_name_reference]( + (name.encode("utf-8")) + )) diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/libcudf/CMakeLists.txt index 699e85ce567..b04e94f1546 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/CMakeLists.txt @@ -12,8 +12,8 @@ # the License. # ============================================================================= -set(cython_sources aggregation.pyx binaryop.pyx copying.pyx reduce.pyx replace.pyx round.pyx - stream_compaction.pyx types.pyx unary.pyx +set(cython_sources aggregation.pyx binaryop.pyx copying.pyx expressions.pyx reduce.pyx replace.pyx + round.pyx stream_compaction.pyx types.pyx unary.pyx ) set(linked_libraries cudf::cudf) diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pxd index 279d969db50..427e16d4ff8 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pxd @@ -1,5 +1,6 @@ # Copyright (c) 2022-2024, NVIDIA CORPORATION. +from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr from libcpp.string cimport string @@ -14,63 +15,63 @@ from cudf._lib.pylibcudf.libcudf.types cimport size_type cdef extern from "cudf/ast/expressions.hpp" namespace "cudf::ast" nogil: - ctypedef enum ast_operator: + cpdef enum class ast_operator(int32_t): # Binary operators - ADD "cudf::ast::ast_operator::ADD" - SUB "cudf::ast::ast_operator::SUB" - MUL "cudf::ast::ast_operator::MUL" - DIV "cudf::ast::ast_operator::DIV" - TRUE_DIV "cudf::ast::ast_operator::TRUE_DIV" - FLOOR_DIV "cudf::ast::ast_operator::FLOOR_DIV" - MOD "cudf::ast::ast_operator::MOD" - PYMOD "cudf::ast::ast_operator::PYMOD" - POW "cudf::ast::ast_operator::POW" - EQUAL "cudf::ast::ast_operator::EQUAL" - NULL_EQUAL "cudf::ast::ast_operator::NULL_EQUAL" - NOT_EQUAL "cudf::ast::ast_operator::NOT_EQUAL" - LESS "cudf::ast::ast_operator::LESS" - GREATER "cudf::ast::ast_operator::GREATER" - LESS_EQUAL "cudf::ast::ast_operator::LESS_EQUAL" - GREATER_EQUAL "cudf::ast::ast_operator::GREATER_EQUAL" - BITWISE_AND "cudf::ast::ast_operator::BITWISE_AND" - BITWISE_OR "cudf::ast::ast_operator::BITWISE_OR" - BITWISE_XOR "cudf::ast::ast_operator::BITWISE_XOR" - NULL_LOGICAL_AND "cudf::ast::ast_operator::NULL_LOGICAL_AND" - LOGICAL_AND "cudf::ast::ast_operator::LOGICAL_AND" - NULL_LOGICAL_OR "cudf::ast::ast_operator::NULL_LOGICAL_OR" - LOGICAL_OR "cudf::ast::ast_operator::LOGICAL_OR" + ADD + SUB + MUL + DIV + TRUE_DIV + FLOOR_DIV + MOD + PYMOD + POW + EQUAL + NULL_EQUAL + NOT_EQUAL + LESS + GREATER + LESS_EQUAL + GREATER_EQUAL + BITWISE_AND + BITWISE_OR + BITWISE_XOR + NULL_LOGICAL_AND + LOGICAL_AND + NULL_LOGICAL_OR + LOGICAL_OR # Unary operators - IDENTITY "cudf::ast::ast_operator::IDENTITY" - IS_NULL "cudf::ast::ast_operator::IS_NULL" - SIN "cudf::ast::ast_operator::SIN" - COS "cudf::ast::ast_operator::COS" - TAN "cudf::ast::ast_operator::TAN" - ARCSIN "cudf::ast::ast_operator::ARCSIN" - ARCCOS "cudf::ast::ast_operator::ARCCOS" - ARCTAN "cudf::ast::ast_operator::ARCTAN" - SINH "cudf::ast::ast_operator::SINH" - COSH "cudf::ast::ast_operator::COSH" - TANH "cudf::ast::ast_operator::TANH" - ARCSINH "cudf::ast::ast_operator::ARCSINH" - ARCCOSH "cudf::ast::ast_operator::ARCCOSH" - ARCTANH "cudf::ast::ast_operator::ARCTANH" - EXP "cudf::ast::ast_operator::EXP" - LOG "cudf::ast::ast_operator::LOG" - SQRT "cudf::ast::ast_operator::SQRT" - CBRT "cudf::ast::ast_operator::CBRT" - CEIL "cudf::ast::ast_operator::CEIL" - FLOOR "cudf::ast::ast_operator::FLOOR" - ABS "cudf::ast::ast_operator::ABS" - RINT "cudf::ast::ast_operator::RINT" - BIT_INVERT "cudf::ast::ast_operator::BIT_INVERT" - NOT "cudf::ast::ast_operator::NOT" + IDENTITY + IS_NULL + SIN + COS + TAN + ARCSIN + ARCCOS + ARCTAN + SINH + COSH + TANH + ARCSINH + ARCCOSH + ARCTANH + EXP + LOG + SQRT + CBRT + CEIL + FLOOR + ABS + RINT + BIT_INVERT + NOT cdef cppclass expression: pass - ctypedef enum table_reference: - LEFT "cudf::ast::table_reference::LEFT" - RIGHT "cudf::ast::table_reference::RIGHT" + cpdef enum class table_reference(int32_t): + LEFT + RIGHT cdef cppclass literal(expression): # Due to https://github.com/cython/cython/issues/3198, we need to diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pyx b/python/cudf/cudf/_lib/pylibcudf/libcudf/expressions.pyx new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/cudf/cudf/_lib/transform.pyx b/python/cudf/cudf/_lib/transform.pyx index 86a4a60eef1..622725e06a3 100644 --- a/python/cudf/cudf/_lib/transform.pyx +++ b/python/cudf/cudf/_lib/transform.pyx @@ -19,8 +19,8 @@ from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer cimport cudf._lib.pylibcudf.libcudf.transform as libcudf_transform from cudf._lib.column cimport Column -from cudf._lib.expressions cimport Expression from cudf._lib.pylibcudf cimport transform as plc_transform +from cudf._lib.pylibcudf.expressions cimport Expression from cudf._lib.pylibcudf.libcudf.column.column cimport column from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view from cudf._lib.pylibcudf.libcudf.expressions cimport expression diff --git a/python/cudf/cudf/core/_internals/expressions.py b/python/cudf/cudf/core/_internals/expressions.py index 393a68dd844..63714a78572 100644 --- a/python/cudf/cudf/core/_internals/expressions.py +++ b/python/cudf/cudf/core/_internals/expressions.py @@ -4,7 +4,10 @@ import ast import functools -from cudf._lib.expressions import ( +import pyarrow as pa + +import cudf._lib.pylibcudf as plc +from cudf._lib.pylibcudf.expressions import ( ASTOperator, ColumnReference, Expression, @@ -122,7 +125,9 @@ def visit_Constant(self, node): f"Unsupported literal {repr(node.value)} of type " "{type(node.value).__name__}" ) - self.stack.append(Literal(node.value)) + self.stack.append( + Literal(plc.interop.from_arrow(pa.scalar(node.value))) + ) def visit_UnaryOp(self, node): self.visit(node.operand) @@ -132,7 +137,7 @@ def visit_UnaryOp(self, node): # operand, so there's no way to know whether this should be a float # or an int. We should maybe see what Spark does, and this will # probably require casting. - self.nodes.append(Literal(-1)) + self.nodes.append(Literal(plc.interop.from_arrow(pa.scalar(-1)))) op = ASTOperator.MUL self.stack.append(Operation(op, self.nodes[-1], self.nodes[-2])) elif isinstance(node.op, ast.UAdd): diff --git a/python/cudf/cudf/pylibcudf_tests/test_expressions.py b/python/cudf/cudf/pylibcudf_tests/test_expressions.py new file mode 100644 index 00000000000..f661512caad --- /dev/null +++ b/python/cudf/cudf/pylibcudf_tests/test_expressions.py @@ -0,0 +1,50 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +import pyarrow as pa +import pytest + +import cudf._lib.pylibcudf as plc + +# We can't really evaluate these expressions, so just make sure +# construction works properly + + +def test_literal_construction_invalid(): + with pytest.raises(ValueError): + plc.expressions.Literal( + plc.interop.from_arrow(pa.scalar(None, type=pa.list_(pa.int64()))) + ) + + +@pytest.mark.parametrize( + "tableref", + [ + plc.expressions.TableReference.LEFT, + plc.expressions.TableReference.RIGHT, + ], +) +def test_columnref_construction(tableref): + plc.expressions.ColumnReference(1.0, tableref) + + +def test_columnnameref_construction(): + plc.expressions.ColumnNameReference("abc") + + +@pytest.mark.parametrize( + "kwargs", + [ + # Unary op + { + "op": plc.expressions.ASTOperator.IDENTITY, + "left": plc.expressions.ColumnReference(1), + }, + # Binop + { + "op": plc.expressions.ASTOperator.ADD, + "left": plc.expressions.ColumnReference(1), + "right": plc.expressions.ColumnReference(2), + }, + ], +) +def test_astoperation_construction(kwargs): + plc.expressions.Operation(**kwargs) From 2f8d514b1687164a94bbe89da1dab8eb37682b35 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 16 Jul 2024 20:15:25 -0400 Subject: [PATCH 19/53] Remove xml from sort_ninja_log.py utility (#16274) Removes xml support from the `sort_ninja_log.py` utility. The xml support was experimental for possible use with Jenkins reporting that never materialized. This script is used in build.sh generally when running local builds. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Srinivas Yadav (https://github.com/srinivasyadav18) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16274 --- cpp/scripts/sort_ninja_log.py | 58 ++++++----------------------------- 1 file changed, 9 insertions(+), 49 deletions(-) diff --git a/cpp/scripts/sort_ninja_log.py b/cpp/scripts/sort_ninja_log.py index 3fe503f749e..42f84e4d0c7 100755 --- a/cpp/scripts/sort_ninja_log.py +++ b/cpp/scripts/sort_ninja_log.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # import argparse import os @@ -9,14 +9,12 @@ from xml.dom import minidom parser = argparse.ArgumentParser() -parser.add_argument( - "log_file", type=str, default=".ninja_log", help=".ninja_log file" -) +parser.add_argument("log_file", type=str, default=".ninja_log", help=".ninja_log file") parser.add_argument( "--fmt", type=str, default="csv", - choices=["csv", "xml", "html"], + choices=["csv", "html"], help="output format (to stdout)", ) parser.add_argument( @@ -37,6 +35,7 @@ output_fmt = args.fmt cmp_file = args.cmp_log + # build a map of the log entries def build_log_map(log_file): entries = {} @@ -68,37 +67,6 @@ def build_log_map(log_file): return entries -# output results in XML format -def output_xml(entries, sorted_list, args): - root = ET.Element("testsuites") - testsuite = ET.Element( - "testsuite", - attrib={ - "name": "build-time", - "tests": str(len(sorted_list)), - "failures": str(0), - "errors": str(0), - }, - ) - root.append(testsuite) - for name in sorted_list: - entry = entries[name] - build_time = float(entry[1] - entry[0]) / 1000 - item = ET.Element( - "testcase", - attrib={ - "classname": "BuildTime", - "name": name, - "time": str(build_time), - }, - ) - testsuite.append(item) - - tree = ET.ElementTree(root) - xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent=" ") - print(xmlstr) - - # utility converts a millisecond value to a column width in pixels def time_to_width(value, end): # map a value from (0,end) to (0,1000) @@ -282,9 +250,7 @@ def output_html(entries, sorted_list, cmp_entries, args): # output detail table in build-time descending order print("") - print( - "", "", "", sep="" - ) + print("", "", "", sep="") if cmp_entries: print("", sep="") print("") @@ -303,9 +269,7 @@ def output_html(entries, sorted_list, cmp_entries, args): print("", sep="", end="") print("", sep="", end="") # output diff column - cmp_entry = ( - cmp_entries[name] if cmp_entries and name in cmp_entries else None - ) + cmp_entry = cmp_entries[name] if cmp_entries and name in cmp_entries else None if cmp_entry: diff_time = build_time - (cmp_entry[1] - cmp_entry[0]) diff_time_str = format_build_time(diff_time) @@ -353,7 +317,7 @@ def output_html(entries, sorted_list, cmp_entries, args): print( "time change < 20%% or build time < 1 minute", + ">time change < 20% or build time < 1 minute", ) print("
FileCompile timeSize
FileCompile timeSizet-cmp
", build_time_str, "", file_size_str, "
") @@ -370,9 +334,7 @@ def output_csv(entries, sorted_list, cmp_entries, args): entry = entries[name] build_time = entry[1] - entry[0] file_size = entry[2] - cmp_entry = ( - cmp_entries[name] if cmp_entries and name in cmp_entries else None - ) + cmp_entry = cmp_entries[name] if cmp_entries and name in cmp_entries else None print(build_time, file_size, name, sep=",", end="") if cmp_entry: diff_time = build_time - (cmp_entry[1] - cmp_entry[0]) @@ -396,9 +358,7 @@ def output_csv(entries, sorted_list, cmp_entries, args): # load the comparison build log if available cmp_entries = build_log_map(cmp_file) if cmp_file else None -if output_fmt == "xml": - output_xml(entries, sorted_list, args) -elif output_fmt == "html": +if output_fmt == "html": output_html(entries, sorted_list, cmp_entries, args) else: output_csv(entries, sorted_list, cmp_entries, args) From 093bcc94ccf156a7e39339a7c4bb7e86543187de Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 16 Jul 2024 20:16:07 -0400 Subject: [PATCH 20/53] Update cudf::detail::grid_1d to use thread_index_type (#16276) Updates the `cudf::detail::grid_1d` to use `thread_index_type` instead of `int` and `size_type` for the number threads and blocks. This has become important for launching kernels with more threads than max `size_type` total bytes for warp-per-row and thread-per-byte algorithms. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/16276 --- cpp/include/cudf/detail/utilities/cuda.cuh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/include/cudf/detail/utilities/cuda.cuh b/cpp/include/cudf/detail/utilities/cuda.cuh index f1775c6d6d7..5007af7f9f1 100644 --- a/cpp/include/cudf/detail/utilities/cuda.cuh +++ b/cpp/include/cudf/detail/utilities/cuda.cuh @@ -41,8 +41,8 @@ static constexpr size_type warp_size{32}; */ class grid_1d { public: - int const num_threads_per_block; - int const num_blocks; + thread_index_type const num_threads_per_block; + thread_index_type const num_blocks; /** * @param overall_num_elements The number of elements the kernel needs to * handle/process, in its main, one-dimensional/linear input (e.g. one or more @@ -55,9 +55,9 @@ class grid_1d { * than a single element; this affects the number of threads the grid must * contain */ - grid_1d(cudf::size_type overall_num_elements, - cudf::size_type num_threads_per_block, - cudf::size_type elements_per_thread = 1) + grid_1d(thread_index_type overall_num_elements, + thread_index_type num_threads_per_block, + thread_index_type elements_per_thread = 1) : num_threads_per_block(num_threads_per_block), num_blocks(util::div_rounding_up_safe(overall_num_elements, elements_per_thread * num_threads_per_block)) From aa466aaf91bc329cc4fced9b9a3426d79bfe7ffc Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 17 Jul 2024 10:48:17 -0400 Subject: [PATCH 21/53] Move kernel vis over to CUDF_HIDDEN (#16165) Use CUDF_HIDDEN instead of the raw `__attribute__((visibility("hidden")))` for symbol visibility controls on the CUDA kernels that we call from multiple TUs. This is primarily a style change so that we have consistent visibility markup across the entire project Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Yunsong Wang (https://github.com/PointKernel) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/16165 --- cpp/src/join/mixed_join_kernel.cuh | 3 ++- cpp/src/join/mixed_join_kernels_semi.cu | 3 ++- cpp/src/join/mixed_join_size_kernel.cuh | 28 ++++++++++++------------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cpp/src/join/mixed_join_kernel.cuh b/cpp/src/join/mixed_join_kernel.cuh index 0fc1c3718b1..ea59f23c77f 100644 --- a/cpp/src/join/mixed_join_kernel.cuh +++ b/cpp/src/join/mixed_join_kernel.cuh @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -38,7 +39,7 @@ namespace cg = cooperative_groups; #pragma GCC diagnostic ignored "-Wattributes" template -__attribute__((visibility("hidden"))) __launch_bounds__(block_size) __global__ +CUDF_HIDDEN __launch_bounds__(block_size) __global__ void mixed_join(table_device_view left_table, table_device_view right_table, table_device_view probe, diff --git a/cpp/src/join/mixed_join_kernels_semi.cu b/cpp/src/join/mixed_join_kernels_semi.cu index 01e3fe09b38..1f31eaa7878 100644 --- a/cpp/src/join/mixed_join_kernels_semi.cu +++ b/cpp/src/join/mixed_join_kernels_semi.cu @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -34,7 +35,7 @@ namespace cg = cooperative_groups; #pragma GCC diagnostic ignored "-Wattributes" template -__attribute__((visibility("hidden"))) __launch_bounds__(block_size) __global__ +CUDF_HIDDEN __launch_bounds__(block_size) __global__ void mixed_join_semi(table_device_view left_table, table_device_view right_table, table_device_view probe, diff --git a/cpp/src/join/mixed_join_size_kernel.cuh b/cpp/src/join/mixed_join_size_kernel.cuh index 618e7a9082e..00a90f8273f 100644 --- a/cpp/src/join/mixed_join_size_kernel.cuh +++ b/cpp/src/join/mixed_join_size_kernel.cuh @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -35,20 +36,19 @@ namespace cg = cooperative_groups; #pragma GCC diagnostic ignored "-Wattributes" template -__attribute__((visibility("hidden"))) __launch_bounds__(block_size) __global__ - void compute_mixed_join_output_size( - table_device_view left_table, - table_device_view right_table, - table_device_view probe, - table_device_view build, - row_hash const hash_probe, - row_equality const equality_probe, - join_kind const join_type, - cudf::detail::mixed_multimap_type::device_view hash_table_view, - ast::detail::expression_device_view device_expression_data, - bool const swap_tables, - std::size_t* output_size, - cudf::device_span matches_per_row) +CUDF_HIDDEN __launch_bounds__(block_size) __global__ void compute_mixed_join_output_size( + table_device_view left_table, + table_device_view right_table, + table_device_view probe, + table_device_view build, + row_hash const hash_probe, + row_equality const equality_probe, + join_kind const join_type, + cudf::detail::mixed_multimap_type::device_view hash_table_view, + ast::detail::expression_device_view device_expression_data, + bool const swap_tables, + std::size_t* output_size, + cudf::device_span matches_per_row) { // The (required) extern storage of the shared memory array leads to // conflicting declarations between different templates. The easiest From 9db6723f2f2fe3451f0a5b81b7a43597358913ea Mon Sep 17 00:00:00 2001 From: jakirkham Date: Wed, 17 Jul 2024 09:54:09 -0700 Subject: [PATCH 22/53] Rename `.devcontainer`s for CUDA 12.5 (#16293) Follow up to PR: https://github.com/rapidsai/cudf/pull/16259 Partially addresses issue: https://github.com/rapidsai/build-planning/issues/73 Renames the `.devcontainer`s for CUDA 12.5 Authors: - https://github.com/jakirkham Approvers: - James Lamb (https://github.com/jameslamb) - Paul Taylor (https://github.com/trxcllnt) URL: https://github.com/rapidsai/cudf/pull/16293 --- .../{cuda12.2-conda => cuda12.5-conda}/devcontainer.json | 0 .devcontainer/{cuda12.2-pip => cuda12.5-pip}/devcontainer.json | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .devcontainer/{cuda12.2-conda => cuda12.5-conda}/devcontainer.json (100%) rename .devcontainer/{cuda12.2-pip => cuda12.5-pip}/devcontainer.json (100%) diff --git a/.devcontainer/cuda12.2-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json similarity index 100% rename from .devcontainer/cuda12.2-conda/devcontainer.json rename to .devcontainer/cuda12.5-conda/devcontainer.json diff --git a/.devcontainer/cuda12.2-pip/devcontainer.json b/.devcontainer/cuda12.5-pip/devcontainer.json similarity index 100% rename from .devcontainer/cuda12.2-pip/devcontainer.json rename to .devcontainer/cuda12.5-pip/devcontainer.json From 1dd63ea8b28339c3b4a351b82dd81d425d985ba3 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 17 Jul 2024 08:47:32 -1000 Subject: [PATCH 23/53] Short circuit some Column methods (#16246) Adds some short circuiting, possibly cached checks (e.g. all values unique, no-NAs, monotonicity), to `dropna`, `isnull`, `notnull`, `argsort`, `unique` and `sort_values` allowing these ops to just copy / return a "simplified" result Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16246 --- python/cudf/cudf/_lib/column.pyx | 12 ++++--- python/cudf/cudf/core/column/column.py | 50 ++++++++++++++++++++------ 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/_lib/column.pyx b/python/cudf/cudf/_lib/column.pyx index 7155017b7af..e030147fdd3 100644 --- a/python/cudf/cudf/_lib/column.pyx +++ b/python/cudf/cudf/_lib/column.pyx @@ -202,11 +202,13 @@ cdef class Column: def _clear_cache(self): self._distinct_count = {} - try: - del self.memory_usage - except AttributeError: - # `self.memory_usage` was never called before, So ignore. - pass + attrs = ("memory_usage", "is_monotonic_increasing", "is_monotonic_decreasing") + for attr in attrs: + try: + delattr(self, attr) + except AttributeError: + # attr was not called yet, so ignore. + pass self._null_count = None def set_mask(self, value): diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index dbdf501e022..9467bbeed15 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -274,7 +274,10 @@ def any(self, skipna: bool = True) -> bool: return libcudf.reduce.reduce("any", self, dtype=np.bool_) def dropna(self) -> Self: - return drop_nulls([self])[0]._with_type_metadata(self.dtype) + if self.has_nulls(): + return drop_nulls([self])[0]._with_type_metadata(self.dtype) + else: + return self.copy() def to_arrow(self) -> pa.Array: """Convert to PyArrow Array @@ -699,6 +702,9 @@ def fillna( def isnull(self) -> ColumnBase: """Identify missing values in a Column.""" + if not self.has_nulls(include_nan=self.dtype.kind == "f"): + return as_column(False, length=len(self)) + result = libcudf.unary.is_null(self) if self.dtype.kind == "f": @@ -710,6 +716,9 @@ def isnull(self) -> ColumnBase: def notnull(self) -> ColumnBase: """Identify non-missing values in a Column.""" + if not self.has_nulls(include_nan=self.dtype.kind == "f"): + return as_column(True, length=len(self)) + result = libcudf.unary.is_valid(self) if self.dtype.kind == "f": @@ -922,15 +931,16 @@ def as_mask(self) -> Buffer: @property def is_unique(self) -> bool: + # distinct_count might already be cached return self.distinct_count(dropna=False) == len(self) - @property + @cached_property def is_monotonic_increasing(self) -> bool: return not self.has_nulls(include_nan=True) and libcudf.sort.is_sorted( [self], [True], None ) - @property + @cached_property def is_monotonic_decreasing(self) -> bool: return not self.has_nulls(include_nan=True) and libcudf.sort.is_sorted( [self], [False], None @@ -941,6 +951,10 @@ def sort_values( ascending: bool = True, na_position: str = "last", ) -> ColumnBase: + if (not ascending and self.is_monotonic_decreasing) or ( + ascending and self.is_monotonic_increasing + ): + return self.copy() return libcudf.sort.sort( [self], column_order=[ascending], null_precedence=[na_position] )[0] @@ -1090,11 +1104,22 @@ def apply_boolean_mask(self, mask) -> ColumnBase: ) def argsort( - self, ascending: bool = True, na_position: str = "last" - ) -> "cudf.core.column.NumericalColumn": - return libcudf.sort.order_by( - [self], [ascending], na_position, stable=True - ) + self, + ascending: bool = True, + na_position: Literal["first", "last"] = "last", + ) -> cudf.core.column.NumericalColumn: + if (ascending and self.is_monotonic_increasing) or ( + not ascending and self.is_monotonic_decreasing + ): + return as_column(range(len(self))) + elif (ascending and self.is_monotonic_decreasing) or ( + not ascending and self.is_monotonic_increasing + ): + return as_column(range(len(self) - 1, -1, -1)) + else: + return libcudf.sort.order_by( + [self], [ascending], na_position, stable=True + ) def __arrow_array__(self, type=None): raise TypeError( @@ -1157,9 +1182,12 @@ def unique(self) -> ColumnBase: """ Get unique values in the data """ - return drop_duplicates([self], keep="first")[0]._with_type_metadata( - self.dtype - ) + if self.is_unique: + return self.copy() + else: + return drop_duplicates([self], keep="first")[ + 0 + ]._with_type_metadata(self.dtype) def serialize(self) -> tuple[dict, list]: # data model: From 8b767e5c237840e0a35848bff7ed479ec5c56bb1 Mon Sep 17 00:00:00 2001 From: Paul Mattione <156858817+pmattione-nvidia@users.noreply.github.com> Date: Wed, 17 Jul 2024 13:33:45 -0600 Subject: [PATCH 24/53] Remove decimal/floating 64/128bit switches due to register pressure (#16287) The decimal <--> floating conversion PR reduced the performance of some of the AST and BINARYOP kernels due to register pressure. This removes the switches that are the primary source of the register pressure, falling back to the old ipow() method for 64bit and 128bit integers. Authors: - Paul Mattione (https://github.com/pmattione-nvidia) Approvers: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/16287 --- cpp/include/cudf/fixed_point/fixed_point.hpp | 4 +- .../cudf/fixed_point/floating_conversion.hpp | 138 +----------------- 2 files changed, 6 insertions(+), 136 deletions(-) diff --git a/cpp/include/cudf/fixed_point/fixed_point.hpp b/cpp/include/cudf/fixed_point/fixed_point.hpp index 6c3c3b4da07..c9cbc603226 100644 --- a/cpp/include/cudf/fixed_point/fixed_point.hpp +++ b/cpp/include/cudf/fixed_point/fixed_point.hpp @@ -84,8 +84,8 @@ template && - is_supported_representation_type())>* = nullptr> -CUDF_HOST_DEVICE inline Rep ipow(T exponent) + cuda::std::is_integral_v)>* = nullptr> +CUDF_HOST_DEVICE inline constexpr Rep ipow(T exponent) { cudf_assert(exponent >= 0 && "integer exponentiation with negative exponent is not possible."); diff --git a/cpp/include/cudf/fixed_point/floating_conversion.hpp b/cpp/include/cudf/fixed_point/floating_conversion.hpp index c64ae8877d4..f12177c6a4b 100644 --- a/cpp/include/cudf/fixed_point/floating_conversion.hpp +++ b/cpp/include/cudf/fixed_point/floating_conversion.hpp @@ -392,30 +392,7 @@ CUDF_HOST_DEVICE inline T divide_power10_32bit(T value, int pow10) template )> CUDF_HOST_DEVICE inline T divide_power10_64bit(T value, int pow10) { - // See comments in divide_power10_32bit() for discussion. - switch (pow10) { - case 0: return value; - case 1: return value / 10U; - case 2: return value / 100U; - case 3: return value / 1000U; - case 4: return value / 10000U; - case 5: return value / 100000U; - case 6: return value / 1000000U; - case 7: return value / 10000000U; - case 8: return value / 100000000U; - case 9: return value / 1000000000U; - case 10: return value / 10000000000ULL; - case 11: return value / 100000000000ULL; - case 12: return value / 1000000000000ULL; - case 13: return value / 10000000000000ULL; - case 14: return value / 100000000000000ULL; - case 15: return value / 1000000000000000ULL; - case 16: return value / 10000000000000000ULL; - case 17: return value / 100000000000000000ULL; - case 18: return value / 1000000000000000000ULL; - case 19: return value / 10000000000000000000ULL; - default: return 0; - } + return value / ipow(pow10); } /** @@ -429,49 +406,7 @@ CUDF_HOST_DEVICE inline T divide_power10_64bit(T value, int pow10) template )> CUDF_HOST_DEVICE inline constexpr T divide_power10_128bit(T value, int pow10) { - // See comments in divide_power10_32bit() for an introduction. - switch (pow10) { - case 0: return value; - case 1: return value / 10U; - case 2: return value / 100U; - case 3: return value / 1000U; - case 4: return value / 10000U; - case 5: return value / 100000U; - case 6: return value / 1000000U; - case 7: return value / 10000000U; - case 8: return value / 100000000U; - case 9: return value / 1000000000U; - case 10: return value / 10000000000ULL; - case 11: return value / 100000000000ULL; - case 12: return value / 1000000000000ULL; - case 13: return value / 10000000000000ULL; - case 14: return value / 100000000000000ULL; - case 15: return value / 1000000000000000ULL; - case 16: return value / 10000000000000000ULL; - case 17: return value / 100000000000000000ULL; - case 18: return value / 1000000000000000000ULL; - case 19: return value / 10000000000000000000ULL; - case 20: return value / large_power_of_10<20>(); - case 21: return value / large_power_of_10<21>(); - case 22: return value / large_power_of_10<22>(); - case 23: return value / large_power_of_10<23>(); - case 24: return value / large_power_of_10<24>(); - case 25: return value / large_power_of_10<25>(); - case 26: return value / large_power_of_10<26>(); - case 27: return value / large_power_of_10<27>(); - case 28: return value / large_power_of_10<28>(); - case 29: return value / large_power_of_10<29>(); - case 30: return value / large_power_of_10<30>(); - case 31: return value / large_power_of_10<31>(); - case 32: return value / large_power_of_10<32>(); - case 33: return value / large_power_of_10<33>(); - case 34: return value / large_power_of_10<34>(); - case 35: return value / large_power_of_10<35>(); - case 36: return value / large_power_of_10<36>(); - case 37: return value / large_power_of_10<37>(); - case 38: return value / large_power_of_10<38>(); - default: return 0; - } + return value / ipow<__uint128_t, Radix::BASE_10>(pow10); } /** @@ -512,30 +447,7 @@ CUDF_HOST_DEVICE inline constexpr T multiply_power10_32bit(T value, int pow10) template )> CUDF_HOST_DEVICE inline constexpr T multiply_power10_64bit(T value, int pow10) { - // See comments in divide_power10_32bit() for discussion. - switch (pow10) { - case 0: return value; - case 1: return value * 10U; - case 2: return value * 100U; - case 3: return value * 1000U; - case 4: return value * 10000U; - case 5: return value * 100000U; - case 6: return value * 1000000U; - case 7: return value * 10000000U; - case 8: return value * 100000000U; - case 9: return value * 1000000000U; - case 10: return value * 10000000000ULL; - case 11: return value * 100000000000ULL; - case 12: return value * 1000000000000ULL; - case 13: return value * 10000000000000ULL; - case 14: return value * 100000000000000ULL; - case 15: return value * 1000000000000000ULL; - case 16: return value * 10000000000000000ULL; - case 17: return value * 100000000000000000ULL; - case 18: return value * 1000000000000000000ULL; - case 19: return value * 10000000000000000000ULL; - default: return 0; - } + return value * ipow(pow10); } /** @@ -549,49 +461,7 @@ CUDF_HOST_DEVICE inline constexpr T multiply_power10_64bit(T value, int pow10) template )> CUDF_HOST_DEVICE inline constexpr T multiply_power10_128bit(T value, int pow10) { - // See comments in divide_power10_128bit() for discussion. - switch (pow10) { - case 0: return value; - case 1: return value * 10U; - case 2: return value * 100U; - case 3: return value * 1000U; - case 4: return value * 10000U; - case 5: return value * 100000U; - case 6: return value * 1000000U; - case 7: return value * 10000000U; - case 8: return value * 100000000U; - case 9: return value * 1000000000U; - case 10: return value * 10000000000ULL; - case 11: return value * 100000000000ULL; - case 12: return value * 1000000000000ULL; - case 13: return value * 10000000000000ULL; - case 14: return value * 100000000000000ULL; - case 15: return value * 1000000000000000ULL; - case 16: return value * 10000000000000000ULL; - case 17: return value * 100000000000000000ULL; - case 18: return value * 1000000000000000000ULL; - case 19: return value * 10000000000000000000ULL; - case 20: return value * large_power_of_10<20>(); - case 21: return value * large_power_of_10<21>(); - case 22: return value * large_power_of_10<22>(); - case 23: return value * large_power_of_10<23>(); - case 24: return value * large_power_of_10<24>(); - case 25: return value * large_power_of_10<25>(); - case 26: return value * large_power_of_10<26>(); - case 27: return value * large_power_of_10<27>(); - case 28: return value * large_power_of_10<28>(); - case 29: return value * large_power_of_10<29>(); - case 30: return value * large_power_of_10<30>(); - case 31: return value * large_power_of_10<31>(); - case 32: return value * large_power_of_10<32>(); - case 33: return value * large_power_of_10<33>(); - case 34: return value * large_power_of_10<34>(); - case 35: return value * large_power_of_10<35>(); - case 36: return value * large_power_of_10<36>(); - case 37: return value * large_power_of_10<37>(); - case 38: return value * large_power_of_10<38>(); - default: return 0; - } + return value * ipow<__uint128_t, Radix::BASE_10>(pow10); } /** From 34dea6fe40fc20966b48257853865111df4a687f Mon Sep 17 00:00:00 2001 From: Jayjeet Chakraborty Date: Wed, 17 Jul 2024 15:27:40 -0700 Subject: [PATCH 25/53] Add TPC-H inspired examples for Libcudf (#16088) This PR adds a suite of `libcudf` examples with queries inspired from the TPC-H benchmarks. This PR also adds some reusable helper functions to perform operations such as joins, groubys, and orderbys for a cleaner and modular implementation of the queries. # Queries implemented so far: - [x] Query 1 - [X] Query 5 - [X] Query 6 - [X] Query 9 Authors: - Jayjeet Chakraborty (https://github.com/JayjeetAtGithub) - Muhammad Haseeb (https://github.com/mhaseeb123) Approvers: - Muhammad Haseeb (https://github.com/mhaseeb123) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/16088 --- cpp/examples/build.sh | 1 + cpp/examples/parquet_io/parquet_io.cpp | 4 +- cpp/examples/parquet_io/parquet_io.hpp | 31 -- cpp/examples/tpch/CMakeLists.txt | 32 ++ cpp/examples/tpch/README.md | 38 ++ cpp/examples/tpch/q1.cpp | 174 ++++++++++ cpp/examples/tpch/q5.cpp | 169 +++++++++ cpp/examples/tpch/q6.cpp | 137 ++++++++ cpp/examples/tpch/q9.cpp | 182 ++++++++++ cpp/examples/tpch/utils.hpp | 457 +++++++++++++++++++++++++ cpp/examples/utilities/timer.hpp | 54 +++ 11 files changed, 1247 insertions(+), 32 deletions(-) create mode 100644 cpp/examples/tpch/CMakeLists.txt create mode 100644 cpp/examples/tpch/README.md create mode 100644 cpp/examples/tpch/q1.cpp create mode 100644 cpp/examples/tpch/q5.cpp create mode 100644 cpp/examples/tpch/q6.cpp create mode 100644 cpp/examples/tpch/q9.cpp create mode 100644 cpp/examples/tpch/utils.hpp create mode 100644 cpp/examples/utilities/timer.hpp diff --git a/cpp/examples/build.sh b/cpp/examples/build.sh index bde6ef7d69c..dce81fb1677 100755 --- a/cpp/examples/build.sh +++ b/cpp/examples/build.sh @@ -57,6 +57,7 @@ build_example() { } build_example basic +build_example tpch build_example strings build_example nested_types build_example parquet_io diff --git a/cpp/examples/parquet_io/parquet_io.cpp b/cpp/examples/parquet_io/parquet_io.cpp index 8be17db3781..274a2599189 100644 --- a/cpp/examples/parquet_io/parquet_io.cpp +++ b/cpp/examples/parquet_io/parquet_io.cpp @@ -16,6 +16,8 @@ #include "parquet_io.hpp" +#include "../utilities/timer.hpp" + /** * @file parquet_io.cpp * @brief Demonstrates usage of the libcudf APIs to read and write @@ -140,7 +142,7 @@ int main(int argc, char const** argv) << page_stat_string << ".." << std::endl; // `timer` is automatically started here - Timer timer; + cudf::examples::timer timer; write_parquet(input->view(), metadata, output_filepath, encoding, compression, page_stats); timer.print_elapsed_millis(); diff --git a/cpp/examples/parquet_io/parquet_io.hpp b/cpp/examples/parquet_io/parquet_io.hpp index d2fc359a2fe..e27cbec4fce 100644 --- a/cpp/examples/parquet_io/parquet_io.hpp +++ b/cpp/examples/parquet_io/parquet_io.hpp @@ -124,34 +124,3 @@ std::shared_ptr create_memory_resource(bool is_ return std::nullopt; } - -/** - * @brief Light-weight timer for parquet reader and writer instrumentation - * - * Timer object constructed from std::chrono, instrumenting at microseconds - * precision. Can display elapsed durations at milli and micro second - * scales. Timer starts at object construction. - */ -class Timer { - public: - using micros = std::chrono::microseconds; - using millis = std::chrono::milliseconds; - - Timer() { reset(); } - void reset() { start_time = std::chrono::high_resolution_clock::now(); } - auto elapsed() { return (std::chrono::high_resolution_clock::now() - start_time); } - void print_elapsed_micros() - { - std::cout << "Elapsed Time: " << std::chrono::duration_cast(elapsed()).count() - << "us\n\n"; - } - void print_elapsed_millis() - { - std::cout << "Elapsed Time: " << std::chrono::duration_cast(elapsed()).count() - << "ms\n\n"; - } - - private: - using time_point_t = std::chrono::time_point; - time_point_t start_time; -}; diff --git a/cpp/examples/tpch/CMakeLists.txt b/cpp/examples/tpch/CMakeLists.txt new file mode 100644 index 00000000000..1b91d07e148 --- /dev/null +++ b/cpp/examples/tpch/CMakeLists.txt @@ -0,0 +1,32 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +cmake_minimum_required(VERSION 3.26.4) + +include(../set_cuda_architecture.cmake) + +rapids_cuda_init_architectures(tpch_example) +rapids_cuda_set_architectures(RAPIDS) + +project( + tpch_example + VERSION 0.0.1 + LANGUAGES CXX CUDA +) + +include(../fetch_dependencies.cmake) + +add_executable(tpch_q1 q1.cpp) +target_link_libraries(tpch_q1 PRIVATE cudf::cudf) +target_compile_features(tpch_q1 PRIVATE cxx_std_17) + +add_executable(tpch_q5 q5.cpp) +target_link_libraries(tpch_q5 PRIVATE cudf::cudf) +target_compile_features(tpch_q5 PRIVATE cxx_std_17) + +add_executable(tpch_q6 q6.cpp) +target_link_libraries(tpch_q6 PRIVATE cudf::cudf) +target_compile_features(tpch_q6 PRIVATE cxx_std_17) + +add_executable(tpch_q9 q9.cpp) +target_link_libraries(tpch_q9 PRIVATE cudf::cudf) +target_compile_features(tpch_q9 PRIVATE cxx_std_17) diff --git a/cpp/examples/tpch/README.md b/cpp/examples/tpch/README.md new file mode 100644 index 00000000000..1ea71ae9824 --- /dev/null +++ b/cpp/examples/tpch/README.md @@ -0,0 +1,38 @@ +# TPC-H Inspired Examples + +Implements TPC-H queries using `libcudf`. We leverage the data generator (wrapper around official TPC-H datagen) from [Apache Datafusion](https://github.com/apache/datafusion) for generating data in Parquet format. + +## Requirements + +- Rust + +## Generating the Dataset + +1. Clone the datafusion repository. +```bash +git clone git@github.com:apache/datafusion.git +``` + +2. Run the data generator. The data will be placed in a `data/` subdirectory. +```bash +cd datafusion/benchmarks/ +./bench.sh data tpch + +# for scale factor 10, +./bench.sh data tpch10 +``` + +## Running Queries + +1. Build the examples. +```bash +cd cpp/examples +./build.sh +``` +The TPC-H query binaries would be built inside `examples/tpch/build`. + +2. Execute the queries. +```bash +./tpch/build/tpch_q1 +``` +A parquet file named `q1.parquet` would be generated holding the results of the query. diff --git a/cpp/examples/tpch/q1.cpp b/cpp/examples/tpch/q1.cpp new file mode 100644 index 00000000000..1bdf039da4a --- /dev/null +++ b/cpp/examples/tpch/q1.cpp @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../utilities/timer.hpp" +#include "utils.hpp" + +#include +#include +#include + +/** + * @file q1.cpp + * @brief Implement query 1 of the TPC-H benchmark. + * + * create view lineitem as select * from '/tables/scale-1/lineitem.parquet'; + * + * select + * l_returnflag, + * l_linestatus, + * sum(l_quantity) as sum_qty, + * sum(l_extendedprice) as sum_base_price, + * sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + * sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + * avg(l_quantity) as avg_qty, + * avg(l_extendedprice) as avg_price, + * avg(l_discount) as avg_disc, + * count(*) as count_order + * from + * lineitem + * where + * l_shipdate <= date '1998-09-02' + * group by + * l_returnflag, + * l_linestatus + * order by + * l_returnflag, + * l_linestatus; + */ + +/** + * @brief Calculate the discount price column + * + * @param discount The discount column + * @param extendedprice The extended price column + * @param stream The CUDA stream used for device memory operations and kernel launches. + * @param mr Device memory resource used to allocate the returned column's device memory. + */ +[[nodiscard]] std::unique_ptr calc_disc_price( + cudf::column_view const& discount, + cudf::column_view const& extendedprice, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()) +{ + auto const one = cudf::numeric_scalar(1); + auto const one_minus_discount = + cudf::binary_operation(one, discount, cudf::binary_operator::SUB, discount.type(), stream, mr); + auto const disc_price_type = cudf::data_type{cudf::type_id::FLOAT64}; + auto disc_price = cudf::binary_operation(extendedprice, + one_minus_discount->view(), + cudf::binary_operator::MUL, + disc_price_type, + stream, + mr); + return disc_price; +} + +/** + * @brief Calculate the charge column + * + * @param tax The tax column + * @param disc_price The discount price column + * @param stream The CUDA stream used for device memory operations and kernel launches. + * @param mr Device memory resource used to allocate the returned column's device memory. + */ +[[nodiscard]] std::unique_ptr calc_charge( + cudf::column_view const& tax, + cudf::column_view const& disc_price, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()) +{ + auto const one = cudf::numeric_scalar(1); + auto const one_plus_tax = + cudf::binary_operation(one, tax, cudf::binary_operator::ADD, tax.type(), stream, mr); + auto const charge_type = cudf::data_type{cudf::type_id::FLOAT64}; + auto charge = cudf::binary_operation( + disc_price, one_plus_tax->view(), cudf::binary_operator::MUL, charge_type, stream, mr); + return charge; +} + +int main(int argc, char const** argv) +{ + auto const args = parse_args(argc, argv); + + // Use a memory pool + auto resource = create_memory_resource(args.memory_resource_type); + rmm::mr::set_current_device_resource(resource.get()); + + cudf::examples::timer timer; + + // Define the column projections and filter predicate for `lineitem` table + std::vector const lineitem_cols = {"l_returnflag", + "l_linestatus", + "l_quantity", + "l_extendedprice", + "l_discount", + "l_shipdate", + "l_orderkey", + "l_tax"}; + auto const shipdate_ref = cudf::ast::column_reference(std::distance( + lineitem_cols.begin(), std::find(lineitem_cols.begin(), lineitem_cols.end(), "l_shipdate"))); + auto shipdate_upper = + cudf::timestamp_scalar(days_since_epoch(1998, 9, 2), true); + auto const shipdate_upper_literal = cudf::ast::literal(shipdate_upper); + auto lineitem_pred = std::make_unique( + cudf::ast::ast_operator::LESS_EQUAL, shipdate_ref, shipdate_upper_literal); + + // Read out the `lineitem` table from parquet file + auto lineitem = + read_parquet(args.dataset_dir + "/lineitem.parquet", lineitem_cols, std::move(lineitem_pred)); + + // Calculate the discount price and charge columns and append to lineitem table + auto disc_price = + calc_disc_price(lineitem->column("l_discount"), lineitem->column("l_extendedprice")); + auto charge = calc_charge(lineitem->column("l_tax"), disc_price->view()); + (*lineitem).append(disc_price, "disc_price").append(charge, "charge"); + + // Perform the group by operation + auto const groupedby_table = apply_groupby( + lineitem, + groupby_context_t{ + {"l_returnflag", "l_linestatus"}, + { + {"l_extendedprice", + {{cudf::aggregation::Kind::SUM, "sum_base_price"}, + {cudf::aggregation::Kind::MEAN, "avg_price"}}}, + {"l_quantity", + {{cudf::aggregation::Kind::SUM, "sum_qty"}, {cudf::aggregation::Kind::MEAN, "avg_qty"}}}, + {"l_discount", + { + {cudf::aggregation::Kind::MEAN, "avg_disc"}, + }}, + {"disc_price", + { + {cudf::aggregation::Kind::SUM, "sum_disc_price"}, + }}, + {"charge", + {{cudf::aggregation::Kind::SUM, "sum_charge"}, + {cudf::aggregation::Kind::COUNT_ALL, "count_order"}}}, + }}); + + // Perform the order by operation + auto const orderedby_table = apply_orderby(groupedby_table, + {"l_returnflag", "l_linestatus"}, + {cudf::order::ASCENDING, cudf::order::ASCENDING}); + + timer.print_elapsed_millis(); + + // Write query result to a parquet file + orderedby_table->to_parquet("q1.parquet"); + return 0; +} diff --git a/cpp/examples/tpch/q5.cpp b/cpp/examples/tpch/q5.cpp new file mode 100644 index 00000000000..e56850b94d6 --- /dev/null +++ b/cpp/examples/tpch/q5.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../utilities/timer.hpp" +#include "utils.hpp" + +#include +#include +#include + +/** + * @file q5.cpp + * @brief Implement query 5 of the TPC-H benchmark. + * + * create view customer as select * from '/tables/scale-1/customer.parquet'; + * create view orders as select * from '/tables/scale-1/orders.parquet'; + * create view lineitem as select * from '/tables/scale-1/lineitem.parquet'; + * create view supplier as select * from '/tables/scale-1/supplier.parquet'; + * create view nation as select * from '/tables/scale-1/nation.parquet'; + * create view region as select * from '/tables/scale-1/region.parquet'; + * + * select + * n_name, + * sum(l_extendedprice * (1 - l_discount)) as revenue + * from + * customer, + * orders, + * lineitem, + * supplier, + * nation, + * region + * where + * c_custkey = o_custkey + * and l_orderkey = o_orderkey + * and l_suppkey = s_suppkey + * and c_nationkey = s_nationkey + * and s_nationkey = n_nationkey + * and n_regionkey = r_regionkey + * and r_name = 'ASIA' + * and o_orderdate >= date '1994-01-01' + * and o_orderdate < date '1995-01-01' + * group by + * n_name + * order by + * revenue desc; + */ + +/** + * @brief Calculate the revenue column + * + * @param extendedprice The extended price column + * @param discount The discount column + * @param stream The CUDA stream used for device memory operations and kernel launches. + * @param mr Device memory resource used to allocate the returned column's device memory. + */ +[[nodiscard]] std::unique_ptr calc_revenue( + cudf::column_view const& extendedprice, + cudf::column_view const& discount, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()) +{ + auto const one = cudf::numeric_scalar(1); + auto const one_minus_discount = + cudf::binary_operation(one, discount, cudf::binary_operator::SUB, discount.type(), stream, mr); + auto const revenue_type = cudf::data_type{cudf::type_id::FLOAT64}; + auto revenue = cudf::binary_operation(extendedprice, + one_minus_discount->view(), + cudf::binary_operator::MUL, + revenue_type, + stream, + mr); + return revenue; +} + +int main(int argc, char const** argv) +{ + auto const args = parse_args(argc, argv); + + // Use a memory pool + auto resource = create_memory_resource(args.memory_resource_type); + rmm::mr::set_current_device_resource(resource.get()); + + cudf::examples::timer timer; + + // Define the column projection and filter predicate for the `orders` table + std::vector const orders_cols = {"o_custkey", "o_orderkey", "o_orderdate"}; + auto const o_orderdate_ref = cudf::ast::column_reference(std::distance( + orders_cols.begin(), std::find(orders_cols.begin(), orders_cols.end(), "o_orderdate"))); + auto o_orderdate_lower = + cudf::timestamp_scalar(days_since_epoch(1994, 1, 1), true); + auto const o_orderdate_lower_limit = cudf::ast::literal(o_orderdate_lower); + auto const o_orderdate_pred_lower = cudf::ast::operation( + cudf::ast::ast_operator::GREATER_EQUAL, o_orderdate_ref, o_orderdate_lower_limit); + auto o_orderdate_upper = + cudf::timestamp_scalar(days_since_epoch(1995, 1, 1), true); + auto const o_orderdate_upper_limit = cudf::ast::literal(o_orderdate_upper); + auto const o_orderdate_pred_upper = + cudf::ast::operation(cudf::ast::ast_operator::LESS, o_orderdate_ref, o_orderdate_upper_limit); + auto orders_pred = std::make_unique( + cudf::ast::ast_operator::LOGICAL_AND, o_orderdate_pred_lower, o_orderdate_pred_upper); + + // Define the column projection and filter predicate for the `region` table + std::vector const region_cols = {"r_regionkey", "r_name"}; + auto const r_name_ref = cudf::ast::column_reference(std::distance( + region_cols.begin(), std::find(region_cols.begin(), region_cols.end(), "r_name"))); + auto r_name_value = cudf::string_scalar("ASIA"); + auto const r_name_literal = cudf::ast::literal(r_name_value); + auto region_pred = std::make_unique( + cudf::ast::ast_operator::EQUAL, r_name_ref, r_name_literal); + + // Read out the tables from parquet files + // while pushing down the column projections and filter predicates + auto const customer = + read_parquet(args.dataset_dir + "/customer.parquet", {"c_custkey", "c_nationkey"}); + auto const orders = + read_parquet(args.dataset_dir + "/orders.parquet", orders_cols, std::move(orders_pred)); + auto const lineitem = read_parquet(args.dataset_dir + "/lineitem.parquet", + {"l_orderkey", "l_suppkey", "l_extendedprice", "l_discount"}); + auto const supplier = + read_parquet(args.dataset_dir + "/supplier.parquet", {"s_suppkey", "s_nationkey"}); + auto const nation = + read_parquet(args.dataset_dir + "/nation.parquet", {"n_nationkey", "n_regionkey", "n_name"}); + auto const region = + read_parquet(args.dataset_dir + "/region.parquet", region_cols, std::move(region_pred)); + + // Perform the joins + auto const join_a = apply_inner_join(region, nation, {"r_regionkey"}, {"n_regionkey"}); + auto const join_b = apply_inner_join(join_a, customer, {"n_nationkey"}, {"c_nationkey"}); + auto const join_c = apply_inner_join(join_b, orders, {"c_custkey"}, {"o_custkey"}); + auto const join_d = apply_inner_join(join_c, lineitem, {"o_orderkey"}, {"l_orderkey"}); + auto joined_table = + apply_inner_join(supplier, join_d, {"s_suppkey", "s_nationkey"}, {"l_suppkey", "n_nationkey"}); + + // Calculate and append the `revenue` column + auto revenue = + calc_revenue(joined_table->column("l_extendedprice"), joined_table->column("l_discount")); + (*joined_table).append(revenue, "revenue"); + + // Perform the groupby operation + auto const groupedby_table = + apply_groupby(joined_table, + groupby_context_t{{"n_name"}, + { + {"revenue", {{cudf::aggregation::Kind::SUM, "revenue"}}}, + }}); + + // Perform the order by operation + auto const orderedby_table = + apply_orderby(groupedby_table, {"revenue"}, {cudf::order::DESCENDING}); + + timer.print_elapsed_millis(); + + // Write query result to a parquet file + orderedby_table->to_parquet("q5.parquet"); + return 0; +} diff --git a/cpp/examples/tpch/q6.cpp b/cpp/examples/tpch/q6.cpp new file mode 100644 index 00000000000..f11b3d6ab3b --- /dev/null +++ b/cpp/examples/tpch/q6.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../utilities/timer.hpp" +#include "utils.hpp" + +#include +#include +#include + +/** + * @file q6.cpp + * @brief Implement query 6 of the TPC-H benchmark. + * + * create view lineitem as select * from '/tables/scale-1/lineitem.parquet'; + * + * select + * sum(l_extendedprice * l_discount) as revenue + * from + * lineitem + * where + * l_shipdate >= date '1994-01-01' + * and l_shipdate < date '1995-01-01' + * and l_discount >= 0.05 + * and l_discount <= 0.07 + * and l_quantity < 24; + */ + +/** + * @brief Calculate the revenue column + * + * @param extendedprice The extended price column + * @param discount The discount column + * @param stream The CUDA stream used for device memory operations and kernel launches. + * @param mr Device memory resource used to allocate the returned column's device memory. + */ +[[nodiscard]] std::unique_ptr calc_revenue( + cudf::column_view const& extendedprice, + cudf::column_view const& discount, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()) +{ + auto const revenue_type = cudf::data_type{cudf::type_id::FLOAT64}; + auto revenue = cudf::binary_operation( + extendedprice, discount, cudf::binary_operator::MUL, revenue_type, stream, mr); + return revenue; +} + +int main(int argc, char const** argv) +{ + auto const args = parse_args(argc, argv); + + // Use a memory pool + auto resource = create_memory_resource(args.memory_resource_type); + rmm::mr::set_current_device_resource(resource.get()); + + cudf::examples::timer timer; + + // Read out the `lineitem` table from parquet file + std::vector const lineitem_cols = { + "l_extendedprice", "l_discount", "l_shipdate", "l_quantity"}; + auto const shipdate_ref = cudf::ast::column_reference(std::distance( + lineitem_cols.begin(), std::find(lineitem_cols.begin(), lineitem_cols.end(), "l_shipdate"))); + auto shipdate_lower = + cudf::timestamp_scalar(days_since_epoch(1994, 1, 1), true); + auto const shipdate_lower_literal = cudf::ast::literal(shipdate_lower); + auto shipdate_upper = + cudf::timestamp_scalar(days_since_epoch(1995, 1, 1), true); + auto const shipdate_upper_literal = cudf::ast::literal(shipdate_upper); + auto const shipdate_pred_a = cudf::ast::operation( + cudf::ast::ast_operator::GREATER_EQUAL, shipdate_ref, shipdate_lower_literal); + auto const shipdate_pred_b = + cudf::ast::operation(cudf::ast::ast_operator::LESS, shipdate_ref, shipdate_upper_literal); + auto lineitem_pred = std::make_unique( + cudf::ast::ast_operator::LOGICAL_AND, shipdate_pred_a, shipdate_pred_b); + auto lineitem = + read_parquet(args.dataset_dir + "/lineitem.parquet", lineitem_cols, std::move(lineitem_pred)); + + // Cast the discount and quantity columns to float32 and append to lineitem table + auto discout_float = + cudf::cast(lineitem->column("l_discount"), cudf::data_type{cudf::type_id::FLOAT32}); + auto quantity_float = + cudf::cast(lineitem->column("l_quantity"), cudf::data_type{cudf::type_id::FLOAT32}); + + (*lineitem).append(discout_float, "l_discount_float").append(quantity_float, "l_quantity_float"); + + // Apply the filters + auto const discount_ref = cudf::ast::column_reference(lineitem->col_id("l_discount_float")); + auto const quantity_ref = cudf::ast::column_reference(lineitem->col_id("l_quantity_float")); + + auto discount_lower = cudf::numeric_scalar(0.05); + auto const discount_lower_literal = cudf::ast::literal(discount_lower); + auto discount_upper = cudf::numeric_scalar(0.07); + auto const discount_upper_literal = cudf::ast::literal(discount_upper); + auto quantity_upper = cudf::numeric_scalar(24); + auto const quantity_upper_literal = cudf::ast::literal(quantity_upper); + + auto const discount_pred_a = cudf::ast::operation( + cudf::ast::ast_operator::GREATER_EQUAL, discount_ref, discount_lower_literal); + + auto const discount_pred_b = + cudf::ast::operation(cudf::ast::ast_operator::LESS_EQUAL, discount_ref, discount_upper_literal); + auto const discount_pred = + cudf::ast::operation(cudf::ast::ast_operator::LOGICAL_AND, discount_pred_a, discount_pred_b); + auto const quantity_pred = + cudf::ast::operation(cudf::ast::ast_operator::LESS, quantity_ref, quantity_upper_literal); + auto const discount_quantity_pred = + cudf::ast::operation(cudf::ast::ast_operator::LOGICAL_AND, discount_pred, quantity_pred); + auto const filtered_table = apply_filter(lineitem, discount_quantity_pred); + + // Calculate the `revenue` column + auto revenue = + calc_revenue(filtered_table->column("l_extendedprice"), filtered_table->column("l_discount")); + + // Sum the `revenue` column + auto const revenue_view = revenue->view(); + auto const result_table = apply_reduction(revenue_view, cudf::aggregation::Kind::SUM, "revenue"); + + timer.print_elapsed_millis(); + + // Write query result to a parquet file + result_table->to_parquet("q6.parquet"); + return 0; +} diff --git a/cpp/examples/tpch/q9.cpp b/cpp/examples/tpch/q9.cpp new file mode 100644 index 00000000000..d3c218253f9 --- /dev/null +++ b/cpp/examples/tpch/q9.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../utilities/timer.hpp" +#include "utils.hpp" + +#include +#include +#include +#include +#include + +/** + * @file q9.cpp + * @brief Implement query 9 of the TPC-H benchmark. + * + * create view part as select * from '/tables/scale-1/part.parquet'; + * create view supplier as select * from '/tables/scale-1/supplier.parquet'; + * create view lineitem as select * from '/tables/scale-1/lineitem.parquet'; + * create view partsupp as select * from '/tables/scale-1/partsupp.parquet'; + * create view orders as select * from '/tables/scale-1/orders.parquet'; + * create view nation as select * from '/tables/scale-1/nation.parquet'; + * + * select + * nation, + * o_year, + * sum(amount) as sum_profit + * from + * ( + * select + * n_name as nation, + * extract(year from o_orderdate) as o_year, + * l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount + * from + * part, + * supplier, + * lineitem, + * partsupp, + * orders, + * nation + * where + * s_suppkey = l_suppkey + * and ps_suppkey = l_suppkey + * and ps_partkey = l_partkey + * and p_partkey = l_partkey + * and o_orderkey = l_orderkey + * and s_nationkey = n_nationkey + * and p_name like '%green%' + * ) as profit + * group by + * nation, + * o_year + * order by + * nation, + * o_year desc; + */ + +/** + * @brief Calculate the amount column + * + * @param discount The discount column + * @param extendedprice The extended price column + * @param supplycost The supply cost column + * @param quantity The quantity column + * @param stream The CUDA stream used for device memory operations and kernel launches. + * @param mr Device memory resource used to allocate the returned column's device memory. + */ +[[nodiscard]] std::unique_ptr calc_amount( + cudf::column_view const& discount, + cudf::column_view const& extendedprice, + cudf::column_view const& supplycost, + cudf::column_view const& quantity, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource()) +{ + auto const one = cudf::numeric_scalar(1); + auto const one_minus_discount = + cudf::binary_operation(one, discount, cudf::binary_operator::SUB, discount.type()); + auto const extendedprice_discounted_type = cudf::data_type{cudf::type_id::FLOAT64}; + auto const extendedprice_discounted = cudf::binary_operation(extendedprice, + one_minus_discount->view(), + cudf::binary_operator::MUL, + extendedprice_discounted_type, + stream, + mr); + auto const supplycost_quantity_type = cudf::data_type{cudf::type_id::FLOAT64}; + auto const supplycost_quantity = cudf::binary_operation( + supplycost, quantity, cudf::binary_operator::MUL, supplycost_quantity_type); + auto amount = cudf::binary_operation(extendedprice_discounted->view(), + supplycost_quantity->view(), + cudf::binary_operator::SUB, + extendedprice_discounted->type(), + stream, + mr); + return amount; +} + +int main(int argc, char const** argv) +{ + auto const args = parse_args(argc, argv); + + // Use a memory pool + auto resource = create_memory_resource(args.memory_resource_type); + rmm::mr::set_current_device_resource(resource.get()); + + cudf::examples::timer timer; + + // Read out the table from parquet files + auto const lineitem = read_parquet( + args.dataset_dir + "/lineitem.parquet", + {"l_suppkey", "l_partkey", "l_orderkey", "l_extendedprice", "l_discount", "l_quantity"}); + auto const nation = read_parquet(args.dataset_dir + "/nation.parquet", {"n_nationkey", "n_name"}); + auto const orders = + read_parquet(args.dataset_dir + "/orders.parquet", {"o_orderkey", "o_orderdate"}); + auto const part = read_parquet(args.dataset_dir + "/part.parquet", {"p_partkey", "p_name"}); + auto const partsupp = read_parquet(args.dataset_dir + "/partsupp.parquet", + {"ps_suppkey", "ps_partkey", "ps_supplycost"}); + auto const supplier = + read_parquet(args.dataset_dir + "/supplier.parquet", {"s_suppkey", "s_nationkey"}); + + // Generating the `profit` table + // Filter the part table using `p_name like '%green%'` + auto const p_name = part->table().column(1); + auto const mask = + cudf::strings::like(cudf::strings_column_view(p_name), cudf::string_scalar("%green%")); + auto const part_filtered = apply_mask(part, mask); + + // Perform the joins + auto const join_a = apply_inner_join(supplier, nation, {"s_nationkey"}, {"n_nationkey"}); + auto const join_b = apply_inner_join(partsupp, join_a, {"ps_suppkey"}, {"s_suppkey"}); + auto const join_c = apply_inner_join(lineitem, part_filtered, {"l_partkey"}, {"p_partkey"}); + auto const join_d = apply_inner_join(orders, join_c, {"o_orderkey"}, {"l_orderkey"}); + auto const joined_table = + apply_inner_join(join_d, join_b, {"l_suppkey", "l_partkey"}, {"s_suppkey", "ps_partkey"}); + + // Calculate the `nation`, `o_year`, and `amount` columns + auto n_name = std::make_unique(joined_table->column("n_name")); + auto o_year = cudf::datetime::extract_year(joined_table->column("o_orderdate")); + auto amount = calc_amount(joined_table->column("l_discount"), + joined_table->column("l_extendedprice"), + joined_table->column("ps_supplycost"), + joined_table->column("l_quantity")); + + // Put together the `profit` table + std::vector> profit_columns; + profit_columns.push_back(std::move(n_name)); + profit_columns.push_back(std::move(o_year)); + profit_columns.push_back(std::move(amount)); + + auto profit_table = std::make_unique(std::move(profit_columns)); + auto const profit = std::make_unique( + std::move(profit_table), std::vector{"nation", "o_year", "amount"}); + + // Perform the groupby operation + auto const groupedby_table = apply_groupby( + profit, + groupby_context_t{{"nation", "o_year"}, + {{"amount", {{cudf::groupby_aggregation::SUM, "sum_profit"}}}}}); + + // Perform the orderby operation + auto const orderedby_table = apply_orderby( + groupedby_table, {"nation", "o_year"}, {cudf::order::ASCENDING, cudf::order::DESCENDING}); + + timer.print_elapsed_millis(); + + // Write query result to a parquet file + orderedby_table->to_parquet("q9.parquet"); + return 0; +} diff --git a/cpp/examples/tpch/utils.hpp b/cpp/examples/tpch/utils.hpp new file mode 100644 index 00000000000..e586da2c802 --- /dev/null +++ b/cpp/examples/tpch/utils.hpp @@ -0,0 +1,457 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +// RMM memory resource creation utilities +inline auto make_cuda() { return std::make_shared(); } +inline auto make_pool() +{ + return rmm::mr::make_owning_wrapper( + make_cuda(), rmm::percent_of_free_device_memory(50)); +} +inline auto make_managed() { return std::make_shared(); } +inline auto make_managed_pool() +{ + return rmm::mr::make_owning_wrapper( + make_managed(), rmm::percent_of_free_device_memory(50)); +} +inline std::shared_ptr create_memory_resource( + std::string const& mode) +{ + if (mode == "cuda") return make_cuda(); + if (mode == "pool") return make_pool(); + if (mode == "managed") return make_managed(); + if (mode == "managed_pool") return make_managed_pool(); + CUDF_FAIL("Unknown rmm_mode parameter: " + mode + + "\nExpecting: cuda, pool, managed, or managed_pool"); +} + +/** + * @brief A class to represent a table with column names attached + */ +class table_with_names { + public: + table_with_names(std::unique_ptr tbl, std::vector col_names) + : tbl(std::move(tbl)), col_names(col_names) + { + } + /** + * @brief Return the table view + */ + [[nodiscard]] cudf::table_view table() const { return tbl->view(); } + /** + * @brief Return the column view for a given column name + * + * @param col_name The name of the column + */ + [[nodiscard]] cudf::column_view column(std::string const& col_name) const + { + return tbl->view().column(col_id(col_name)); + } + /** + * @param Return the column names of the table + */ + [[nodiscard]] std::vector column_names() const { return col_names; } + /** + * @brief Translate a column name to a column index + * + * @param col_name The name of the column + */ + [[nodiscard]] cudf::size_type col_id(std::string const& col_name) const + { + CUDF_FUNC_RANGE(); + auto it = std::find(col_names.begin(), col_names.end(), col_name); + if (it == col_names.end()) { throw std::runtime_error("Column not found"); } + return std::distance(col_names.begin(), it); + } + /** + * @brief Append a column to the table + * + * @param col The column to append + * @param col_name The name of the appended column + */ + table_with_names& append(std::unique_ptr& col, std::string const& col_name) + { + CUDF_FUNC_RANGE(); + auto cols = tbl->release(); + cols.push_back(std::move(col)); + tbl = std::make_unique(std::move(cols)); + col_names.push_back(col_name); + return (*this); + } + /** + * @brief Select a subset of columns from the table + * + * @param col_names The names of the columns to select + */ + [[nodiscard]] cudf::table_view select(std::vector const& col_names) const + { + CUDF_FUNC_RANGE(); + std::vector col_indices; + for (auto const& col_name : col_names) { + col_indices.push_back(col_id(col_name)); + } + return tbl->select(col_indices); + } + /** + * @brief Write the table to a parquet file + * + * @param filepath The path to the parquet file + */ + void to_parquet(std::string const& filepath) const + { + CUDF_FUNC_RANGE(); + auto const sink_info = cudf::io::sink_info(filepath); + cudf::io::table_metadata metadata; + metadata.schema_info = + std::vector(col_names.begin(), col_names.end()); + auto const table_input_metadata = cudf::io::table_input_metadata{metadata}; + auto builder = cudf::io::parquet_writer_options::builder(sink_info, tbl->view()); + builder.metadata(table_input_metadata); + auto const options = builder.build(); + cudf::io::write_parquet(options); + } + + private: + std::unique_ptr tbl; + std::vector col_names; +}; + +/** + * @brief Concatenate two vectors + * + * @param lhs The left vector + * @param rhs The right vector + */ +template +std::vector concat(std::vector const& lhs, std::vector const& rhs) +{ + std::vector result; + result.reserve(lhs.size() + rhs.size()); + std::copy(lhs.begin(), lhs.end(), std::back_inserter(result)); + std::copy(rhs.begin(), rhs.end(), std::back_inserter(result)); + return result; +} + +/** + * @brief Inner join two tables and gather the result + * + * @param left_input The left input table + * @param right_input The right input table + * @param left_on The columns to join on in the left table + * @param right_on The columns to join on in the right table + * @param compare_nulls The null equality policy + */ +[[nodiscard]] std::unique_ptr join_and_gather( + cudf::table_view const& left_input, + cudf::table_view const& right_input, + std::vector const& left_on, + std::vector const& right_on, + cudf::null_equality compare_nulls) +{ + CUDF_FUNC_RANGE(); + constexpr auto oob_policy = cudf::out_of_bounds_policy::DONT_CHECK; + auto const left_selected = left_input.select(left_on); + auto const right_selected = right_input.select(right_on); + auto const [left_join_indices, right_join_indices] = cudf::inner_join( + left_selected, right_selected, compare_nulls, rmm::mr::get_current_device_resource()); + + auto const left_indices_span = cudf::device_span{*left_join_indices}; + auto const right_indices_span = cudf::device_span{*right_join_indices}; + + auto const left_indices_col = cudf::column_view{left_indices_span}; + auto const right_indices_col = cudf::column_view{right_indices_span}; + + auto const left_result = cudf::gather(left_input, left_indices_col, oob_policy); + auto const right_result = cudf::gather(right_input, right_indices_col, oob_policy); + + auto joined_cols = left_result->release(); + auto right_cols = right_result->release(); + joined_cols.insert(joined_cols.end(), + std::make_move_iterator(right_cols.begin()), + std::make_move_iterator(right_cols.end())); + return std::make_unique(std::move(joined_cols)); +} + +/** + * @brief Apply an inner join operation to two tables + * + * @param left_input The left input table + * @param right_input The right input table + * @param left_on The columns to join on in the left table + * @param right_on The columns to join on in the right table + * @param compare_nulls The null equality policy + */ +[[nodiscard]] std::unique_ptr apply_inner_join( + std::unique_ptr const& left_input, + std::unique_ptr const& right_input, + std::vector const& left_on, + std::vector const& right_on, + cudf::null_equality compare_nulls = cudf::null_equality::EQUAL) +{ + CUDF_FUNC_RANGE(); + std::vector left_on_indices; + std::vector right_on_indices; + std::transform( + left_on.begin(), left_on.end(), std::back_inserter(left_on_indices), [&](auto const& col_name) { + return left_input->col_id(col_name); + }); + std::transform(right_on.begin(), + right_on.end(), + std::back_inserter(right_on_indices), + [&](auto const& col_name) { return right_input->col_id(col_name); }); + auto table = join_and_gather( + left_input->table(), right_input->table(), left_on_indices, right_on_indices, compare_nulls); + return std::make_unique( + std::move(table), concat(left_input->column_names(), right_input->column_names())); +} + +/** + * @brief Apply a filter predicated to a table + * + * @param table The input table + * @param predicate The filter predicate + */ +[[nodiscard]] std::unique_ptr apply_filter( + std::unique_ptr const& table, cudf::ast::operation const& predicate) +{ + CUDF_FUNC_RANGE(); + auto const boolean_mask = cudf::compute_column(table->table(), predicate); + auto result_table = cudf::apply_boolean_mask(table->table(), boolean_mask->view()); + return std::make_unique(std::move(result_table), table->column_names()); +} + +/** + * @brief Apply a boolean mask to a table + * + * @param table The input table + * @param mask The boolean mask + */ +[[nodiscard]] std::unique_ptr apply_mask( + std::unique_ptr const& table, std::unique_ptr const& mask) +{ + CUDF_FUNC_RANGE(); + auto result_table = cudf::apply_boolean_mask(table->table(), mask->view()); + return std::make_unique(std::move(result_table), table->column_names()); +} + +struct groupby_context_t { + std::vector keys; + std::unordered_map>> + values; +}; + +/** + * @brief Apply a groupby operation to a table + * + * @param table The input table + * @param ctx The groupby context + */ +[[nodiscard]] std::unique_ptr apply_groupby( + std::unique_ptr const& table, groupby_context_t const& ctx) +{ + CUDF_FUNC_RANGE(); + auto const keys = table->select(ctx.keys); + cudf::groupby::groupby groupby_obj(keys); + std::vector result_column_names; + result_column_names.insert(result_column_names.end(), ctx.keys.begin(), ctx.keys.end()); + std::vector requests; + for (auto& [value_col, aggregations] : ctx.values) { + requests.emplace_back(cudf::groupby::aggregation_request()); + for (auto& agg : aggregations) { + if (agg.first == cudf::aggregation::Kind::SUM) { + requests.back().aggregations.push_back( + cudf::make_sum_aggregation()); + } else if (agg.first == cudf::aggregation::Kind::MEAN) { + requests.back().aggregations.push_back( + cudf::make_mean_aggregation()); + } else if (agg.first == cudf::aggregation::Kind::COUNT_ALL) { + requests.back().aggregations.push_back( + cudf::make_count_aggregation()); + } else { + throw std::runtime_error("Unsupported aggregation"); + } + result_column_names.push_back(agg.second); + } + requests.back().values = table->column(value_col); + } + auto agg_results = groupby_obj.aggregate(requests); + std::vector> result_columns; + for (size_t i = 0; i < agg_results.first->num_columns(); i++) { + auto col = std::make_unique(agg_results.first->get_column(i)); + result_columns.push_back(std::move(col)); + } + for (size_t i = 0; i < agg_results.second.size(); i++) { + for (size_t j = 0; j < agg_results.second[i].results.size(); j++) { + result_columns.push_back(std::move(agg_results.second[i].results[j])); + } + } + auto result_table = std::make_unique(std::move(result_columns)); + return std::make_unique(std::move(result_table), result_column_names); +} + +/** + * @brief Apply an order by operation to a table + * + * @param table The input table + * @param sort_keys The sort keys + * @param sort_key_orders The sort key orders + */ +[[nodiscard]] std::unique_ptr apply_orderby( + std::unique_ptr const& table, + std::vector const& sort_keys, + std::vector const& sort_key_orders) +{ + CUDF_FUNC_RANGE(); + std::vector column_views; + for (auto& key : sort_keys) { + column_views.push_back(table->column(key)); + } + auto result_table = + cudf::sort_by_key(table->table(), cudf::table_view{column_views}, sort_key_orders); + return std::make_unique(std::move(result_table), table->column_names()); +} + +/** + * @brief Apply a reduction operation to a column + * + * @param column The input column + * @param agg_kind The aggregation kind + * @param col_name The name of the output column + */ +[[nodiscard]] std::unique_ptr apply_reduction( + cudf::column_view const& column, + cudf::aggregation::Kind const& agg_kind, + std::string const& col_name) +{ + CUDF_FUNC_RANGE(); + auto const agg = cudf::make_sum_aggregation(); + auto const result = cudf::reduce(column, *agg, column.type()); + cudf::size_type const len = 1; + auto col = cudf::make_column_from_scalar(*result, len); + std::vector> columns; + columns.push_back(std::move(col)); + auto result_table = std::make_unique(std::move(columns)); + std::vector col_names = {col_name}; + return std::make_unique(std::move(result_table), col_names); +} + +/** + * @brief Read a parquet file into a table + * + * @param filename The path to the parquet file + * @param columns The columns to read + * @param predicate The filter predicate to pushdown + */ +[[nodiscard]] std::unique_ptr read_parquet( + std::string const& filename, + std::vector const& columns = {}, + std::unique_ptr const& predicate = nullptr) +{ + CUDF_FUNC_RANGE(); + auto const source = cudf::io::source_info(filename); + auto builder = cudf::io::parquet_reader_options_builder(source); + if (!columns.empty()) { builder.columns(columns); } + if (predicate) { builder.filter(*predicate); } + auto const options = builder.build(); + auto table_with_metadata = cudf::io::read_parquet(options); + std::vector column_names; + for (auto const& col_info : table_with_metadata.metadata.schema_info) { + column_names.push_back(col_info.name); + } + return std::make_unique(std::move(table_with_metadata.tbl), column_names); +} + +/** + * @brief Generate the `std::tm` structure from year, month, and day + * + * @param year The year + * @param month The month + * @param day The day + */ +std::tm make_tm(int year, int month, int day) +{ + std::tm tm{}; + tm.tm_year = year - 1900; + tm.tm_mon = month - 1; + tm.tm_mday = day; + return tm; +} + +/** + * @brief Calculate the number of days since the UNIX epoch + * + * @param year The year + * @param month The month + * @param day The day + */ +int32_t days_since_epoch(int year, int month, int day) +{ + std::tm tm = make_tm(year, month, day); + std::tm epoch = make_tm(1970, 1, 1); + std::time_t time = std::mktime(&tm); + std::time_t epoch_time = std::mktime(&epoch); + double diff = std::difftime(time, epoch_time) / (60 * 60 * 24); + return static_cast(diff); +} + +struct tpch_example_args { + std::string dataset_dir; + std::string memory_resource_type; +}; + +/** + * @brief Parse command line arguments into a struct + * + * @param argc The number of command line arguments + * @param argv The command line arguments + */ +tpch_example_args parse_args(int argc, char const** argv) +{ + if (argc < 3) { + std::string usage_message = "Usage: " + std::string(argv[0]) + + " \n The query result will be " + "saved to a parquet file named q{query_no}.parquet in the current " + "working directory "; + throw std::runtime_error(usage_message); + } + tpch_example_args args; + args.dataset_dir = argv[1]; + args.memory_resource_type = argv[2]; + return args; +} diff --git a/cpp/examples/utilities/timer.hpp b/cpp/examples/utilities/timer.hpp new file mode 100644 index 00000000000..65fa92e74cf --- /dev/null +++ b/cpp/examples/utilities/timer.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +namespace cudf { +namespace examples { +/** + * @brief Light-weight timer for measuring elapsed time. + * + * A timer object constructed from std::chrono, instrumenting at microseconds + * precision. Can display elapsed durations at milli and micro second + * scales. The timer starts at object construction. + */ +class timer { + public: + using micros = std::chrono::microseconds; + using millis = std::chrono::milliseconds; + + timer() { reset(); } + void reset() { start_time = std::chrono::high_resolution_clock::now(); } + auto elapsed() const { return (std::chrono::high_resolution_clock::now() - start_time); } + void print_elapsed_micros() const + { + std::cout << "Elapsed Time: " << std::chrono::duration_cast(elapsed()).count() + << "us\n\n"; + } + void print_elapsed_millis() const + { + std::cout << "Elapsed Time: " << std::chrono::duration_cast(elapsed()).count() + << "ms\n\n"; + } + + private: + using time_point_t = std::chrono::time_point; + time_point_t start_time; +}; + +} // namespace examples +}; // namespace cudf From c4471c4ee81ed967f1818bc03c5f7829b15cfe56 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:44:04 -0400 Subject: [PATCH 26/53] Fix split_record for all empty strings column (#16291) Fixes `cudf::strings::split_record` handling of an all empty strings column. This caused a kernel launch with no threads eventually reporting a CUDA error. A new gtest was added to check this condition and includes tests for `rsplit_record` as well. Closes #16284 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/16291 --- cpp/src/strings/split/split.cuh | 6 ++++++ cpp/tests/strings/split_tests.cpp | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/cpp/src/strings/split/split.cuh b/cpp/src/strings/split/split.cuh index 23614ac0733..4d7096c02ca 100644 --- a/cpp/src/strings/split/split.cuh +++ b/cpp/src/strings/split/split.cuh @@ -357,6 +357,12 @@ std::pair, rmm::device_uvector> split auto const chars_bytes = get_offset_value(input.offsets(), input.offset() + strings_count, stream) - get_offset_value(input.offsets(), input.offset(), stream); + if (chars_bytes == 0) { + auto offsets = cudf::make_column_from_scalar( + numeric_scalar(0, true, stream), strings_count + 1, stream, mr); + auto tokens = rmm::device_uvector(0, stream); + return std::pair{std::move(offsets), std::move(tokens)}; + } auto const d_offsets = cudf::detail::offsetalator_factory::make_input_iterator(input.offsets(), input.offset()); diff --git a/cpp/tests/strings/split_tests.cpp b/cpp/tests/strings/split_tests.cpp index d53c64ed539..4c020cb4c29 100644 --- a/cpp/tests/strings/split_tests.cpp +++ b/cpp/tests/strings/split_tests.cpp @@ -307,6 +307,26 @@ TEST_F(StringsSplitTest, SplitRecordWhitespaceWithMaxSplit) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); } +TEST_F(StringsSplitTest, SplitRecordAllEmpty) +{ + auto input = cudf::test::strings_column_wrapper({"", "", "", ""}); + auto sv = cudf::strings_column_view(input); + auto delimiter = cudf::string_scalar("s"); + auto empty = cudf::string_scalar(""); + + using LCW = cudf::test::lists_column_wrapper; + LCW expected({LCW{}, LCW{}, LCW{}, LCW{}}); + auto result = cudf::strings::split_record(sv, delimiter); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + result = cudf::strings::split_record(sv, empty); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + + result = cudf::strings::rsplit_record(sv, delimiter); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + result = cudf::strings::rsplit_record(sv, empty); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); +} + TEST_F(StringsSplitTest, MultiByteDelimiters) { // Overlapping delimiters From faddc8c3d37e5cf8ec69341118218c245e087c26 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 18 Jul 2024 08:03:55 -0700 Subject: [PATCH 27/53] Migrate CSV reader to pylibcudf (#16011) xref #15162 Authors: - Thomas Li (https://github.com/lithomas1) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16011 --- .../user_guide/api_docs/pylibcudf/io/csv.rst | 6 + .../api_docs/pylibcudf/io/index.rst | 1 + python/cudf/cudf/_lib/csv.pyx | 436 ++++++------------ .../cudf/_lib/pylibcudf/io/CMakeLists.txt | 6 +- .../cudf/cudf/_lib/pylibcudf/io/__init__.pxd | 1 + .../cudf/cudf/_lib/pylibcudf/io/__init__.py | 2 +- python/cudf/cudf/_lib/pylibcudf/io/csv.pyx | 264 +++++++++++ python/cudf/cudf/_lib/pylibcudf/io/types.pxd | 3 + python/cudf/cudf/_lib/pylibcudf/io/types.pyx | 2 +- python/cudf/cudf/_lib/types.pyx | 3 + .../cudf/cudf/pylibcudf_tests/common/utils.py | 43 +- python/cudf/cudf/pylibcudf_tests/conftest.py | 14 + .../cudf/cudf/pylibcudf_tests/io/test_csv.py | 280 +++++++++++ 13 files changed, 751 insertions(+), 310 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/io/csv.rst create mode 100644 python/cudf/cudf/_lib/pylibcudf/io/csv.pyx create mode 100644 python/cudf/cudf/pylibcudf_tests/io/test_csv.py diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/csv.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/csv.rst new file mode 100644 index 00000000000..5a2276f8b2d --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/csv.rst @@ -0,0 +1,6 @@ +=== +CSV +=== + +.. automodule:: cudf._lib.pylibcudf.io.csv + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst index bde6d8094ce..697bce739de 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst @@ -16,4 +16,5 @@ I/O Functions :maxdepth: 1 avro + csv json diff --git a/python/cudf/cudf/_lib/csv.pyx b/python/cudf/cudf/_lib/csv.pyx index 9fecff5f5f6..099b61d62ae 100644 --- a/python/cudf/cudf/_lib/csv.pyx +++ b/python/cudf/cudf/_lib/csv.pyx @@ -1,7 +1,6 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libcpp cimport bool -from libcpp.map cimport map from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move @@ -9,8 +8,12 @@ from libcpp.vector cimport vector cimport cudf._lib.pylibcudf.libcudf.types as libcudf_types from cudf._lib.pylibcudf.io.datasource cimport Datasource, NativeFileDatasource -from cudf._lib.pylibcudf.libcudf.types cimport data_type -from cudf._lib.types cimport dtype_to_data_type +from cudf._lib.types cimport dtype_to_pylibcudf_type + +import errno +import os +from collections import abc +from io import BytesIO, StringIO import numpy as np import pandas as pd @@ -18,65 +21,24 @@ import pandas as pd import cudf from cudf.core.buffer import acquire_spill_lock -from cudf._lib.pylibcudf.libcudf.types cimport size_type - -import errno -import os -from collections import abc -from enum import IntEnum -from io import BytesIO, StringIO - -from libc.stdint cimport int32_t from libcpp cimport bool -from cudf._lib.io.utils cimport make_sink_info, make_source_info +from cudf._lib.io.utils cimport make_sink_info from cudf._lib.pylibcudf.libcudf.io.csv cimport ( - csv_reader_options, csv_writer_options, - read_csv as cpp_read_csv, write_csv as cpp_write_csv, ) from cudf._lib.pylibcudf.libcudf.io.data_sink cimport data_sink -from cudf._lib.pylibcudf.libcudf.io.types cimport ( - compression_type, - quote_style, - sink_info, - source_info, - table_with_metadata, -) +from cudf._lib.pylibcudf.libcudf.io.types cimport compression_type, sink_info from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.utils cimport data_from_unique_ptr, table_view_from_table +from cudf._lib.utils cimport data_from_pylibcudf_io, table_view_from_table from pyarrow.lib import NativeFile +import cudf._lib.pylibcudf as plc from cudf.api.types import is_hashable -ctypedef int32_t underlying_type_t_compression - - -class Compression(IntEnum): - INFER = ( - compression_type.AUTO - ) - SNAPPY = ( - compression_type.SNAPPY - ) - GZIP = ( - compression_type.GZIP - ) - BZ2 = ( - compression_type.BZIP2 - ) - BROTLI = ( - compression_type.BROTLI - ) - ZIP = ( - compression_type.ZIP - ) - XZ = ( - compression_type.XZ - ) - +from cudf._lib.pylibcudf.types cimport DataType CSV_HEX_TYPE_MAP = { "hex": np.dtype("int64"), @@ -84,234 +46,6 @@ CSV_HEX_TYPE_MAP = { "hex32": np.dtype("int32") } -cdef csv_reader_options make_csv_reader_options( - object datasource, - object lineterminator, - object quotechar, - int quoting, - bool doublequote, - object header, - bool mangle_dupe_cols, - object usecols, - object delimiter, - bool delim_whitespace, - bool skipinitialspace, - object names, - object dtype, - int skipfooter, - int skiprows, - bool dayfirst, - object compression, - object thousands, - object decimal, - object true_values, - object false_values, - object nrows, - object byte_range, - bool skip_blank_lines, - object parse_dates, - object comment, - object na_values, - bool keep_default_na, - bool na_filter, - object prefix, - object index_col, -) except *: - cdef source_info c_source_info = make_source_info([datasource]) - cdef compression_type c_compression - cdef vector[string] c_names - cdef size_t c_byte_range_offset = ( - byte_range[0] if byte_range is not None else 0 - ) - cdef size_t c_byte_range_size = ( - byte_range[1] if byte_range is not None else 0 - ) - cdef vector[int] c_use_cols_indexes - cdef vector[string] c_use_cols_names - cdef size_type c_nrows = nrows if nrows is not None else -1 - cdef quote_style c_quoting - cdef vector[string] c_parse_dates_names - cdef vector[int] c_parse_dates_indexes - cdef vector[string] c_hex_col_names - cdef vector[data_type] c_dtypes_list - cdef map[string, data_type] c_dtypes_map - cdef vector[int] c_hex_col_indexes - cdef vector[string] c_true_values - cdef vector[string] c_false_values - cdef vector[string] c_na_values - - # Reader settings - if compression is None: - c_compression = compression_type.NONE - else: - compression = str(compression) - compression = Compression[compression.upper()] - c_compression = ( - compression - ) - - if quoting == 1: - c_quoting = quote_style.ALL - elif quoting == 2: - c_quoting = quote_style.NONNUMERIC - elif quoting == 3: - c_quoting = quote_style.NONE - else: - # Default value - c_quoting = quote_style.MINIMAL - - cdef csv_reader_options csv_reader_options_c = move( - csv_reader_options.builder(c_source_info) - .compression(c_compression) - .mangle_dupe_cols(mangle_dupe_cols) - .byte_range_offset(c_byte_range_offset) - .byte_range_size(c_byte_range_size) - .nrows(c_nrows) - .skiprows(skiprows) - .skipfooter(skipfooter) - .quoting(c_quoting) - .lineterminator(ord(lineterminator)) - .quotechar(ord(quotechar)) - .decimal(ord(decimal)) - .delim_whitespace(delim_whitespace) - .skipinitialspace(skipinitialspace) - .skip_blank_lines(skip_blank_lines) - .doublequote(doublequote) - .keep_default_na(keep_default_na) - .na_filter(na_filter) - .dayfirst(dayfirst) - .build() - ) - - if names is not None: - # explicitly mentioned name, so don't check header - if header is None or header == 'infer': - csv_reader_options_c.set_header(-1) - else: - csv_reader_options_c.set_header(header) - - c_names.reserve(len(names)) - for name in names: - c_names.push_back(str(name).encode()) - csv_reader_options_c.set_names(c_names) - else: - if header is None: - csv_reader_options_c.set_header(-1) - elif header == 'infer': - csv_reader_options_c.set_header(0) - else: - csv_reader_options_c.set_header(header) - - if prefix is not None: - csv_reader_options_c.set_prefix(prefix.encode()) - - if usecols is not None: - all_int = all(isinstance(col, int) for col in usecols) - if all_int: - c_use_cols_indexes.reserve(len(usecols)) - c_use_cols_indexes = usecols - csv_reader_options_c.set_use_cols_indexes(c_use_cols_indexes) - else: - c_use_cols_names.reserve(len(usecols)) - for col_name in usecols: - c_use_cols_names.push_back( - str(col_name).encode() - ) - csv_reader_options_c.set_use_cols_names(c_use_cols_names) - - if delimiter is not None: - csv_reader_options_c.set_delimiter(ord(delimiter)) - - if thousands is not None: - csv_reader_options_c.set_thousands(ord(thousands)) - - if comment is not None: - csv_reader_options_c.set_comment(ord(comment)) - - if parse_dates is not None: - if isinstance(parse_dates, abc.Mapping): - raise NotImplementedError( - "`parse_dates`: dictionaries are unsupported") - if not isinstance(parse_dates, abc.Iterable): - raise NotImplementedError( - "`parse_dates`: an iterable is required") - for col in parse_dates: - if isinstance(col, str): - c_parse_dates_names.push_back(str(col).encode()) - elif isinstance(col, int): - c_parse_dates_indexes.push_back(col) - else: - raise NotImplementedError( - "`parse_dates`: Nesting is unsupported") - csv_reader_options_c.set_parse_dates(c_parse_dates_names) - csv_reader_options_c.set_parse_dates(c_parse_dates_indexes) - - if dtype is not None: - if isinstance(dtype, abc.Mapping): - for k, v in dtype.items(): - col_type = v - if is_hashable(v) and v in CSV_HEX_TYPE_MAP: - col_type = CSV_HEX_TYPE_MAP[v] - c_hex_col_names.push_back(str(k).encode()) - - c_dtypes_map[str(k).encode()] = \ - _get_cudf_data_type_from_dtype( - cudf.dtype(col_type)) - csv_reader_options_c.set_dtypes(c_dtypes_map) - csv_reader_options_c.set_parse_hex(c_hex_col_names) - elif ( - cudf.api.types.is_scalar(dtype) or - isinstance(dtype, ( - np.dtype, pd.api.extensions.ExtensionDtype, type - )) - ): - c_dtypes_list.reserve(1) - if is_hashable(dtype) and dtype in CSV_HEX_TYPE_MAP: - dtype = CSV_HEX_TYPE_MAP[dtype] - c_hex_col_indexes.push_back(0) - - c_dtypes_list.push_back( - _get_cudf_data_type_from_dtype(dtype) - ) - csv_reader_options_c.set_dtypes(c_dtypes_list) - csv_reader_options_c.set_parse_hex(c_hex_col_indexes) - elif isinstance(dtype, abc.Collection): - c_dtypes_list.reserve(len(dtype)) - for index, col_dtype in enumerate(dtype): - if is_hashable(col_dtype) and col_dtype in CSV_HEX_TYPE_MAP: - col_dtype = CSV_HEX_TYPE_MAP[col_dtype] - c_hex_col_indexes.push_back(index) - - c_dtypes_list.push_back( - _get_cudf_data_type_from_dtype(col_dtype) - ) - csv_reader_options_c.set_dtypes(c_dtypes_list) - csv_reader_options_c.set_parse_hex(c_hex_col_indexes) - else: - raise ValueError( - "dtype should be a scalar/str/list-like/dict-like" - ) - - if true_values is not None: - c_true_values.reserve(len(true_values)) - for tv in true_values: - c_true_values.push_back(tv.encode()) - csv_reader_options_c.set_true_values(c_true_values) - - if false_values is not None: - c_false_values.reserve(len(false_values)) - for fv in false_values: - c_false_values.push_back(fv.encode()) - csv_reader_options_c.set_false_values(c_false_values) - - if na_values is not None: - c_na_values.reserve(len(na_values)) - for nv in na_values: - c_na_values.push_back(nv.encode()) - csv_reader_options_c.set_na_values(c_na_values) - - return csv_reader_options_c - def validate_args( object delimiter, @@ -381,7 +115,6 @@ def read_csv( bool na_filter=True, object prefix=None, object index_col=None, - **kwargs, ): """ Cython function to call into libcudf API, see `read_csv`. @@ -413,23 +146,120 @@ def read_csv( if delimiter is None: delimiter = sep - cdef csv_reader_options read_csv_options_c = make_csv_reader_options( - datasource, lineterminator, quotechar, quoting, doublequote, - header, mangle_dupe_cols, usecols, delimiter, delim_whitespace, - skipinitialspace, names, dtype, skipfooter, skiprows, dayfirst, - compression, thousands, decimal, true_values, false_values, nrows, - byte_range, skip_blank_lines, parse_dates, comment, na_values, - keep_default_na, na_filter, prefix, index_col) + delimiter = str(delimiter) + + if byte_range is None: + byte_range = (0, 0) + + if compression is None: + c_compression = compression_type.NONE + else: + compression_map = { + "infer": compression_type.AUTO, + "gzip": compression_type.GZIP, + "bz2": compression_type.BZIP2, + "zip": compression_type.ZIP, + } + c_compression = compression_map[compression] - cdef table_with_metadata c_result - with nogil: - c_result = move(cpp_read_csv(read_csv_options_c)) + # We need this later when setting index cols + orig_header = header + + if names is not None: + # explicitly mentioned name, so don't check header + if header is None or header == 'infer': + header = -1 + else: + header = header + names = list(names) + else: + if header is None: + header = -1 + elif header == 'infer': + header = 0 - meta_names = [info.name.decode() for info in c_result.metadata.schema_info] - df = cudf.DataFrame._from_data(*data_from_unique_ptr( - move(c_result.tbl), - column_names=meta_names - )) + hex_cols = [] + + new_dtypes = [] + if dtype is not None: + if isinstance(dtype, abc.Mapping): + new_dtypes = dict() + for k, v in dtype.items(): + col_type = v + if is_hashable(v) and v in CSV_HEX_TYPE_MAP: + col_type = CSV_HEX_TYPE_MAP[v] + hex_cols.append(str(k)) + + new_dtypes[k] = _get_plc_data_type_from_dtype( + cudf.dtype(col_type) + ) + elif ( + cudf.api.types.is_scalar(dtype) or + isinstance(dtype, ( + np.dtype, pd.api.extensions.ExtensionDtype, type + )) + ): + if is_hashable(dtype) and dtype in CSV_HEX_TYPE_MAP: + dtype = CSV_HEX_TYPE_MAP[dtype] + hex_cols.append(0) + + new_dtypes.append( + _get_plc_data_type_from_dtype(dtype) + ) + elif isinstance(dtype, abc.Collection): + for index, col_dtype in enumerate(dtype): + if is_hashable(col_dtype) and col_dtype in CSV_HEX_TYPE_MAP: + col_dtype = CSV_HEX_TYPE_MAP[col_dtype] + hex_cols.append(index) + + new_dtypes.append( + _get_plc_data_type_from_dtype(col_dtype) + ) + else: + raise ValueError( + "dtype should be a scalar/str/list-like/dict-like" + ) + + lineterminator = str(lineterminator) + + df = cudf.DataFrame._from_data( + *data_from_pylibcudf_io( + plc.io.csv.read_csv( + plc.io.SourceInfo([datasource]), + lineterminator=lineterminator, + quotechar = quotechar, + quoting = quoting, + doublequote = doublequote, + header = header, + mangle_dupe_cols = mangle_dupe_cols, + usecols = usecols, + delimiter = delimiter, + delim_whitespace = delim_whitespace, + skipinitialspace = skipinitialspace, + col_names = names, + dtypes = new_dtypes, + skipfooter = skipfooter, + skiprows = skiprows, + dayfirst = dayfirst, + compression = c_compression, + thousands = thousands, + decimal = decimal, + true_values = true_values, + false_values = false_values, + nrows = nrows if nrows is not None else -1, + byte_range_offset = byte_range[0], + byte_range_size = byte_range[1], + skip_blank_lines = skip_blank_lines, + parse_dates = parse_dates, + parse_hex = hex_cols, + comment = comment, + na_values = na_values, + keep_default_na = keep_default_na, + na_filter = na_filter, + prefix = prefix, + ) + ) + ) if dtype is not None: if isinstance(dtype, abc.Mapping): @@ -459,7 +289,7 @@ def read_csv( index_col_name = df._data.select_by_index(index_col).names[0] df = df.set_index(index_col_name) if isinstance(index_col_name, str) and \ - names is None and header in ("infer",): + names is None and orig_header == "infer": if index_col_name.startswith("Unnamed:"): # TODO: Try to upstream it to libcudf # csv reader in future @@ -550,7 +380,7 @@ def write_csv( ) -cdef data_type _get_cudf_data_type_from_dtype(object dtype) except *: +cdef DataType _get_plc_data_type_from_dtype(object dtype) except *: # TODO: Remove this work-around Dictionary types # in libcudf are fully mapped to categorical columns: # https://github.com/rapidsai/cudf/issues/3960 @@ -561,36 +391,36 @@ cdef data_type _get_cudf_data_type_from_dtype(object dtype) except *: if isinstance(dtype, str): if str(dtype) == "date32": - return libcudf_types.data_type( + return DataType( libcudf_types.type_id.TIMESTAMP_DAYS ) elif str(dtype) in ("date", "date64"): - return libcudf_types.data_type( + return DataType( libcudf_types.type_id.TIMESTAMP_MILLISECONDS ) elif str(dtype) == "timestamp": - return libcudf_types.data_type( + return DataType( libcudf_types.type_id.TIMESTAMP_MILLISECONDS ) elif str(dtype) == "timestamp[us]": - return libcudf_types.data_type( + return DataType( libcudf_types.type_id.TIMESTAMP_MICROSECONDS ) elif str(dtype) == "timestamp[s]": - return libcudf_types.data_type( + return DataType( libcudf_types.type_id.TIMESTAMP_SECONDS ) elif str(dtype) == "timestamp[ms]": - return libcudf_types.data_type( + return DataType( libcudf_types.type_id.TIMESTAMP_MILLISECONDS ) elif str(dtype) == "timestamp[ns]": - return libcudf_types.data_type( + return DataType( libcudf_types.type_id.TIMESTAMP_NANOSECONDS ) dtype = cudf.dtype(dtype) - return dtype_to_data_type(dtype) + return dtype_to_pylibcudf_type(dtype) def columns_apply_na_rep(column_names, na_rep): diff --git a/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt index 084b341ec48..8dd08d11dc8 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt @@ -12,7 +12,7 @@ # the License. # ============================================================================= -set(cython_sources avro.pyx datasource.pyx json.pyx types.pyx) +set(cython_sources avro.pyx csv.pyx datasource.pyx json.pyx types.pyx) set(linked_libraries cudf::cudf) rapids_cython_create_modules( @@ -21,7 +21,7 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX pylibcudf_io_ ASSOCIATED_TARGETS cudf ) -set(targets_using_arrow_headers pylibcudf_io_avro pylibcudf_io_datasource pylibcudf_io_json - pylibcudf_io_types +set(targets_using_arrow_headers pylibcudf_io_avro pylibcudf_io_csv pylibcudf_io_datasource + pylibcudf_io_json pylibcudf_io_types ) link_to_pyarrow_headers("${targets_using_arrow_headers}") diff --git a/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd index ef4c65b277e..5b3272d60e0 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd @@ -1,4 +1,5 @@ # Copyright (c) 2024, NVIDIA CORPORATION. +# CSV is removed since it is def not cpdef (to force kw-only arguments) from . cimport avro, datasource, json, types from .types cimport SourceInfo, TableWithMetadata diff --git a/python/cudf/cudf/_lib/pylibcudf/io/__init__.py b/python/cudf/cudf/_lib/pylibcudf/io/__init__.py index fb4e4c7e4bb..e17deaa4663 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/io/__init__.py @@ -1,4 +1,4 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from . import avro, datasource, json, types +from . import avro, csv, datasource, json, types from .types import SinkInfo, SourceInfo, TableWithMetadata diff --git a/python/cudf/cudf/_lib/pylibcudf/io/csv.pyx b/python/cudf/cudf/_lib/pylibcudf/io/csv.pyx new file mode 100644 index 00000000000..e9efb5befee --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/io/csv.pyx @@ -0,0 +1,264 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp cimport bool +from libcpp.map cimport map +from libcpp.string cimport string +from libcpp.utility cimport move +from libcpp.vector cimport vector + +from cudf._lib.pylibcudf.io.types cimport SourceInfo, TableWithMetadata +from cudf._lib.pylibcudf.libcudf.io.csv cimport ( + csv_reader_options, + read_csv as cpp_read_csv, +) +from cudf._lib.pylibcudf.libcudf.io.types cimport ( + compression_type, + quote_style, + table_with_metadata, +) +from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type +from cudf._lib.pylibcudf.types cimport DataType + + +cdef tuple _process_parse_dates_hex(list cols): + cdef vector[string] str_cols + cdef vector[int] int_cols + for col in cols: + if isinstance(col, str): + str_cols.push_back(col.encode()) + else: + int_cols.push_back(col) + return str_cols, int_cols + +cdef vector[string] _make_str_vector(list vals): + cdef vector[string] res + for val in vals: + res.push_back((val).encode()) + return res + + +def read_csv( + SourceInfo source_info, + *, + compression_type compression = compression_type.AUTO, + size_t byte_range_offset = 0, + size_t byte_range_size = 0, + list col_names = None, + str prefix = "", + bool mangle_dupe_cols = True, + list usecols = None, + size_type nrows = -1, + size_type skiprows = 0, + size_type skipfooter = 0, + size_type header = 0, + str lineterminator = "\n", + str delimiter = None, + str thousands = None, + str decimal = ".", + str comment = None, + bool delim_whitespace = False, + bool skipinitialspace = False, + bool skip_blank_lines = True, + quote_style quoting = quote_style.MINIMAL, + str quotechar = '"', + bool doublequote = True, + list parse_dates = None, + list parse_hex = None, + # Technically this should be dict/list + # but using a fused type prevents using None as default + object dtypes = None, + list true_values = None, + list false_values = None, + list na_values = None, + bool keep_default_na = True, + bool na_filter = True, + bool dayfirst = False, + # Note: These options are supported by the libcudf reader + # but are not exposed here since there is no demand for them + # on the Python side yet. + # bool detect_whitespace_around_quotes = False, + # DataType timestamp_type = DataType(type_id.EMPTY), +): + """Reads a CSV file into a :py:class:`~.types.TableWithMetadata`. + + Parameters + ---------- + source_info : SourceInfo + The SourceInfo to read the CSV file from. + compression : compression_type, default CompressionType.AUTO + The compression format of the CSV source. + byte_range_offset : size_type, default 0 + Number of bytes to skip from source start. + byte_range_size : size_type, default 0 + Number of bytes to read. By default, will read all bytes. + col_names : list, default None + The column names to use. + prefix : string, default '' + The prefix to apply to the column names. + mangle_dupe_cols : bool, default True + If True, rename duplicate column names. + usecols : list, default None + Specify the string column names/integer column indices of columns to be read. + nrows : size_type, default -1 + The number of rows to read. + skiprows : size_type, default 0 + The number of rows to skip from the start before reading + skipfooter : size_type, default 0 + The number of rows to skip from the end + header : size_type, default 0 + The index of the row that will be used for header names. + Pass -1 to use default column names. + lineterminator : str, default '\\n' + The character used to determine the end of a line. + delimiter : str, default "," + The character used to separate fields in a row. + thousands : str, default None + The character used as the thousands separator. + Cannot match delimiter. + decimal : str, default '.' + The character used as the decimal separator. + Cannot match delimiter. + comment : str, default None + The character used to identify the start of a comment line. + (which will be skipped by the reader) + delim_whitespace : bool, default False + If True, treat whitespace as the field delimiter. + skipinitialspace : bool, default False + If True, skip whitespace after the delimiter. + skip_blank_lines : bool, default True + If True, ignore empty lines (otherwise line values are parsed as null). + quoting : QuoteStyle, default QuoteStyle.MINIMAL + The quoting style used in the input CSV data. One of + { QuoteStyle.MINIMAL, QuoteStyle.ALL, QuoteStyle.NONNUMERIC, QuoteStyle.NONE } + quotechar : str, default '"' + The character used to indicate quoting. + doublequote : bool, default True + If True, a quote inside a value is double-quoted. + parse_dates : list, default None + A list of integer column indices/string column names + of columns to read as datetime. + parse_hex : list, default None + A list of integer column indices/string column names + of columns to read as hexadecimal. + dtypes : Union[Dict[str, DataType], List[DataType]], default None + A list of data types or a dictionary mapping column names + to a DataType. + true_values : List[str], default None + A list of additional values to recognize as True. + false_values : List[str], default None + A list of additional values to recognize as False. + na_values : List[str], default None + A list of additional values to recognize as null. + keep_default_na : bool, default True + Whether to keep the built-in default N/A values. + na_filter : bool, default True + Whether to detect missing values. If False, can + improve performance. + dayfirst : bool, default False + If True, interpret dates as being in the DD/MM format. + + Returns + ------- + TableWithMetadata + The Table and its corresponding metadata (column names) that were read in. + """ + cdef vector[string] c_parse_dates_names + cdef vector[int] c_parse_dates_indexes + cdef vector[int] c_parse_hex_names + cdef vector[int] c_parse_hex_indexes + cdef vector[data_type] c_dtypes_list + cdef map[string, data_type] c_dtypes_map + + cdef csv_reader_options options = move( + csv_reader_options.builder(source_info.c_obj) + .compression(compression) + .mangle_dupe_cols(mangle_dupe_cols) + .byte_range_offset(byte_range_offset) + .byte_range_size(byte_range_size) + .nrows(nrows) + .skiprows(skiprows) + .skipfooter(skipfooter) + .quoting(quoting) + .lineterminator(ord(lineterminator)) + .quotechar(ord(quotechar)) + .decimal(ord(decimal)) + .delim_whitespace(delim_whitespace) + .skipinitialspace(skipinitialspace) + .skip_blank_lines(skip_blank_lines) + .doublequote(doublequote) + .keep_default_na(keep_default_na) + .na_filter(na_filter) + .dayfirst(dayfirst) + .build() + ) + + options.set_header(header) + + if col_names is not None: + options.set_names([str(name).encode() for name in col_names]) + + if prefix is not None: + options.set_prefix(prefix.encode()) + + if usecols is not None: + if all([isinstance(col, int) for col in usecols]): + options.set_use_cols_indexes(list(usecols)) + else: + options.set_use_cols_names([str(name).encode() for name in usecols]) + + if delimiter is not None: + options.set_delimiter(ord(delimiter)) + + if thousands is not None: + options.set_thousands(ord(thousands)) + + if comment is not None: + options.set_comment(ord(comment)) + + if parse_dates is not None: + if not all([isinstance(col, (str, int)) for col in parse_dates]): + raise NotImplementedError( + "`parse_dates`: Must pass a list of column names/indices") + + # Set both since users are allowed to mix column names and indices + c_parse_dates_names, c_parse_dates_indexes = \ + _process_parse_dates_hex(parse_dates) + options.set_parse_dates(c_parse_dates_names) + options.set_parse_dates(c_parse_dates_indexes) + + if parse_hex is not None: + if not all([isinstance(col, (str, int)) for col in parse_hex]): + raise NotImplementedError( + "`parse_hex`: Must pass a list of column names/indices") + + # Set both since users are allowed to mix column names and indices + c_parse_hex_names, c_parse_hex_indexes = _process_parse_dates_hex(parse_hex) + options.set_parse_hex(c_parse_hex_names) + options.set_parse_hex(c_parse_hex_indexes) + + if isinstance(dtypes, list): + for dtype in dtypes: + c_dtypes_list.push_back((dtype).c_obj) + options.set_dtypes(c_dtypes_list) + elif isinstance(dtypes, dict): + # dtypes_t is dict + for k, v in dtypes.items(): + c_dtypes_map[str(k).encode()] = (v).c_obj + options.set_dtypes(c_dtypes_map) + elif dtypes is not None: + raise TypeError("dtypes must either by a list/dict") + + if true_values is not None: + options.set_true_values(_make_str_vector(true_values)) + + if false_values is not None: + options.set_false_values(_make_str_vector(false_values)) + + if na_values is not None: + options.set_na_values(_make_str_vector(na_values)) + + cdef table_with_metadata c_result + with nogil: + c_result = move(cpp_read_csv(options)) + + return TableWithMetadata.from_libcudf(c_result) diff --git a/python/cudf/cudf/_lib/pylibcudf/io/types.pxd b/python/cudf/cudf/_lib/pylibcudf/io/types.pxd index ab223c16a72..0094bf6032c 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/types.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/io/types.pxd @@ -38,6 +38,9 @@ cdef class TableWithMetadata: cdef class SourceInfo: cdef source_info c_obj + # Keep the bytes converted from stringio alive + # (otherwise we end up with a use after free when they get gc'ed) + cdef list byte_sources cdef class SinkInfo: # This vector just exists to keep the unique_ptrs to the sinks alive diff --git a/python/cudf/cudf/_lib/pylibcudf/io/types.pyx b/python/cudf/cudf/_lib/pylibcudf/io/types.pyx index df0b729b711..68498ff88f4 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/types.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/io/types.pyx @@ -178,7 +178,7 @@ cdef class SourceInfo: raise ValueError("All sources must be of the same type!") new_sources.append(buffer.read().encode()) sources = new_sources - + self.byte_sources = sources if isinstance(sources[0], bytes): empty_buffer = True for buffer in sources: diff --git a/python/cudf/cudf/_lib/types.pyx b/python/cudf/cudf/_lib/types.pyx index 895e1afc502..fc672caa574 100644 --- a/python/cudf/cudf/_lib/types.pyx +++ b/python/cudf/cudf/_lib/types.pyx @@ -239,6 +239,9 @@ cdef dtype_from_column_view(column_view cv): ] cdef libcudf_types.data_type dtype_to_data_type(dtype) except *: + # Note: This function is to be phased out in favor of + # dtype_to_pylibcudf_type which will return a pylibcudf + # DataType object cdef libcudf_types.type_id tid if isinstance(dtype, cudf.ListDtype): tid = libcudf_types.type_id.LIST diff --git a/python/cudf/cudf/pylibcudf_tests/common/utils.py b/python/cudf/cudf/pylibcudf_tests/common/utils.py index efb192b3251..e029edfa2ed 100644 --- a/python/cudf/cudf/pylibcudf_tests/common/utils.py +++ b/python/cudf/cudf/pylibcudf_tests/common/utils.py @@ -4,6 +4,7 @@ import io import os +import numpy as np import pyarrow as pa import pytest @@ -109,7 +110,10 @@ def _make_fields_nullable(typ): lhs_type = _make_fields_nullable(lhs.type) lhs = rhs.cast(lhs_type) - assert lhs.equals(rhs) + if pa.types.is_floating(lhs.type) and pa.types.is_floating(rhs.type): + np.testing.assert_array_almost_equal(lhs, rhs) + else: + assert lhs.equals(rhs) def assert_table_eq(pa_table: pa.Table, plc_table: plc.Table) -> None: @@ -125,6 +129,8 @@ def assert_table_and_meta_eq( pa_table: pa.Table, plc_table_w_meta: plc.io.types.TableWithMetadata, check_field_nullability=True, + check_types_if_empty=True, + check_names=True, ) -> None: """Verify that the pylibcudf TableWithMetadata and PyArrow table are equal""" @@ -135,11 +141,17 @@ def assert_table_and_meta_eq( plc_shape == pa_table.shape ), f"{plc_shape} is not equal to {pa_table.shape}" + if not check_types_if_empty and plc_table.num_rows() == 0: + return + for plc_col, pa_col in zip(plc_table.columns(), pa_table.columns): assert_column_eq(pa_col, plc_col, check_field_nullability) # Check column name equality - assert plc_table_w_meta.column_names() == pa_table.column_names + if check_names: + assert ( + plc_table_w_meta.column_names() == pa_table.column_names + ), f"{plc_table_w_meta.column_names()} != {pa_table.column_names}" def cudf_raises(expected_exception: BaseException, *args, **kwargs): @@ -174,6 +186,33 @@ def is_nested_list(typ): return nesting_level(typ)[0] > 1 +def _convert_numeric_types_to_floating(pa_table): + """ + Useful little helper for testing the + dtypes option in I/O readers. + + Returns a tuple containing the pylibcudf dtypes + and the new pyarrow schema + """ + dtypes = [] + new_fields = [] + for i in range(len(pa_table.schema)): + field = pa_table.schema.field(i) + child_types = [] + + plc_type = plc.interop.from_arrow(field.type) + if pa.types.is_integer(field.type) or pa.types.is_unsigned_integer( + field.type + ): + plc_type = plc.interop.from_arrow(pa.float64()) + field = field.with_type(pa.float64()) + + dtypes.append((field.name, plc_type, child_types)) + + new_fields.append(field) + return dtypes, new_fields + + def write_source_str(source, input_str): """ Write a string to the source diff --git a/python/cudf/cudf/pylibcudf_tests/conftest.py b/python/cudf/cudf/pylibcudf_tests/conftest.py index 53e207f29cb..4a7194a6d8d 100644 --- a/python/cudf/cudf/pylibcudf_tests/conftest.py +++ b/python/cudf/cudf/pylibcudf_tests/conftest.py @@ -141,6 +141,20 @@ def _generate_nested_data(typ): ), pa_table +@pytest.fixture(params=[(0, 0), ("half", 0), (-1, "half")]) +def nrows_skiprows(table_data, request): + """ + Parametrized nrows fixture that accompanies table_data + """ + _, pa_table = table_data + nrows, skiprows = request.param + if nrows == "half": + nrows = len(pa_table) // 2 + if skiprows == "half": + skiprows = (len(pa_table) - nrows) // 2 + return nrows, skiprows + + @pytest.fixture( params=["a.txt", pathlib.Path("a.txt"), io.BytesIO, io.StringIO], ) diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_csv.py b/python/cudf/cudf/pylibcudf_tests/io/test_csv.py new file mode 100644 index 00000000000..95326a8b681 --- /dev/null +++ b/python/cudf/cudf/pylibcudf_tests/io/test_csv.py @@ -0,0 +1,280 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +import io +import os +from io import StringIO + +import pandas as pd +import pyarrow as pa +import pytest +from utils import ( + _convert_numeric_types_to_floating, + assert_table_and_meta_eq, + make_source, + write_source_str, +) + +import cudf._lib.pylibcudf as plc +from cudf._lib.pylibcudf.io.types import CompressionType + +# Shared kwargs to pass to make_source +_COMMON_CSV_SOURCE_KWARGS = { + "format": "csv", + "index": False, +} + + +@pytest.fixture(scope="module") +def csv_table_data(table_data): + """ + Like the table_data but with nested types dropped + since the CSV reader can't handle that + uint64 is also dropped since it can get confused with int64 + """ + _, pa_table = table_data + pa_table = pa_table.drop_columns( + [ + "col_uint64", + "col_list", + "col_list>", + "col_struct", + "col_struct not null>", + ] + ) + return plc.interop.from_arrow(pa_table), pa_table + + +@pytest.mark.parametrize("delimiter", [",", ";"]) +def test_read_csv_basic( + csv_table_data, + source_or_sink, + text_compression_type, + nrows_skiprows, + delimiter, +): + _, pa_table = csv_table_data + compression_type = text_compression_type + nrows, skiprows = nrows_skiprows + + # can't compress non-binary data with pandas + if isinstance(source_or_sink, io.StringIO): + compression_type = CompressionType.NONE + + source = make_source( + source_or_sink, + pa_table, + compression=compression_type, + sep=delimiter, + **_COMMON_CSV_SOURCE_KWARGS, + ) + + # Rename the table (by reversing the names) to test names argument + pa_table = pa_table.rename_columns(pa_table.column_names[::-1]) + column_names = pa_table.column_names + + # Adapt to nrows/skiprows + pa_table = pa_table.slice( + offset=skiprows, length=nrows if nrows != -1 else None + ) + + res = plc.io.csv.read_csv( + plc.io.SourceInfo([source]), + delimiter=delimiter, + compression=compression_type, + col_names=column_names, + nrows=nrows, + skiprows=skiprows, + ) + + assert_table_and_meta_eq( + pa_table, + res, + check_types_if_empty=False, + check_names=False if skiprows > 0 and column_names is None else True, + ) + + +# Note: make sure chunk size is big enough so that dtype inference +# infers correctly +@pytest.mark.parametrize("chunk_size", [1000, 5999]) +def test_read_csv_byte_range(table_data, chunk_size, tmp_path): + _, pa_table = table_data + if len(pa_table) == 0: + # pandas writes nothing when we have empty table + # and header=None + pytest.skip("Don't test empty table case") + source = f"{tmp_path}/a.csv" + source = make_source( + source, pa_table, header=False, **_COMMON_CSV_SOURCE_KWARGS + ) + file_size = os.stat(source).st_size + tbls_w_meta = [] + for segment in range((file_size + chunk_size - 1) // chunk_size): + tbls_w_meta.append( + plc.io.csv.read_csv( + plc.io.SourceInfo([source]), + byte_range_offset=segment * chunk_size, + byte_range_size=chunk_size, + header=-1, + col_names=pa_table.column_names, + ) + ) + if isinstance(source, io.IOBase): + source.seek(0) + exp = pd.read_csv(source, names=pa_table.column_names, header=None) + tbls = [] + for tbl_w_meta in tbls_w_meta: + if tbl_w_meta.tbl.num_rows() > 0: + tbls.append(plc.interop.to_arrow(tbl_w_meta.tbl)) + full_tbl = pa.concat_tables(tbls) + + full_tbl_plc = plc.io.TableWithMetadata( + plc.interop.from_arrow(full_tbl), + tbls_w_meta[0].column_names(include_children=True), + ) + assert_table_and_meta_eq(pa.Table.from_pandas(exp), full_tbl_plc) + + +@pytest.mark.parametrize("usecols", [None, ["col_int64", "col_bool"], [0, 1]]) +def test_read_csv_dtypes(csv_table_data, source_or_sink, usecols): + # Simple test for dtypes where we read in + # all numeric data as floats + _, pa_table = csv_table_data + + source = make_source( + source_or_sink, + pa_table, + **_COMMON_CSV_SOURCE_KWARGS, + ) + # Adjust table for usecols + if usecols is not None: + pa_table = pa_table.select(usecols) + + dtypes, new_fields = _convert_numeric_types_to_floating(pa_table) + # Extract the dtype out of the (name, type, child_types) tuple + # (read_csv doesn't support this format since it doesn't support nested columns) + dtypes = {name: dtype for name, dtype, _ in dtypes} + + new_schema = pa.schema(new_fields) + + res = plc.io.csv.read_csv( + plc.io.SourceInfo([source]), dtypes=dtypes, usecols=usecols + ) + new_table = pa_table.cast(new_schema) + + assert_table_and_meta_eq(new_table, res) + + +@pytest.mark.parametrize("skip_blanks", [True, False]) +@pytest.mark.parametrize("decimal, quotechar", [(".", "'"), ("_", '"')]) +@pytest.mark.parametrize("lineterminator", ["\n", "\r\n"]) +def test_read_csv_parse_options( + source_or_sink, decimal, quotechar, skip_blanks, lineterminator +): + lines = [ + "# first comment line", + "# third comment line", + "1,2,3,4_4,'z'", + '4,5,6,5_5,""', + "7,8,9,9_87,'123'", + "# last comment line", + "1,1,1,10_11,abc", + ] + buffer = lineterminator.join(lines) + + write_source_str(source_or_sink, buffer) + + plc_table_w_meta = plc.io.csv.read_csv( + plc.io.SourceInfo([source_or_sink]), + comment="#", + decimal=decimal, + skip_blank_lines=skip_blanks, + quotechar=quotechar, + ) + df = pd.read_csv( + StringIO(buffer), + comment="#", + decimal=decimal, + skip_blank_lines=skip_blanks, + quotechar=quotechar, + ) + assert_table_and_meta_eq(pa.Table.from_pandas(df), plc_table_w_meta) + + +@pytest.mark.parametrize("na_filter", [True, False]) +@pytest.mark.parametrize("na_values", [["n/a"], ["NV_NAN"]]) +@pytest.mark.parametrize("keep_default_na", [True, False]) +def test_read_csv_na_values( + source_or_sink, na_filter, na_values, keep_default_na +): + lines = ["a,b,c", "n/a,NaN,NV_NAN", "1.0,2.0,3.0"] + buffer = "\n".join(lines) + + write_source_str(source_or_sink, buffer) + + plc_table_w_meta = plc.io.csv.read_csv( + plc.io.SourceInfo([source_or_sink]), + na_filter=na_filter, + na_values=na_values if na_filter else None, + keep_default_na=keep_default_na, + ) + df = pd.read_csv( + StringIO(buffer), + na_filter=na_filter, + na_values=na_values if na_filter else None, + keep_default_na=keep_default_na, + ) + assert_table_and_meta_eq(pa.Table.from_pandas(df), plc_table_w_meta) + + +@pytest.mark.parametrize("header", [0, 10, -1]) +def test_read_csv_header(csv_table_data, source_or_sink, header): + _, pa_table = csv_table_data + + source = make_source( + source_or_sink, + pa_table, + **_COMMON_CSV_SOURCE_KWARGS, + ) + + plc_table_w_meta = plc.io.csv.read_csv( + plc.io.SourceInfo([source]), header=header + ) + if header > 0: + if header < len(pa_table): + names_row = pa_table.take([header - 1]).to_pylist()[0].values() + pa_table = pa_table.slice(header) + col_names = [str(name) for name in names_row] + pa_table = pa_table.rename_columns(col_names) + else: + pa_table = pa.table([]) + elif header < 0: + # neg header means use user-provided names (in this case nothing) + # (the original column names are now data) + tbl_dict = pa_table.to_pydict() + new_tbl_dict = {} + for i, (name, vals) in enumerate(tbl_dict.items()): + str_vals = [str(val) for val in vals] + new_tbl_dict[str(i)] = [name] + str_vals + pa_table = pa.table(new_tbl_dict) + + assert_table_and_meta_eq( + pa_table, + plc_table_w_meta, + check_types_if_empty=False, + ) + + +# TODO: test these +# str prefix = "", +# bool mangle_dupe_cols = True, +# size_type skipfooter = 0, +# str thousands = None, +# bool delim_whitespace = False, +# bool skipinitialspace = False, +# quote_style quoting = quote_style.MINIMAL, +# bool doublequote = True, +# bool detect_whitespace_around_quotes = False, +# list parse_dates = None, +# list true_values = None, +# list false_values = None, +# bool dayfirst = False, From c6c21d7f9281f295e32ff72c95f95b600470df0e Mon Sep 17 00:00:00 2001 From: jakirkham Date: Thu, 18 Jul 2024 12:41:21 -0700 Subject: [PATCH 28/53] Drop `{{ pin_compatible('numpy', max_pin='x') }}` (#16301) Part of issue: https://github.com/rapidsai/build-planning/issues/82 Drop `{{ pin_compatible('numpy', max_pin='x') }}` as it is no longer needed. `numpy` has its own `run_exports`, which constraints `numpy` to an API compatible version. More details in issue: https://github.com/orgs/rapidsai/projects/132 So `cudf` now uses that in its recipe builds. Also update `requirements/run` to set the `numpy` lower bound to `1.23` as required by us. Lastly add todo comments for NumPy 2 update lines. Authors: - https://github.com/jakirkham Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/16301 --- conda/recipes/cudf/meta.yaml | 4 +++- dependencies.yaml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 3cdc2050631..9137f099ad1 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -64,6 +64,7 @@ requirements: - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.7.0 - dlpack >=0.8,<1.0 + # TODO: Change to `2.0` for NumPy 2 - numpy 1.23 - pyarrow ==16.1.0.* - libcudf ={{ version }} @@ -82,7 +83,8 @@ requirements: - pandas >=2.0,<2.2.3dev0 - cupy >=12.0.0 - numba >=0.57 - - {{ pin_compatible('numpy', max_pin='x') }} + # TODO: Update `numpy` in `host` when dropping `<2.0a0` + - numpy >=1.23,<2.0a0 - {{ pin_compatible('pyarrow', max_pin='x.x') }} - libcudf ={{ version }} - {{ pin_compatible('rmm', max_pin='x.x') }} diff --git a/dependencies.yaml b/dependencies.yaml index 67ed3773b44..a19574b7658 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -323,6 +323,7 @@ dependencies: packages: # Hard pin the patch version used during the build. # Sync with conda build constraint & wheel run constraint. + # TODO: Change to `2.0.*` for NumPy 2 - numpy==1.23.* build_python_cudf: common: @@ -551,6 +552,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - fsspec>=0.6.0 + # TODO: Update `numpy` in `build_python_common` when dropping `<2.0a0` - numpy>=1.23,<2.0a0 - pandas>=2.0,<2.2.3dev0 run_cudf: From aeef0a1f4159d4c87f987d20225401040973d10f Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 18 Jul 2024 16:56:30 -0400 Subject: [PATCH 29/53] Remove hash_character_ngrams dependency from jaccard_index (#16241) Removes internal dependency of `nvtext::hash_character_ngrams` from `nvtext::jaccard_index`. Works around the size-type limit imposed by `hash_character_ngrams` which returns a `list` column. This also specializes the hashing logic for the jaccard calculation specifically. The overall algorithm has not changed. Code has moved around a bit and internal list-columns have been replaced with just offsets and values vectors. Closes #16157 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/16241 --- cpp/benchmarks/text/jaccard.cpp | 4 +- cpp/src/text/jaccard.cu | 478 ++++++++++++++++++++++---------- 2 files changed, 339 insertions(+), 143 deletions(-) diff --git a/cpp/benchmarks/text/jaccard.cpp b/cpp/benchmarks/text/jaccard.cpp index d05c195d077..d5b74da6773 100644 --- a/cpp/benchmarks/text/jaccard.cpp +++ b/cpp/benchmarks/text/jaccard.cpp @@ -59,6 +59,6 @@ static void bench_jaccard(nvbench::state& state) NVBENCH_BENCH(bench_jaccard) .set_name("jaccard") - .add_int64_axis("num_rows", {1024, 4096, 8192, 16364, 32768, 262144}) - .add_int64_axis("row_width", {128, 512, 2048}) + .add_int64_axis("num_rows", {32768, 131072, 262144}) + .add_int64_axis("row_width", {128, 512, 1024, 2048}) .add_int64_axis("substring_width", {5, 10}); diff --git a/cpp/src/text/jaccard.cu b/cpp/src/text/jaccard.cu index 9cf934165f6..e465fb79c89 100644 --- a/cpp/src/text/jaccard.cu +++ b/cpp/src/text/jaccard.cu @@ -19,16 +19,19 @@ #include #include #include +#include #include -#include +#include +#include +#include #include #include #include -#include #include #include +#include #include #include @@ -36,127 +39,375 @@ #include #include #include +#include +#include +#include #include namespace nvtext { namespace detail { namespace { +constexpr cudf::thread_index_type block_size = 256; +constexpr cudf::thread_index_type bytes_per_thread = 4; + /** * @brief Retrieve the row data (span) for the given column/row-index * - * @param d_input Input lists column + * @param values Flat vector of all values + * @param offsets Offsets identifying rows within values * @param idx Row index to retrieve * @return A device-span of the row values */ -__device__ auto get_row(cudf::column_device_view const& d_input, cudf::size_type idx) +__device__ auto get_row(uint32_t const* values, int64_t const* offsets, cudf::size_type row_idx) { - auto const offsets = - d_input.child(cudf::lists_column_view::offsets_column_index).data(); - auto const offset = offsets[idx]; - auto const size = offsets[idx + 1] - offset; - auto const begin = - d_input.child(cudf::lists_column_view::child_column_index).data() + offset; + auto const offset = offsets[row_idx]; + auto const size = offsets[row_idx + 1] - offset; + auto const begin = values + offset; return cudf::device_span(begin, size); } /** - * @brief Count the unique values within each row of the input column + * @brief Kernel to count the unique values within each row of the input column + * + * This is called with a warp per row. * - * This is called with a warp per row + * @param d_values Sorted hash values to count uniqueness + * @param d_offsets Offsets to each set of row elements in d_values + * @param rows Number of rows in the output + * @param d_results Number of unique values in each row */ -struct sorted_unique_fn { - cudf::column_device_view const d_input; - cudf::size_type* d_results; +CUDF_KERNEL void sorted_unique_fn(uint32_t const* d_values, + int64_t const* d_offsets, + cudf::size_type rows, + cudf::size_type* d_results) +{ + auto const idx = cudf::detail::grid_1d::global_thread_id(); + if (idx >= (static_cast(rows) * cudf::detail::warp_size)) { return; } - // warp per row - __device__ void operator()(cudf::size_type idx) const - { - using warp_reduce = cub::WarpReduce; - __shared__ typename warp_reduce::TempStorage temp_storage; + using warp_reduce = cub::WarpReduce; + __shared__ typename warp_reduce::TempStorage temp_storage; - auto const row_idx = idx / cudf::detail::warp_size; - auto const lane_idx = idx % cudf::detail::warp_size; - auto const row = get_row(d_input, row_idx); - auto const begin = row.begin(); + auto const row_idx = idx / cudf::detail::warp_size; + auto const lane_idx = idx % cudf::detail::warp_size; + auto const row = get_row(d_values, d_offsets, row_idx); + auto const begin = row.begin(); - cudf::size_type count = 0; - for (auto itr = begin + lane_idx; itr < row.end(); itr += cudf::detail::warp_size) { - count += (itr == begin || *itr != *(itr - 1)); - } - auto const result = warp_reduce(temp_storage).Sum(count); - if (lane_idx == 0) { d_results[row_idx] = result; } + cudf::size_type count = 0; + for (auto itr = begin + lane_idx; itr < row.end(); itr += cudf::detail::warp_size) { + count += (itr == begin || *itr != *(itr - 1)); } -}; + auto const result = warp_reduce(temp_storage).Sum(count); + if (lane_idx == 0) { d_results[row_idx] = result; } +} -rmm::device_uvector compute_unique_counts(cudf::column_view const& input, +/** + * @brief Count the unique values within each row of the input column + * + * @param values Sorted hash values to count uniqueness + * @param offsets Offsets to each set of row elements in d_values + * @param rows Number of rows in the output + * @param stream CUDA stream used for device memory operations and kernel launches + * @return Number of unique values + */ +rmm::device_uvector compute_unique_counts(uint32_t const* values, + int64_t const* offsets, + cudf::size_type rows, rmm::cuda_stream_view stream) { - auto const d_input = cudf::column_device_view::create(input, stream); - auto d_results = rmm::device_uvector(input.size(), stream); - sorted_unique_fn fn{*d_input, d_results.data()}; - thrust::for_each_n(rmm::exec_policy(stream), - thrust::counting_iterator(0), - input.size() * cudf::detail::warp_size, - fn); + auto d_results = rmm::device_uvector(rows, stream); + auto const num_blocks = cudf::util::div_rounding_up_safe( + static_cast(rows) * cudf::detail::warp_size, block_size); + sorted_unique_fn<<>>( + values, offsets, rows, d_results.data()); return d_results; } +/** + * @brief Kernel to count the number of common values within each row of the 2 input columns + * + * This is called with a warp per row. + * + * @param d_values1 Sorted hash values to check against d_values2 + * @param d_offsets1 Offsets to each set of row elements in d_values1 + * @param d_values2 Sorted hash values to check against d_values1 + * @param d_offsets2 Offsets to each set of row elements in d_values2 + * @param rows Number of rows in the output + * @param d_results Number of common values in each row + */ +CUDF_KERNEL void sorted_intersect_fn(uint32_t const* d_values1, + int64_t const* d_offsets1, + uint32_t const* d_values2, + int64_t const* d_offsets2, + cudf::size_type rows, + cudf::size_type* d_results) +{ + auto const idx = cudf::detail::grid_1d::global_thread_id(); + if (idx >= (static_cast(rows) * cudf::detail::warp_size)) { return; } + + using warp_reduce = cub::WarpReduce; + __shared__ typename warp_reduce::TempStorage temp_storage; + + auto const row_idx = idx / cudf::detail::warp_size; + auto const lane_idx = idx % cudf::detail::warp_size; + + auto const needles = get_row(d_values1, d_offsets1, row_idx); + auto const haystack = get_row(d_values2, d_offsets2, row_idx); + + auto begin = haystack.begin(); + auto const end = haystack.end(); + + cudf::size_type count = 0; + for (auto itr = needles.begin() + lane_idx; itr < needles.end() && begin < end; + itr += cudf::detail::warp_size) { + if (itr != needles.begin() && *itr == *(itr - 1)) { continue; } // skip duplicates + // search haystack for this needle (*itr) + auto const found = thrust::lower_bound(thrust::seq, begin, end, *itr); + count += (found != end) && (*found == *itr); // increment if found; + begin = found; // shorten the next lower-bound range + } + // sum up the counts across this warp + auto const result = warp_reduce(temp_storage).Sum(count); + if (lane_idx == 0) { d_results[row_idx] = result; } +} + /** * @brief Count the number of common values within each row of the 2 input columns * - * This is called with a warp per row + * @param d_values1 Sorted hash values to check against d_values2 + * @param d_offsets1 Offsets to each set of row elements in d_values1 + * @param d_values2 Sorted hash values to check against d_values1 + * @param d_offsets2 Offsets to each set of row elements in d_values2 + * @param rows Number of rows in the output + * @param stream CUDA stream used for device memory operations and kernel launches + * @return Number of common values */ -struct sorted_intersect_fn { - cudf::column_device_view const d_input1; - cudf::column_device_view const d_input2; - cudf::size_type* d_results; +rmm::device_uvector compute_intersect_counts(uint32_t const* values1, + int64_t const* offsets1, + uint32_t const* values2, + int64_t const* offsets2, + cudf::size_type rows, + rmm::cuda_stream_view stream) +{ + auto d_results = rmm::device_uvector(rows, stream); + auto const num_blocks = cudf::util::div_rounding_up_safe( + static_cast(rows) * cudf::detail::warp_size, block_size); + sorted_intersect_fn<<>>( + values1, offsets1, values2, offsets2, rows, d_results.data()); + return d_results; +} - // warp per row - __device__ void operator()(cudf::size_type idx) const - { - using warp_reduce = cub::WarpReduce; - __shared__ typename warp_reduce::TempStorage temp_storage; +/** + * @brief Counts the number of substrings in each row of the given strings column + * + * Each warp processes a single string. + * Formula is `count = max(1, str.length() - width + 1)` + * If a string has less than width characters (but not empty), the count is 1 + * since the entire string is still hashed. + * + * @param d_strings Input column of strings + * @param width Substring size in characters + * @param d_counts Output number of substring per row of input + */ +CUDF_KERNEL void count_substrings_kernel(cudf::column_device_view const d_strings, + cudf::size_type width, + int64_t* d_counts) +{ + auto const idx = cudf::detail::grid_1d::global_thread_id(); + if (idx >= (static_cast(d_strings.size()) * cudf::detail::warp_size)) { + return; + } - auto const row_idx = idx / cudf::detail::warp_size; - auto const lane_idx = idx % cudf::detail::warp_size; + auto const str_idx = static_cast(idx / cudf::detail::warp_size); + if (d_strings.is_null(str_idx)) { + d_counts[str_idx] = 0; + return; + } - auto const needles = get_row(d_input1, row_idx); - auto const haystack = get_row(d_input2, row_idx); + auto const d_str = d_strings.element(str_idx); + if (d_str.empty()) { + d_counts[str_idx] = 0; + return; + } - auto begin = haystack.begin(); - auto const end = haystack.end(); + using warp_reduce = cub::WarpReduce; + __shared__ typename warp_reduce::TempStorage temp_storage; - // TODO: investigate cuCollections device-side static-map to match row values + auto const end = d_str.data() + d_str.size_bytes(); + auto const lane_idx = idx % cudf::detail::warp_size; + cudf::size_type count = 0; + for (auto itr = d_str.data() + (lane_idx * bytes_per_thread); itr < end; + itr += cudf::detail::warp_size * bytes_per_thread) { + for (auto s = itr; (s < (itr + bytes_per_thread)) && (s < end); ++s) { + count += static_cast(cudf::strings::detail::is_begin_utf8_char(*s)); + } + } + auto const char_count = warp_reduce(temp_storage).Sum(count); + if (lane_idx == 0) { d_counts[str_idx] = std::max(1, char_count - width + 1); } +} + +/** + * @brief Kernel to hash the substrings for each input row + * + * Each warp processes a single string. + * Substrings of string "hello world" with width=4 produce: + * "hell", "ello", "llo ", "lo w", "o wo", " wor", "worl", "orld" + * Each of these substrings is hashed and the hash stored in d_results + * + * @param d_strings Input column of strings + * @param width Substring size in characters + * @param d_output_offsets Offsets into d_results + * @param d_results Hash values for each substring + */ +CUDF_KERNEL void substring_hash_kernel(cudf::column_device_view const d_strings, + cudf::size_type width, + int64_t const* d_output_offsets, + uint32_t* d_results) +{ + auto const idx = cudf::detail::grid_1d::global_thread_id(); + if (idx >= (static_cast(d_strings.size()) * cudf::detail::warp_size)) { + return; + } - cudf::size_type count = 0; - for (auto itr = needles.begin() + lane_idx; itr < needles.end() && begin < end; - itr += cudf::detail::warp_size) { - if (itr != needles.begin() && *itr == *(itr - 1)) { continue; } // skip duplicates - // search haystack for this needle (*itr) - auto const found = thrust::lower_bound(thrust::seq, begin, end, *itr); - count += (found != end) && (*found == *itr); // increment if found; - begin = found; // shorten the next lower-bound range + auto const str_idx = idx / cudf::detail::warp_size; + auto const lane_idx = idx % cudf::detail::warp_size; + + if (d_strings.is_null(str_idx)) { return; } + auto const d_str = d_strings.element(str_idx); + if (d_str.empty()) { return; } + + __shared__ uint32_t hvs[block_size]; // temp store for hash values + + auto const hasher = cudf::hashing::detail::MurmurHash3_x86_32{0}; + auto const end = d_str.data() + d_str.size_bytes(); + auto const warp_count = (d_str.size_bytes() / cudf::detail::warp_size) + 1; + + auto d_hashes = d_results + d_output_offsets[str_idx]; + auto itr = d_str.data() + lane_idx; + for (auto i = 0; i < warp_count; ++i) { + uint32_t hash = 0; + if (itr < end && cudf::strings::detail::is_begin_utf8_char(*itr)) { + // resolve substring + auto const sub_str = + cudf::string_view(itr, static_cast(thrust::distance(itr, end))); + auto const [bytes, left] = cudf::strings::detail::bytes_to_character_position(sub_str, width); + // hash only if we have the full width of characters or this is the beginning of the string + if ((left == 0) || (itr == d_str.data())) { hash = hasher(cudf::string_view(itr, bytes)); } } - // sum up the counts across this warp - auto const result = warp_reduce(temp_storage).Sum(count); - if (lane_idx == 0) { d_results[row_idx] = result; } + hvs[threadIdx.x] = hash; // store hash into shared memory + __syncwarp(); + if (lane_idx == 0) { + // copy valid hash values for this warp into d_hashes + auto const hashes = &hvs[threadIdx.x]; + auto const hashes_end = hashes + cudf::detail::warp_size; + d_hashes = + thrust::copy_if(thrust::seq, hashes, hashes_end, d_hashes, [](auto h) { return h != 0; }); + } + __syncwarp(); + itr += cudf::detail::warp_size; } -}; +} -rmm::device_uvector compute_intersect_counts(cudf::column_view const& input1, - cudf::column_view const& input2, - rmm::cuda_stream_view stream) +void segmented_sort(uint32_t const* input, + uint32_t* output, + int64_t items, + cudf::size_type segments, + int64_t const* offsets, + rmm::cuda_stream_view stream) { - auto const d_input1 = cudf::column_device_view::create(input1, stream); - auto const d_input2 = cudf::column_device_view::create(input2, stream); - auto d_results = rmm::device_uvector(input1.size(), stream); - sorted_intersect_fn fn{*d_input1, *d_input2, d_results.data()}; - thrust::for_each_n(rmm::exec_policy(stream), - thrust::counting_iterator(0), - input1.size() * cudf::detail::warp_size, - fn); - return d_results; + rmm::device_buffer temp; + std::size_t temp_bytes = 0; + cub::DeviceSegmentedSort::SortKeys( + temp.data(), temp_bytes, input, output, items, segments, offsets, offsets + 1, stream.value()); + temp = rmm::device_buffer(temp_bytes, stream); + cub::DeviceSegmentedSort::SortKeys( + temp.data(), temp_bytes, input, output, items, segments, offsets, offsets + 1, stream.value()); +} + +/** + * @brief Create hashes for each substring + * + * The hashes are sorted using a segmented-sort as setup to + * perform the unique and intersect operations. + * + * @param input Input strings column to hash + * @param width Substring width in characters + * @param stream CUDA stream used for device memory operations and kernel launches + * @return The sorted hash values and offsets to each row + */ +std::pair, rmm::device_uvector> hash_substrings( + cudf::strings_column_view const& input, cudf::size_type width, rmm::cuda_stream_view stream) +{ + auto const d_strings = cudf::column_device_view::create(input.parent(), stream); + + // count substrings + auto offsets = rmm::device_uvector(input.size() + 1, stream); + auto const num_blocks = cudf::util::div_rounding_up_safe( + static_cast(input.size()) * cudf::detail::warp_size, block_size); + count_substrings_kernel<<>>( + *d_strings, width, offsets.data()); + auto const total_hashes = + cudf::detail::sizes_to_offsets(offsets.begin(), offsets.end(), offsets.begin(), stream); + + // hash substrings + rmm::device_uvector hashes(total_hashes, stream); + substring_hash_kernel<<>>( + *d_strings, width, offsets.data(), hashes.data()); + + // sort hashes + rmm::device_uvector sorted(total_hashes, stream); + if (total_hashes < static_cast(std::numeric_limits::max())) { + segmented_sort( + hashes.begin(), sorted.begin(), sorted.size(), input.size(), offsets.begin(), stream); + } else { + // The CUB segmented sort can only handle max total values + // so this code calls it in sections. + auto const section_size = std::numeric_limits::max() / 2L; + auto const sort_sections = cudf::util::div_rounding_up_safe(total_hashes, section_size); + auto const offset_indices = [&] { + // build a set of indices that point to offsets subsections + auto sub_offsets = rmm::device_uvector(sort_sections + 1, stream); + thrust::sequence( + rmm::exec_policy(stream), sub_offsets.begin(), sub_offsets.end(), 0L, section_size); + auto indices = rmm::device_uvector(sub_offsets.size(), stream); + thrust::lower_bound(rmm::exec_policy(stream), + offsets.begin(), + offsets.end(), + sub_offsets.begin(), + sub_offsets.end(), + indices.begin()); + return cudf::detail::make_std_vector_sync(indices, stream); + }(); + + // Call segmented sort with the sort sections + for (auto i = 0L; i < sort_sections; ++i) { + auto const index1 = offset_indices[i]; + auto const index2 = std::min(offset_indices[i + 1], static_cast(offsets.size() - 1)); + auto const offset1 = offsets.element(index1, stream); + auto const offset2 = offsets.element(index2, stream); + + auto const num_items = offset2 - offset1; + auto const num_segments = index2 - index1; + + // There is a bug in the CUB segmented sort and the workaround is to + // shift the offset values so the first offset is 0. + // This transform can be removed once the bug is fixed. + auto sort_offsets = rmm::device_uvector(num_segments + 1, stream); + thrust::transform(rmm::exec_policy(stream), + offsets.begin() + index1, + offsets.begin() + index2 + 1, + sort_offsets.begin(), + [offset1] __device__(auto const o) { return o - offset1; }); + + segmented_sort(hashes.begin() + offset1, + sorted.begin() + offset1, + num_items, + num_segments, + sort_offsets.begin(), + stream); + } + } + return std::make_pair(std::move(sorted), std::move(offsets)); } /** @@ -186,62 +437,6 @@ struct jaccard_fn { } }; -/** - * @brief Create hashes for each substring - * - * Uses the hash_character_ngrams to hash substrings of the input column. - * This returns a lists column where each row is the hashes for the substrings - * of the corresponding input string row. - * - * The hashes are then sorted using a segmented-sort as setup to - * perform the unique and intersect operations. - */ -std::unique_ptr hash_substrings(cudf::strings_column_view const& col, - cudf::size_type width, - rmm::cuda_stream_view stream) -{ - auto hashes = hash_character_ngrams(col, width, stream, rmm::mr::get_current_device_resource()); - auto const input = cudf::lists_column_view(hashes->view()); - auto const offsets = input.offsets_begin(); - auto const data = input.child().data(); - - rmm::device_uvector sorted(input.child().size(), stream); - - // this is wicked fast and much faster than using cudf::lists::detail::sort_list - rmm::device_buffer d_temp_storage; - size_t temp_storage_bytes = 0; - cub::DeviceSegmentedSort::SortKeys(d_temp_storage.data(), - temp_storage_bytes, - data, - sorted.data(), - sorted.size(), - input.size(), - offsets, - offsets + 1, - stream.value()); - d_temp_storage = rmm::device_buffer{temp_storage_bytes, stream}; - cub::DeviceSegmentedSort::SortKeys(d_temp_storage.data(), - temp_storage_bytes, - data, - sorted.data(), - sorted.size(), - input.size(), - offsets, - offsets + 1, - stream.value()); - - auto contents = hashes->release(); - // the offsets are taken from the hashes column since they are the same - // before and after the segmented-sort - return cudf::make_lists_column( - col.size(), - std::move(contents.children.front()), - std::make_unique(std::move(sorted), rmm::device_buffer{}, 0), - 0, - rmm::device_buffer{}, - stream, - rmm::mr::get_current_device_resource()); -} } // namespace std::unique_ptr jaccard_index(cudf::strings_column_view const& input1, @@ -261,13 +456,14 @@ std::unique_ptr jaccard_index(cudf::strings_column_view const& inp auto const [d_uniques1, d_uniques2, d_intersects] = [&] { // build hashes of the substrings - auto const hash1 = hash_substrings(input1, width, stream); - auto const hash2 = hash_substrings(input2, width, stream); + auto const [hash1, offsets1] = hash_substrings(input1, width, stream); + auto const [hash2, offsets2] = hash_substrings(input2, width, stream); // compute the unique counts in each set and the intersection counts - auto d_uniques1 = compute_unique_counts(hash1->view(), stream); - auto d_uniques2 = compute_unique_counts(hash2->view(), stream); - auto d_intersects = compute_intersect_counts(hash1->view(), hash2->view(), stream); + auto d_uniques1 = compute_unique_counts(hash1.data(), offsets1.data(), input1.size(), stream); + auto d_uniques2 = compute_unique_counts(hash2.data(), offsets2.data(), input2.size(), stream); + auto d_intersects = compute_intersect_counts( + hash1.data(), offsets1.data(), hash2.data(), offsets2.data(), input1.size(), stream); return std::tuple{std::move(d_uniques1), std::move(d_uniques2), std::move(d_intersects)}; }(); From 4acca4d57303f52907aa158a2ef996c9d42a73d6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:07:07 -1000 Subject: [PATCH 30/53] Use Column.can_cast_safely instead of some ad-hoc dtype functions in .where (#16303) There were a couple of dedicated functions in `python/cudf/cudf/utils/dtypes.py` specific to `.where` that could be subsumed by `Column.can_cast_safely`. The minor downside is that we need to cast where's argument to a Column first, but IMO it's probably OK given the deduplication Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16303 --- python/cudf/cudf/core/_internals/where.py | 78 ++++++++++++++---- python/cudf/cudf/utils/dtypes.py | 96 +---------------------- 2 files changed, 62 insertions(+), 112 deletions(-) diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 4a36be76b6d..6003a0f6aea 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -9,12 +9,7 @@ import cudf from cudf.api.types import _is_non_decimal_numeric_dtype, is_scalar from cudf.core.dtypes import CategoricalDtype -from cudf.utils.dtypes import ( - _can_cast, - _dtype_can_hold_element, - find_common_type, - is_mixed_with_object_dtype, -) +from cudf.utils.dtypes import find_common_type, is_mixed_with_object_dtype if TYPE_CHECKING: from cudf._typing import ScalarLike @@ -44,6 +39,8 @@ def _check_and_cast_columns_with_other( inplace: bool, ) -> tuple[ColumnBase, ScalarLike | ColumnBase]: # Returns type-casted `source_col` & `other` based on `inplace`. + from cudf.core.column import as_column + source_dtype = source_col.dtype if isinstance(source_dtype, CategoricalDtype): return _normalize_categorical(source_col, other) @@ -84,17 +81,9 @@ def _check_and_cast_columns_with_other( ) return _normalize_categorical(source_col, other.astype(source_dtype)) - if ( - _is_non_decimal_numeric_dtype(source_dtype) - and not other_is_scalar # can-cast fails for Python scalars - and _can_cast(other, source_dtype) - ): - common_dtype = source_dtype - elif ( - isinstance(source_col, cudf.core.column.NumericalColumn) - and other_is_scalar - and _dtype_can_hold_element(source_dtype, other) - ): + if _is_non_decimal_numeric_dtype(source_dtype) and as_column( + other + ).can_cast_safely(source_dtype): common_dtype = source_dtype else: common_dtype = find_common_type( @@ -130,3 +119,58 @@ def _make_categorical_like(result, column): ordered=column.ordered, ) return result + + +def _can_cast(from_dtype, to_dtype): + """ + Utility function to determine if we can cast + from `from_dtype` to `to_dtype`. This function primarily calls + `np.can_cast` but with some special handling around + cudf specific dtypes. + """ + if cudf.utils.utils.is_na_like(from_dtype): + return True + if isinstance(from_dtype, type): + from_dtype = cudf.dtype(from_dtype) + if isinstance(to_dtype, type): + to_dtype = cudf.dtype(to_dtype) + + # TODO : Add precision & scale checking for + # decimal types in future + + if isinstance(from_dtype, cudf.core.dtypes.DecimalDtype): + if isinstance(to_dtype, cudf.core.dtypes.DecimalDtype): + return True + elif isinstance(to_dtype, np.dtype): + if to_dtype.kind in {"i", "f", "u", "U", "O"}: + return True + else: + return False + elif isinstance(from_dtype, np.dtype): + if isinstance(to_dtype, np.dtype): + return np.can_cast(from_dtype, to_dtype) + elif isinstance(to_dtype, cudf.core.dtypes.DecimalDtype): + if from_dtype.kind in {"i", "f", "u", "U", "O"}: + return True + else: + return False + elif isinstance(to_dtype, cudf.core.types.CategoricalDtype): + return True + else: + return False + elif isinstance(from_dtype, cudf.core.dtypes.ListDtype): + # TODO: Add level based checks too once casting of + # list columns is supported + if isinstance(to_dtype, cudf.core.dtypes.ListDtype): + return np.can_cast(from_dtype.leaf_type, to_dtype.leaf_type) + else: + return False + elif isinstance(from_dtype, cudf.core.dtypes.CategoricalDtype): + if isinstance(to_dtype, cudf.core.dtypes.CategoricalDtype): + return True + elif isinstance(to_dtype, np.dtype): + return np.can_cast(from_dtype._categories.dtype, to_dtype) + else: + return False + else: + return np.can_cast(from_dtype, to_dtype) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 59e5ec1df04..af912bee342 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -10,8 +10,6 @@ from pandas.core.dtypes.common import infer_dtype_from_object import cudf -from cudf._typing import DtypeObj -from cudf.api.types import is_bool, is_float, is_integer """Map numpy dtype to pyarrow types. Note that np.bool_ bitwidth (8) is different from pa.bool_ (1). Special @@ -584,61 +582,6 @@ def _dtype_pandas_compatible(dtype): return dtype -def _can_cast(from_dtype, to_dtype): - """ - Utility function to determine if we can cast - from `from_dtype` to `to_dtype`. This function primarily calls - `np.can_cast` but with some special handling around - cudf specific dtypes. - """ - if cudf.utils.utils.is_na_like(from_dtype): - return True - if isinstance(from_dtype, type): - from_dtype = cudf.dtype(from_dtype) - if isinstance(to_dtype, type): - to_dtype = cudf.dtype(to_dtype) - - # TODO : Add precision & scale checking for - # decimal types in future - - if isinstance(from_dtype, cudf.core.dtypes.DecimalDtype): - if isinstance(to_dtype, cudf.core.dtypes.DecimalDtype): - return True - elif isinstance(to_dtype, np.dtype): - if to_dtype.kind in {"i", "f", "u", "U", "O"}: - return True - else: - return False - elif isinstance(from_dtype, np.dtype): - if isinstance(to_dtype, np.dtype): - return np.can_cast(from_dtype, to_dtype) - elif isinstance(to_dtype, cudf.core.dtypes.DecimalDtype): - if from_dtype.kind in {"i", "f", "u", "U", "O"}: - return True - else: - return False - elif isinstance(to_dtype, cudf.core.types.CategoricalDtype): - return True - else: - return False - elif isinstance(from_dtype, cudf.core.dtypes.ListDtype): - # TODO: Add level based checks too once casting of - # list columns is supported - if isinstance(to_dtype, cudf.core.dtypes.ListDtype): - return np.can_cast(from_dtype.leaf_type, to_dtype.leaf_type) - else: - return False - elif isinstance(from_dtype, cudf.core.dtypes.CategoricalDtype): - if isinstance(to_dtype, cudf.core.dtypes.CategoricalDtype): - return True - elif isinstance(to_dtype, np.dtype): - return np.can_cast(from_dtype._categories.dtype, to_dtype) - else: - return False - else: - return np.can_cast(from_dtype, to_dtype) - - def _maybe_convert_to_default_type(dtype): """Convert `dtype` to default if specified by user. @@ -661,44 +604,7 @@ def _maybe_convert_to_default_type(dtype): return dtype -def _dtype_can_hold_range(rng: range, dtype: np.dtype) -> bool: - if not len(rng): - return True - return np.can_cast(rng[0], dtype) and np.can_cast(rng[-1], dtype) - - -def _dtype_can_hold_element(dtype: np.dtype, element) -> bool: - if dtype.kind in {"i", "u"}: - if isinstance(element, range): - if _dtype_can_hold_range(element, dtype): - return True - return False - - elif is_integer(element) or ( - is_float(element) and element.is_integer() - ): - info = np.iinfo(dtype) - if info.min <= element <= info.max: - return True - return False - - elif dtype.kind == "f": - if is_integer(element) or is_float(element): - casted = dtype.type(element) - if np.isnan(casted) or casted == element: - return True - # otherwise e.g. overflow see TestCoercionFloat32 - return False - - elif dtype.kind == "b": - if is_bool(element): - return True - return False - - raise NotImplementedError(f"Unsupported dtype: {dtype}") - - -def _get_base_dtype(dtype: DtypeObj) -> DtypeObj: +def _get_base_dtype(dtype: pd.DatetimeTZDtype) -> np.dtype: # TODO: replace the use of this function with just `dtype.base` # when Pandas 2.1.0 is the minimum version we support: # https://github.com/pandas-dev/pandas/pull/52706 From debbef0bc12f523054740432983030dd0b24f9c4 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Fri, 19 Jul 2024 15:12:56 +0100 Subject: [PATCH 31/53] Update vendored thread_pool implementation (#16210) Since we introduced the vendored thread_pool in #8752, upstream has introduced some new features, and particularly now uses condition variables/notification to handle when there are no tasks in the queue. This avoids the issue described in #16209 where the thread pool by default artificially introduces a delay of 1000microseconds to all tasks whenever the task queue is emptied. - Closes #16209 Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - Bradley Dice (https://github.com/bdice) - Robert Maynard (https://github.com/robertmaynard) URL: https://github.com/rapidsai/cudf/pull/16210 --- cpp/CMakeLists.txt | 4 +- .../groupby/group_max_multithreaded.cpp | 10 +- .../io/orc/orc_reader_multithreaded.cpp | 26 +- .../io/parquet/parquet_reader_multithread.cpp | 26 +- cpp/cmake/thirdparty/get_thread_pool.cmake | 31 ++ cpp/include/cudf/utilities/thread_pool.hpp | 381 ------------------ cpp/src/io/utilities/file_io_utilities.cpp | 6 +- cpp/src/io/utilities/file_io_utilities.hpp | 7 +- 8 files changed, 66 insertions(+), 425 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_thread_pool.cmake delete mode 100644 cpp/include/cudf/utilities/thread_pool.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 903cff27be4..65347bd6689 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -216,6 +216,8 @@ include(cmake/thirdparty/get_fmt.cmake) include(cmake/thirdparty/get_spdlog.cmake) # find nanoarrow include(cmake/thirdparty/get_nanoarrow.cmake) +# find thread_pool +include(cmake/thirdparty/get_thread_pool.cmake) # Workaround until https://github.com/rapidsai/rapids-cmake/issues/176 is resolved if(NOT BUILD_SHARED_LIBS) @@ -804,7 +806,7 @@ add_dependencies(cudf jitify_preprocess_run) # Specify the target module library dependencies target_link_libraries( cudf - PUBLIC ${ARROW_LIBRARIES} CCCL::CCCL rmm::rmm + PUBLIC ${ARROW_LIBRARIES} CCCL::CCCL rmm::rmm $ PRIVATE $ cuco::cuco ZLIB::ZLIB nvcomp::nvcomp kvikio::kvikio $ nanoarrow ) diff --git a/cpp/benchmarks/groupby/group_max_multithreaded.cpp b/cpp/benchmarks/groupby/group_max_multithreaded.cpp index 3b8faba618f..bf1a1a5fcf7 100644 --- a/cpp/benchmarks/groupby/group_max_multithreaded.cpp +++ b/cpp/benchmarks/groupby/group_max_multithreaded.cpp @@ -20,8 +20,8 @@ #include #include #include -#include +#include #include template @@ -58,7 +58,7 @@ void bench_groupby_max_multithreaded(nvbench::state& state, nvbench::type_list> requests(num_threads); for (auto& thread_requests : requests) { @@ -75,10 +75,8 @@ void bench_groupby_max_multithreaded(nvbench::state& state, nvbench::type_list #include #include -#include +#include #include #include @@ -90,7 +90,7 @@ void BM_orc_multithreaded_read_common(nvbench::state& state, auto const num_threads = state.get_int64("num_threads"); auto streams = cudf::detail::fork_streams(cudf::get_default_stream(), num_threads); - cudf::detail::thread_pool threads(num_threads); + BS::thread_pool threads(num_threads); auto [source_sink_vector, total_file_size, num_files] = write_file_data(state, d_types); std::vector source_info_vector; @@ -112,13 +112,11 @@ void BM_orc_multithreaded_read_common(nvbench::state& state, cudf::io::read_orc(read_opts, stream, rmm::mr::get_current_device_resource()); }; - threads.paused = true; - for (size_t i = 0; i < num_files; ++i) { - threads.submit(read_func, i); - } + threads.pause(); + threads.detach_sequence(decltype(num_files){0}, num_files, read_func); timer.start(); - threads.paused = false; - threads.wait_for_tasks(); + threads.unpause(); + threads.wait(); cudf::detail::join_streams(streams, cudf::get_default_stream()); timer.stop(); }); @@ -170,7 +168,7 @@ void BM_orc_multithreaded_read_chunked_common(nvbench::state& state, size_t const output_limit = state.get_int64("output_limit"); auto streams = cudf::detail::fork_streams(cudf::get_default_stream(), num_threads); - cudf::detail::thread_pool threads(num_threads); + BS::thread_pool threads(num_threads); auto [source_sink_vector, total_file_size, num_files] = write_file_data(state, d_types); std::vector source_info_vector; std::transform(source_sink_vector.begin(), @@ -203,13 +201,11 @@ void BM_orc_multithreaded_read_chunked_common(nvbench::state& state, } while (reader.has_next()); }; - threads.paused = true; - for (size_t i = 0; i < num_files; ++i) { - threads.submit(read_func, i); - } + threads.pause(); + threads.detach_sequence(decltype(num_files){0}, num_files, read_func); timer.start(); - threads.paused = false; - threads.wait_for_tasks(); + threads.unpause(); + threads.wait(); cudf::detail::join_streams(streams, cudf::get_default_stream()); timer.stop(); }); diff --git a/cpp/benchmarks/io/parquet/parquet_reader_multithread.cpp b/cpp/benchmarks/io/parquet/parquet_reader_multithread.cpp index b4c8ed78ed8..9e76ebb71ab 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_multithread.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_multithread.cpp @@ -23,10 +23,10 @@ #include #include #include -#include #include +#include #include #include @@ -93,7 +93,7 @@ void BM_parquet_multithreaded_read_common(nvbench::state& state, auto const num_threads = state.get_int64("num_threads"); auto streams = cudf::detail::fork_streams(cudf::get_default_stream(), num_threads); - cudf::detail::thread_pool threads(num_threads); + BS::thread_pool threads(num_threads); auto [source_sink_vector, total_file_size, num_files] = write_file_data(state, d_types); std::vector source_info_vector; @@ -114,13 +114,11 @@ void BM_parquet_multithreaded_read_common(nvbench::state& state, cudf::io::read_parquet(read_opts, stream, rmm::mr::get_current_device_resource()); }; - threads.paused = true; - for (size_t i = 0; i < num_files; ++i) { - threads.submit(read_func, i); - } + threads.pause(); + threads.detach_sequence(decltype(num_files){0}, num_files, read_func); timer.start(); - threads.paused = false; - threads.wait_for_tasks(); + threads.unpause(); + threads.wait(); cudf::detail::join_streams(streams, cudf::get_default_stream()); timer.stop(); }); @@ -176,7 +174,7 @@ void BM_parquet_multithreaded_read_chunked_common(nvbench::state& state, size_t const output_limit = state.get_int64("output_limit"); auto streams = cudf::detail::fork_streams(cudf::get_default_stream(), num_threads); - cudf::detail::thread_pool threads(num_threads); + BS::thread_pool threads(num_threads); auto [source_sink_vector, total_file_size, num_files] = write_file_data(state, d_types); std::vector source_info_vector; std::transform(source_sink_vector.begin(), @@ -207,13 +205,11 @@ void BM_parquet_multithreaded_read_chunked_common(nvbench::state& state, } while (reader.has_next()); }; - threads.paused = true; - for (size_t i = 0; i < num_files; ++i) { - threads.submit(read_func, i); - } + threads.pause(); + threads.detach_sequence(decltype(num_files){0}, num_files, read_func); timer.start(); - threads.paused = false; - threads.wait_for_tasks(); + threads.unpause(); + threads.wait(); cudf::detail::join_streams(streams, cudf::get_default_stream()); timer.stop(); }); diff --git a/cpp/cmake/thirdparty/get_thread_pool.cmake b/cpp/cmake/thirdparty/get_thread_pool.cmake new file mode 100644 index 00000000000..264257c7199 --- /dev/null +++ b/cpp/cmake/thirdparty/get_thread_pool.cmake @@ -0,0 +1,31 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +# This function finds rmm and sets any additional necessary environment variables. +function(find_and_configure_thread_pool) + rapids_cpm_find( + BS_thread_pool 4.1.0 + CPM_ARGS + GIT_REPOSITORY https://github.com/bshoshany/thread-pool.git + GIT_TAG 097aa718f25d44315cadb80b407144ad455ee4f9 + GIT_SHALLOW TRUE + ) + if(NOT TARGET BS_thread_pool) + add_library(BS_thread_pool INTERFACE) + target_include_directories(BS_thread_pool INTERFACE ${BS_thread_pool_SOURCE_DIR}/include) + target_compile_definitions(BS_thread_pool INTERFACE "BS_THREAD_POOL_ENABLE_PAUSE=1") + endif() +endfunction() + +find_and_configure_thread_pool() diff --git a/cpp/include/cudf/utilities/thread_pool.hpp b/cpp/include/cudf/utilities/thread_pool.hpp deleted file mode 100644 index c8c3eb097c4..00000000000 --- a/cpp/include/cudf/utilities/thread_pool.hpp +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright (c) 2021-2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -/** - * Modified from https://github.com/bshoshany/thread-pool - * @copyright Copyright (c) 2021 Barak Shoshany. Licensed under the MIT license. - * See file LICENSE for detail or copy at https://opensource.org/licenses/MIT - */ - -#include // std::atomic -#include // std::chrono -#include // std::int_fast64_t, std::uint_fast32_t -#include // std::function -#include // std::future, std::promise -#include // std::shared_ptr, std::unique_ptr -#include // std::mutex, std::scoped_lock -#include // std::queue -#include // std::this_thread, std::thread -#include // std::decay_t, std::enable_if_t, std::is_void_v, std::invoke_result_t -#include // std::move, std::swap - -namespace cudf { -namespace detail { - -/** - * @brief A C++17 thread pool class. The user submits tasks to be executed into a queue. Whenever a - * thread becomes available, it pops a task from the queue and executes it. Each task is - * automatically assigned a future, which can be used to wait for the task to finish executing - * and/or obtain its eventual return value. - */ -class thread_pool { - using ui32 = int; - - public: - /** - * @brief Construct a new thread pool. - * - * @param _thread_count The number of threads to use. The default value is the total number of - * hardware threads available, as reported by the implementation. With a hyperthreaded CPU, this - * will be twice the number of CPU cores. If the argument is zero, the default value will be used - * instead. - */ - thread_pool(ui32 const& _thread_count = std::thread::hardware_concurrency()) - : thread_count(_thread_count ? _thread_count : std::thread::hardware_concurrency()), - threads(new std::thread[_thread_count ? _thread_count : std::thread::hardware_concurrency()]) - { - create_threads(); - } - - /** - * @brief Destruct the thread pool. Waits for all tasks to complete, then destroys all threads. - * Note that if the variable paused is set to true, then any tasks still in the queue will never - * be executed. - */ - ~thread_pool() - { - wait_for_tasks(); - running = false; - destroy_threads(); - } - - /** - * @brief Get the number of tasks currently waiting in the queue to be executed by the threads. - * - * @return The number of queued tasks. - */ - [[nodiscard]] size_t get_tasks_queued() const - { - std::scoped_lock const lock(queue_mutex); - return tasks.size(); - } - - /** - * @brief Get the number of tasks currently being executed by the threads. - * - * @return The number of running tasks. - */ - [[nodiscard]] ui32 get_tasks_running() const { return tasks_total - (ui32)get_tasks_queued(); } - - /** - * @brief Get the total number of unfinished tasks - either still in the queue, or running in a - * thread. - * - * @return The total number of tasks. - */ - [[nodiscard]] ui32 get_tasks_total() const { return tasks_total; } - - /** - * @brief Get the number of threads in the pool. - * - * @return The number of threads. - */ - [[nodiscard]] ui32 get_thread_count() const { return thread_count; } - - /** - * @brief Parallelize a loop by splitting it into blocks, submitting each block separately to the - * thread pool, and waiting for all blocks to finish executing. The loop will be equivalent to: - * for (T i = first_index; i <= last_index; i++) loop(i); - * - * @tparam T The type of the loop index. Should be a signed or unsigned integer. - * @tparam F The type of the function to loop through. - * @param first_index The first index in the loop (inclusive). - * @param last_index The last index in the loop (inclusive). - * @param loop The function to loop through. Should take exactly one argument, the loop index. - * @param num_tasks The maximum number of tasks to split the loop into. The default is to use the - * number of threads in the pool. - */ - template - void parallelize_loop(T first_index, T last_index, F const& loop, ui32 num_tasks = 0) - { - if (num_tasks == 0) num_tasks = thread_count; - if (last_index < first_index) std::swap(last_index, first_index); - size_t total_size = last_index - first_index + 1; - size_t block_size = total_size / num_tasks; - if (block_size == 0) { - block_size = 1; - num_tasks = (ui32)total_size > 1 ? (ui32)total_size : 1; - } - std::atomic blocks_running = 0; - for (ui32 t = 0; t < num_tasks; t++) { - T start = (T)(t * block_size + first_index); - T end = (t == num_tasks - 1) ? last_index : (T)((t + 1) * block_size + first_index - 1); - blocks_running++; - push_task([start, end, &loop, &blocks_running] { - for (T i = start; i <= end; i++) - loop(i); - blocks_running--; - }); - } - while (blocks_running != 0) { - sleep_or_yield(); - } - } - - /** - * @brief Push a function with no arguments or return value into the task queue. - * - * @tparam F The type of the function. - * @param task The function to push. - */ - template - void push_task(F const& task) - { - tasks_total++; - { - std::scoped_lock const lock(queue_mutex); - tasks.push(std::function(task)); - } - } - - /** - * @brief Push a function with arguments, but no return value, into the task queue. - * @details The function is wrapped inside a lambda in order to hide the arguments, as the tasks - * in the queue must be of type std::function, so they cannot have any arguments or return - * value. If no arguments are provided, the other overload will be used, in order to avoid the - * (slight) overhead of using a lambda. - * - * @tparam F The type of the function. - * @tparam A The types of the arguments. - * @param task The function to push. - * @param args The arguments to pass to the function. - */ - template - void push_task(F const& task, A const&... args) - { - push_task([task, args...] { task(args...); }); - } - - /** - * @brief Reset the number of threads in the pool. Waits for all currently running tasks to be - * completed, then destroys all threads in the pool and creates a new thread pool with the new - * number of threads. Any tasks that were waiting in the queue before the pool was reset will then - * be executed by the new threads. If the pool was paused before resetting it, the new pool will - * be paused as well. - * - * @param _thread_count The number of threads to use. The default value is the total number of - * hardware threads available, as reported by the implementation. With a hyperthreaded CPU, this - * will be twice the number of CPU cores. If the argument is zero, the default value will be used - * instead. - */ - void reset(ui32 const& _thread_count = std::thread::hardware_concurrency()) - { - bool was_paused = paused; - paused = true; - wait_for_tasks(); - running = false; - destroy_threads(); - thread_count = _thread_count ? _thread_count : std::thread::hardware_concurrency(); - threads = std::make_unique(thread_count); - paused = was_paused; - create_threads(); - running = true; - } - - /** - * @brief Submit a function with zero or more arguments and a return value into the task queue, - * and get a future for its eventual returned value. - * - * @tparam F The type of the function. - * @tparam A The types of the zero or more arguments to pass to the function. - * @tparam R The return type of the function. - * @param task The function to submit. - * @param args The zero or more arguments to pass to the function. - * @return A future to be used later to obtain the function's returned value, waiting for it to - * finish its execution if needed. - */ - template , std::decay_t...>> - std::future submit(F const& task, A const&... args) - { - std::shared_ptr> promise(new std::promise); - std::future future = promise->get_future(); - push_task([task, args..., promise] { - try { - if constexpr (std::is_void_v) { - task(args...); - promise->set_value(); - } else { - promise->set_value(task(args...)); - } - } catch (...) { - promise->set_exception(std::current_exception()); - }; - }); - return future; - } - - /** - * @brief Wait for tasks to be completed. Normally, this function waits for all tasks, both those - * that are currently running in the threads and those that are still waiting in the queue. - * However, if the variable paused is set to true, this function only waits for the currently - * running tasks (otherwise it would wait forever). To wait for a specific task, use submit() - * instead, and call the wait() member function of the generated future. - */ - void wait_for_tasks() - { - while (true) { - if (!paused) { - if (tasks_total == 0) break; - } else { - if (get_tasks_running() == 0) break; - } - sleep_or_yield(); - } - } - - /** - * @brief An atomic variable indicating to the workers to pause. When set to true, the workers - * temporarily stop popping new tasks out of the queue, although any tasks already executed will - * keep running until they are done. Set to false again to resume popping tasks. - */ - std::atomic paused = false; - - /** - * @brief The duration, in microseconds, that the worker function should sleep for when it cannot - * find any tasks in the queue. If set to 0, then instead of sleeping, the worker function will - * execute std::this_thread::yield() if there are no tasks in the queue. The default value is - * 1000. - */ - ui32 sleep_duration = 1000; - - private: - /** - * @brief Create the threads in the pool and assign a worker to each thread. - */ - void create_threads() - { - for (ui32 i = 0; i < thread_count; i++) { - threads[i] = std::thread(&thread_pool::worker, this); - } - } - - /** - * @brief Destroy the threads in the pool by joining them. - */ - void destroy_threads() - { - for (ui32 i = 0; i < thread_count; i++) { - threads[i].join(); - } - } - - /** - * @brief Try to pop a new task out of the queue. - * - * @param task A reference to the task. Will be populated with a function if the queue is not - * empty. - * @return true if a task was found, false if the queue is empty. - */ - bool pop_task(std::function& task) - { - std::scoped_lock const lock(queue_mutex); - if (tasks.empty()) - return false; - else { - task = std::move(tasks.front()); - tasks.pop(); - return true; - } - } - - /** - * @brief Sleep for sleep_duration microseconds. If that variable is set to zero, yield instead. - * - */ - void sleep_or_yield() - { - if (sleep_duration) - std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); - else - std::this_thread::yield(); - } - - /** - * @brief A worker function to be assigned to each thread in the pool. Continuously pops tasks out - * of the queue and executes them, as long as the atomic variable running is set to true. - */ - void worker() - { - while (running) { - std::function task; - if (!paused && pop_task(task)) { - task(); - tasks_total--; - } else { - sleep_or_yield(); - } - } - } - - /** - * @brief A mutex to synchronize access to the task queue by different threads. - */ - mutable std::mutex queue_mutex; - - /** - * @brief An atomic variable indicating to the workers to keep running. When set to false, the - * workers permanently stop working. - */ - std::atomic running = true; - - /** - * @brief A queue of tasks to be executed by the threads. - */ - std::queue> tasks; - - /** - * @brief The number of threads in the pool. - */ - ui32 thread_count; - - /** - * @brief A smart pointer to manage the memory allocated for the threads. - */ - std::unique_ptr threads; - - /** - * @brief An atomic variable to keep track of the total number of unfinished tasks - either still - * in the queue, or running in a thread. - */ - std::atomic tasks_total = 0; -}; - -} // namespace detail -} // namespace cudf diff --git a/cpp/src/io/utilities/file_io_utilities.cpp b/cpp/src/io/utilities/file_io_utilities.cpp index 9fe5959436d..d7b54399f8d 100644 --- a/cpp/src/io/utilities/file_io_utilities.cpp +++ b/cpp/src/io/utilities/file_io_utilities.cpp @@ -223,7 +223,6 @@ cufile_input_impl::cufile_input_impl(std::string const& filepath) // The benefit from multithreaded read plateaus around 16 threads pool(getenv_or("LIBCUDF_CUFILE_THREAD_COUNT", 16)) { - pool.sleep_duration = 10; } namespace { @@ -232,14 +231,15 @@ template > std::vector> make_sliced_tasks( - F function, DataT* ptr, size_t offset, size_t size, cudf::detail::thread_pool& pool) + F function, DataT* ptr, size_t offset, size_t size, BS::thread_pool& pool) { constexpr size_t default_max_slice_size = 4 * 1024 * 1024; static auto const max_slice_size = getenv_or("LIBCUDF_CUFILE_SLICE_SIZE", default_max_slice_size); auto const slices = make_file_io_slices(size, max_slice_size); std::vector> slice_tasks; std::transform(slices.cbegin(), slices.cend(), std::back_inserter(slice_tasks), [&](auto& slice) { - return pool.submit(function, ptr + slice.offset, slice.size, offset + slice.offset); + return pool.submit_task( + [&] { return function(ptr + slice.offset, slice.size, offset + slice.offset); }); }); return slice_tasks; } diff --git a/cpp/src/io/utilities/file_io_utilities.hpp b/cpp/src/io/utilities/file_io_utilities.hpp index 91ef41fba6e..441bede200d 100644 --- a/cpp/src/io/utilities/file_io_utilities.hpp +++ b/cpp/src/io/utilities/file_io_utilities.hpp @@ -19,8 +19,7 @@ #ifdef CUFILE_FOUND #include -#include - +#include #include #endif @@ -150,7 +149,7 @@ class cufile_input_impl final : public cufile_input { private: cufile_shim const* shim = nullptr; cufile_registered_file const cf_file; - cudf::detail::thread_pool pool; + BS::thread_pool pool; }; /** @@ -167,7 +166,7 @@ class cufile_output_impl final : public cufile_output { private: cufile_shim const* shim = nullptr; cufile_registered_file const cf_file; - cudf::detail::thread_pool pool; + BS::thread_pool pool; }; #else From 8ff27ed5bcaf8fc5fc8d1f546dee30c59861c320 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Fri, 19 Jul 2024 15:15:20 +0100 Subject: [PATCH 32/53] Support Literals in groupby-agg (#16218) To do this, we just need to collect the appropriate aggregation information, and broadcast literals to the correct size. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16218 --- python/cudf_polars/cudf_polars/dsl/expr.py | 15 +++++++++++++++ python/cudf_polars/cudf_polars/dsl/ir.py | 4 ++-- python/cudf_polars/tests/test_groupby.py | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/expr.py b/python/cudf_polars/cudf_polars/dsl/expr.py index f37cb3f475c..a034d55120a 100644 --- a/python/cudf_polars/cudf_polars/dsl/expr.py +++ b/python/cudf_polars/cudf_polars/dsl/expr.py @@ -370,6 +370,10 @@ def do_evaluate( # datatype of pyarrow scalar is correct by construction. return Column(plc.Column.from_scalar(plc.interop.from_arrow(self.value), 1)) + def collect_agg(self, *, depth: int) -> AggInfo: + """Collect information about aggregations in groupbys.""" + return AggInfo([]) + class LiteralColumn(Expr): __slots__ = ("value",) @@ -382,6 +386,13 @@ def __init__(self, dtype: plc.DataType, value: pl.Series) -> None: data = value.to_arrow() self.value = data.cast(dtypes.downcast_arrow_lists(data.type)) + def get_hash(self) -> int: + """Compute a hash of the column.""" + # This is stricter than necessary, but we only need this hash + # for identity in groupby replacements so it's OK. And this + # way we avoid doing potentially expensive compute. + return hash((type(self), self.dtype, id(self.value))) + def do_evaluate( self, df: DataFrame, @@ -393,6 +404,10 @@ def do_evaluate( # datatype of pyarrow array is correct by construction. return Column(plc.interop.from_arrow(self.value)) + def collect_agg(self, *, depth: int) -> AggInfo: + """Collect information about aggregations in groupbys.""" + return AggInfo([]) + class Col(Expr): __slots__ = ("name",) diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index cce0c4a3d94..01834ab75a5 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -514,7 +514,7 @@ def check_agg(agg: expr.Expr) -> int: return max(GroupBy.check_agg(child) for child in agg.children) elif isinstance(agg, expr.Agg): return 1 + max(GroupBy.check_agg(child) for child in agg.children) - elif isinstance(agg, (expr.Len, expr.Col, expr.Literal)): + elif isinstance(agg, (expr.Len, expr.Col, expr.Literal, expr.LiteralColumn)): return 0 else: raise NotImplementedError(f"No handler for {agg=}") @@ -574,7 +574,7 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: results = [ req.evaluate(result_subs, mapping=mapping) for req in self.agg_requests ] - return DataFrame([*result_keys, *results]).slice(self.options.slice) + return DataFrame(broadcast(*result_keys, *results)).slice(self.options.slice) @dataclasses.dataclass diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index b07d8e38217..b650fee5079 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -155,3 +155,20 @@ def test_groupby_nan_minmax_raises(op): q = df.group_by("key").agg(op(pl.col("value"))) assert_ir_translation_raises(q, NotImplementedError) + + +@pytest.mark.parametrize("key", [1, pl.col("key1")]) +@pytest.mark.parametrize( + "expr", + [ + pl.lit(1).alias("value"), + pl.lit([[4, 5, 6]]).alias("value"), + pl.col("float") * (1 - pl.col("int")), + [pl.lit(2).alias("value"), pl.col("float") * 2], + ], +) +def test_groupby_literal_in_agg(df, key, expr): + # check_row_order=False doesn't work for list aggregations + # so just sort by the group key + q = df.group_by(key).agg(expr).sort(key, maintain_order=True) + assert_gpu_result_equal(q) From 9a713e3adb8abb1f41de0445b8ea896fdb48c560 Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:34:16 -0400 Subject: [PATCH 33/53] Migrate lists/count_elements to pylibcudf (#16072) Apart of #15162 Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Thomas Li (https://github.com/lithomas1) URL: https://github.com/rapidsai/cudf/pull/16072 --- python/cudf/cudf/_lib/lists.pyx | 18 +++---------- .../libcudf/lists/count_elements.pxd | 2 +- python/cudf/cudf/_lib/pylibcudf/lists.pxd | 2 ++ python/cudf/cudf/_lib/pylibcudf/lists.pyx | 27 +++++++++++++++++++ .../cudf/cudf/pylibcudf_tests/test_lists.py | 10 +++++++ 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/_lib/lists.pyx b/python/cudf/cudf/_lib/lists.pyx index ceae1b148aa..76f37c3b845 100644 --- a/python/cudf/cudf/_lib/lists.pyx +++ b/python/cudf/cudf/_lib/lists.pyx @@ -8,9 +8,6 @@ from libcpp.utility cimport move from cudf._lib.column cimport Column from cudf._lib.pylibcudf.libcudf.column.column cimport column -from cudf._lib.pylibcudf.libcudf.lists.count_elements cimport ( - count_elements as cpp_count_elements, -) from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( lists_column_view, ) @@ -36,19 +33,10 @@ from cudf._lib.pylibcudf cimport Scalar @acquire_spill_lock() def count_elements(Column col): - - # shared_ptr required because lists_column_view has no default - # ctor - cdef shared_ptr[lists_column_view] list_view = ( - make_shared[lists_column_view](col.view()) + return Column.from_pylibcudf( + pylibcudf.lists.count_elements( + col.to_pylibcudf(mode="read")) ) - cdef unique_ptr[column] c_result - - with nogil: - c_result = move(cpp_count_elements(list_view.get()[0])) - - result = Column.from_unique_ptr(move(c_result)) - return result @acquire_spill_lock() diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/count_elements.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/count_elements.pxd index 38bdd4db0bb..ba57a839fbc 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/count_elements.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/lists/count_elements.pxd @@ -9,4 +9,4 @@ from cudf._lib.pylibcudf.libcudf.lists.lists_column_view cimport ( cdef extern from "cudf/lists/count_elements.hpp" namespace "cudf::lists" nogil: - cdef unique_ptr[column] count_elements(const lists_column_view) except + + cdef unique_ptr[column] count_elements(const lists_column_view&) except + diff --git a/python/cudf/cudf/_lib/pylibcudf/lists.pxd b/python/cudf/cudf/_lib/pylibcudf/lists.pxd index 38a479e4791..38eb575ee8d 100644 --- a/python/cudf/cudf/_lib/pylibcudf/lists.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/lists.pxd @@ -33,3 +33,5 @@ cpdef Column reverse(Column) cpdef Column segmented_gather(Column, Column) cpdef Column extract_list_element(Column, ColumnOrSizeType) + +cpdef Column count_elements(Column) diff --git a/python/cudf/cudf/_lib/pylibcudf/lists.pyx b/python/cudf/cudf/_lib/pylibcudf/lists.pyx index 19c961aa014..ea469642dd5 100644 --- a/python/cudf/cudf/_lib/pylibcudf/lists.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/lists.pyx @@ -17,6 +17,9 @@ from cudf._lib.pylibcudf.libcudf.lists.combine cimport ( concatenate_null_policy, concatenate_rows as cpp_concatenate_rows, ) +from cudf._lib.pylibcudf.libcudf.lists.count_elements cimport ( + count_elements as cpp_count_elements, +) from cudf._lib.pylibcudf.libcudf.lists.extract cimport ( extract_list_element as cpp_extract_list_element, ) @@ -293,3 +296,27 @@ cpdef Column extract_list_element(Column input, ColumnOrSizeType index): index.view() if ColumnOrSizeType is Column else index, )) return Column.from_libcudf(move(c_result)) + + +cpdef Column count_elements(Column input): + """Count the number of rows in each + list element in the given lists column. + For details, see :cpp:func:`count_elements`. + + Parameters + ---------- + input : Column + The input column + + Returns + ------- + Column + A new Column of the lengths of each list element + """ + cdef ListColumnView list_view = input.list_view() + cdef unique_ptr[column] c_result + + with nogil: + c_result = move(cpp_count_elements(list_view.view())) + + return Column.from_libcudf(move(c_result)) diff --git a/python/cudf/cudf/pylibcudf_tests/test_lists.py b/python/cudf/cudf/pylibcudf_tests/test_lists.py index 07ecaed5012..7cfed884f90 100644 --- a/python/cudf/cudf/pylibcudf_tests/test_lists.py +++ b/python/cudf/cudf/pylibcudf_tests/test_lists.py @@ -181,3 +181,13 @@ def test_extract_list_element_column(test_data): expect = pa.array([0, None, None, 7]) assert_column_eq(expect, res) + + +def test_count_elements(test_data): + arr = pa.array(test_data[0][1]) + plc_column = plc.interop.from_arrow(arr) + res = plc.lists.count_elements(plc_column) + + expect = pa.array([1, 1, 0, 3], type=pa.int32()) + + assert_column_eq(expect, res) From 2bbeee95ec338c30c0c876dc6a58376fbb0a5a06 Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Fri, 19 Jul 2024 12:43:49 -0400 Subject: [PATCH 34/53] DOC: use intersphinx mapping in pandas-compat ext (#15846) ~~If https://github.com/rapidsai/cudf/pull/15704 is merged~~ This PR changes the header in the admonition (pandas compat box) to be hyperlinked to the pandas docs instead of just text. See https://raybellwaves.github.io/compatsphinxext/compat.html which is the docs of a minimal repo where I have been testing Authors: - Ray Bell (https://github.com/raybellwaves) - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15846 --- .../source/developer_guide/documentation.md | 2 +- python/cudf/cudf/core/column/lists.py | 12 +++++- python/cudf/cudf/core/column/string.py | 16 ++++---- python/cudf/cudf/core/dataframe.py | 37 ++++++++++--------- python/cudf/cudf/core/frame.py | 10 ++--- python/cudf/cudf/core/groupby/groupby.py | 9 +++-- python/cudf/cudf/core/indexed_frame.py | 28 +++++++------- python/cudf/cudf/core/series.py | 14 +++---- python/cudf/cudf/core/tools/numeric.py | 2 +- python/cudf/cudf/core/window/ewm.py | 2 +- 10 files changed, 72 insertions(+), 60 deletions(-) diff --git a/docs/cudf/source/developer_guide/documentation.md b/docs/cudf/source/developer_guide/documentation.md index c8da689479c..4f5a57fec02 100644 --- a/docs/cudf/source/developer_guide/documentation.md +++ b/docs/cudf/source/developer_guide/documentation.md @@ -164,7 +164,7 @@ The directive should be used inside docstrings like so: Docstring body .. pandas-compat:: - **$API_NAME** + :meth:`pandas.DataFrame.METHOD` Explanation of differences ``` diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index cc15e78314e..46b844413f7 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -646,9 +646,17 @@ def sort_values( dtype: list .. pandas-compat:: - **ListMethods.sort_values** + `pandas.Series.list.sort_values` - The ``inplace`` and ``kind`` arguments are currently not supported. + This method does not exist in pandas but it can be run + as: + + >>> import pandas as pd + >>> s = pd.Series([[3, 2, 1], [2, 4, 3]]) + >>> print(s.apply(sorted)) + 0 [1, 2, 3] + 1 [2, 3, 4] + dtype: object """ if inplace: raise NotImplementedError("`inplace` not currently implemented.") diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 96f9cdfd655..ec95c50f455 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -612,7 +612,7 @@ def extract( dtype: object .. pandas-compat:: - **StringMethods.extract** + :meth:`pandas.Series.str.extract` The `flags` parameter currently only supports re.DOTALL and re.MULTILINE. @@ -738,7 +738,7 @@ def contains( dtype: bool .. pandas-compat:: - **StringMethods.contains** + :meth:`pandas.Series.str.contains` The parameters `case` and `na` are not yet supported and will raise a NotImplementedError if anything other than the default @@ -974,7 +974,7 @@ def replace( dtype: object .. pandas-compat:: - **StringMethods.replace** + :meth:`pandas.Series.str.replace` The parameters `case` and `flags` are not yet supported and will raise a `NotImplementedError` if anything other than the default @@ -2803,7 +2803,7 @@ def partition(self, sep: str = " ", expand: bool = True) -> SeriesOrIndex: ) .. pandas-compat:: - **StringMethods.partition** + :meth:`pandas.Series.str.partition` The parameter `expand` is not yet supported and will raise a `NotImplementedError` if anything other than the default @@ -3527,7 +3527,7 @@ def count(self, pat: str, flags: int = 0) -> SeriesOrIndex: Index([0, 0, 2, 1], dtype='int64') .. pandas-compat:: - **StringMethods.count** + :meth:`pandas.Series.str.count` - `flags` parameter currently only supports re.DOTALL and re.MULTILINE. @@ -3607,7 +3607,7 @@ def findall(self, pat: str, flags: int = 0) -> SeriesOrIndex: dtype: list .. pandas-compat:: - **StringMethods.findall** + :meth:`pandas.Series.str.findall` The `flags` parameter currently only supports re.DOTALL and re.MULTILINE. @@ -3811,7 +3811,7 @@ def endswith(self, pat: str) -> SeriesOrIndex: dtype: bool .. pandas-compat:: - **StringMethods.endswith** + :meth:`pandas.Series.str.endswith` `na` parameter is not yet supported, as cudf uses native strings instead of Python objects. @@ -4264,7 +4264,7 @@ def match( dtype: bool .. pandas-compat:: - **StringMethods.match** + :meth:`pandas.Series.str.match` Parameters `case` and `na` are currently not supported. The `flags` parameter currently only supports re.DOTALL and diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index b3d938829c9..f06e45277e2 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -2750,7 +2750,7 @@ def reindex( Chrome 200 0.02 .. pandas-compat:: - **DataFrame.reindex** + :meth:`pandas.DataFrame.reindex` Note: One difference from Pandas is that ``NA`` is used for rows that do not match, rather than ``NaN``. One side effect of this is @@ -3350,7 +3350,7 @@ def diff(self, periods=1, axis=0): 5 2 5 20 .. pandas-compat:: - **DataFrame.diff** + :meth:`pandas.DataFrame.diff` Diff currently only supports numeric dtype columns. """ @@ -3555,7 +3555,7 @@ def rename( 30 3 6 .. pandas-compat:: - **DataFrame.rename** + :meth:`pandas.DataFrame.rename` * Not Supporting: level @@ -3670,7 +3670,7 @@ def agg(self, aggs, axis=None): ``DataFrame`` is returned. .. pandas-compat:: - **DataFrame.agg** + :meth:`pandas.DataFrame.agg` * Not supporting: ``axis``, ``*args``, ``**kwargs`` @@ -3843,7 +3843,7 @@ def nlargest(self, n, columns, keep="first"): Brunei 434000 12128 BN .. pandas-compat:: - **DataFrame.nlargest** + :meth:`pandas.DataFrame.nlargest` - Only a single column is supported in *columns* """ @@ -3915,7 +3915,7 @@ def nsmallest(self, n, columns, keep="first"): Nauru 337000 182 NR .. pandas-compat:: - **DataFrame.nsmallest** + :meth:`pandas.DataFrame.nsmallest` - Only a single column is supported in *columns* """ @@ -3997,7 +3997,7 @@ def transpose(self): a new (ncol x nrow) dataframe. self is (nrow x ncol) .. pandas-compat:: - **DataFrame.transpose, DataFrame.T** + :meth:`pandas.DataFrame.transpose`, :attr:`pandas.DataFrame.T` Not supporting *copy* because default and only behavior is copy=True @@ -4188,7 +4188,7 @@ def merge( from both sides. .. pandas-compat:: - **DataFrame.merge** + :meth:`pandas.DataFrame.merge` DataFrames merges in cuDF result in non-deterministic row ordering. @@ -4263,7 +4263,7 @@ def join( joined : DataFrame .. pandas-compat:: - **DataFrame.join** + :meth:`pandas.DataFrame.join` - *other* must be a single DataFrame for now. - *on* is not supported yet due to lack of multi-index support. @@ -4385,7 +4385,7 @@ def query(self, expr, local_dict=None): 1 2018-10-08 .. pandas-compat:: - **DataFrame.query** + :meth:`pandas.DataFrame.query` One difference from pandas is that ``query`` currently only supports numeric, datetime, timedelta, or bool dtypes. @@ -5447,10 +5447,11 @@ def from_arrow(cls, table): 2 3 6 .. pandas-compat:: - **DataFrame.from_arrow** + `pandas.DataFrame.from_arrow` - - Does not support automatically setting index column(s) similar - to how ``to_pandas`` works for PyArrow Tables. + This method does not exist in pandas but it is similar to + how :meth:`pyarrow.Table.to_pandas` works for PyArrow Tables i.e. + it does not support automatically setting index column(s). """ index_col = None col_index_names = None @@ -5884,7 +5885,7 @@ def quantile( 0.5 2.5 55.0 .. pandas-compat:: - **DataFrame.quantile** + :meth:`pandas.DataFrame.quantile` One notable difference from Pandas is when DataFrame is of non-numeric types and result is expected to be a Series in case of @@ -6174,7 +6175,7 @@ def count(self, axis=0, numeric_only=False): dtype: int64 .. pandas-compat:: - **DataFrame.count** + :meth:`pandas.DataFrame.count` Parameters currently not supported are `axis` and `numeric_only`. """ @@ -6412,7 +6413,7 @@ def mode(self, axis=0, numeric_only=False, dropna=True): 1 2.0 .. pandas-compat:: - **DataFrame.mode** + :meth:`pandas.DataFrame.transpose` ``axis`` parameter is currently not supported. """ @@ -7594,7 +7595,7 @@ def interleave_columns(self): The interleaved columns as a single column .. pandas-compat:: - **DataFrame.interleave_columns** + `pandas.DataFrame.interleave_columns` This method does not exist in pandas but it can be run as ``pd.Series(np.vstack(df.to_numpy()).reshape((-1,)))``. @@ -7696,7 +7697,7 @@ def eval(self, expr: str, inplace: bool = False, **kwargs): 4 5 2 7 3 .. pandas-compat:: - **DataFrame.eval** + :meth:`pandas.DataFrame.eval` * Additional kwargs are not supported. * Bitwise and logical operators are not dtype-dependent. diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 802751e47ad..111225a5fc2 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -591,7 +591,7 @@ def where(self, cond, other=None, inplace: bool = False) -> Self | None: dtype: int64 .. pandas-compat:: - **DataFrame.where, Series.where** + :meth:`pandas.DataFrame.where`, :meth:`pandas.Series.where` Note that ``where`` treats missing values as falsy, in parallel with pandas treatment of nullable data: @@ -1641,7 +1641,7 @@ def min( 1 .. pandas-compat:: - **DataFrame.min, Series.min** + :meth:`pandas.DataFrame.min`, :meth:`pandas.Series.min` Parameters currently not supported are `level`, `numeric_only`. """ @@ -1689,7 +1689,7 @@ def max( dtype: int64 .. pandas-compat:: - **DataFrame.max, Series.max** + :meth:`pandas.DataFrame.max`, :meth:`pandas.Series.max` Parameters currently not supported are `level`, `numeric_only`. """ @@ -1742,7 +1742,7 @@ def all(self, axis=0, skipna=True, **kwargs): dtype: bool .. pandas-compat:: - **DataFrame.all, Series.all** + :meth:`pandas.DataFrame.all`, :meth:`pandas.Series.all` Parameters currently not supported are `axis`, `bool_only`, `level`. @@ -1795,7 +1795,7 @@ def any(self, axis=0, skipna=True, **kwargs): dtype: bool .. pandas-compat:: - **DataFrame.any, Series.any** + :meth:`pandas.DataFrame.any`, :meth:`pandas.Series.any` Parameters currently not supported are `axis`, `bool_only`, `level`. diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index d2c75715be2..3f91be71f29 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -744,7 +744,8 @@ def _reduce( Computed {op} of values within each group. .. pandas-compat:: - **{cls}.{op}** + :meth:`pandas.core.groupby.DataFrameGroupBy.{op}`, + :meth:`pandas.core.groupby.SeriesGroupBy.{op}` The numeric_only, min_count """ @@ -1482,7 +1483,8 @@ def mult(df): 6 2 6 12 .. pandas-compat:: - **GroupBy.apply** + :meth:`pandas.core.groupby.DataFrameGroupBy.apply`, + :meth:`pandas.core.groupby.SeriesGroupBy.apply` cuDF's ``groupby.apply`` is limited compared to pandas. In some situations, Pandas returns the grouped keys as part of @@ -2358,7 +2360,8 @@ def shift(self, periods=1, freq=None, axis=0, fill_value=None): Object shifted within each group. .. pandas-compat:: - **GroupBy.shift** + :meth:`pandas.core.groupby.DataFrameGroupBy.shift`, + :meth:`pandas.core.groupby.SeriesGroupBy.shift` Parameter ``freq`` is unsupported. """ diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 30b68574960..77675edc0f0 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -497,7 +497,7 @@ def empty(self): True .. pandas-compat:: - **DataFrame.empty, Series.empty** + :attr:`pandas.DataFrame.empty`, :attr:`pandas.Series.empty` If DataFrame/Series contains only `null` values, it is still not considered empty. See the example above. @@ -831,7 +831,7 @@ def replace( 4 4 9 e .. pandas-compat:: - **DataFrame.replace, Series.replace** + :meth:`pandas.DataFrame.replace`, :meth:`pandas.Series.replace` Parameters that are currently not supported are: `limit`, `regex`, `method` @@ -1372,7 +1372,7 @@ def sum( dtype: int64 .. pandas-compat:: - **DataFrame.sum, Series.sum** + :meth:`pandas.DataFrame.sum`, :meth:`pandas.Series.sum` Parameters currently not supported are `level`, `numeric_only`. """ @@ -1433,7 +1433,7 @@ def product( dtype: int64 .. pandas-compat:: - **DataFrame.product, Series.product** + :meth:`pandas.DataFrame.product`, :meth:`pandas.Series.product` Parameters currently not supported are level`, `numeric_only`. """ @@ -1530,7 +1530,7 @@ def median( 17.0 .. pandas-compat:: - **DataFrame.median, Series.median** + :meth:`pandas.DataFrame.median`, :meth:`pandas.Series.median` Parameters currently not supported are `level` and `numeric_only`. """ @@ -1586,7 +1586,7 @@ def std( dtype: float64 .. pandas-compat:: - **DataFrame.std, Series.std** + :meth:`pandas.DataFrame.std`, :meth:`pandas.Series.std` Parameters currently not supported are `level` and `numeric_only` @@ -1645,7 +1645,7 @@ def var( dtype: float64 .. pandas-compat:: - **DataFrame.var, Series.var** + :meth:`pandas.DataFrame.var`, :meth:`pandas.Series.var` Parameters currently not supported are `level` and `numeric_only` @@ -1701,7 +1701,7 @@ def kurtosis(self, axis=0, skipna=True, numeric_only=False, **kwargs): dtype: float64 .. pandas-compat:: - **DataFrame.kurtosis** + :meth:`pandas.DataFrame.kurtosis` Parameters currently not supported are `level` and `numeric_only` """ @@ -1763,7 +1763,7 @@ def skew(self, axis=0, skipna=True, numeric_only=False, **kwargs): dtype: float64 .. pandas-compat:: - **DataFrame.skew, Series.skew, Frame.skew** + :meth:`pandas.DataFrame.skew`, :meth:`pandas.Series.skew` The `axis` parameter is not currently supported. """ @@ -2229,7 +2229,7 @@ def truncate(self, before=None, after=None, axis=0, copy=True): 2021-01-01 23:45:27 1 2 .. pandas-compat:: - **DataFrame.truncate, Series.truncate** + :meth:`pandas.DataFrame.truncate`, :meth:`pandas.Series.truncate` The ``copy`` parameter is only present for API compatibility, but ``copy=False`` is not supported. This method always generates a @@ -2665,7 +2665,7 @@ def sort_index( 2 3 1 .. pandas-compat:: - **DataFrame.sort_index, Series.sort_index** + :meth:`pandas.DataFrame.sort_index`, :meth:`pandas.Series.sort_index` * Not supporting: kind, sort_remaining=False """ @@ -3497,7 +3497,7 @@ def sort_values( 1 1 2 .. pandas-compat:: - **DataFrame.sort_values, Series.sort_values** + :meth:`pandas.DataFrame.sort_values`, :meth:`pandas.Series.sort_values` * Support axis='index' only. * Not supporting: inplace, kind @@ -4008,7 +4008,7 @@ def resample( .. pandas-compat:: - **DataFrame.resample, Series.resample** + :meth:`pandas.DataFrame.resample`, :meth:`pandas.Series.resample` Note that the dtype of the index (or the 'on' column if using 'on=') in the result will be of a frequency closest to the @@ -4564,7 +4564,7 @@ def sample( 1 2 4 .. pandas-compat:: - **DataFrame.sample, Series.sample** + :meth:`pandas.DataFrame.sample`, :meth:`pandas.Series.sample` When sampling from ``axis=0/'index'``, ``random_state`` can be either a numpy random state (``numpy.random.RandomState``) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index e12cc3d52fb..c9d24890d15 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -960,7 +960,7 @@ def reindex(self, *args, **kwargs): dtype: int64 .. pandas-compat:: - **Series.reindex** + :meth:`pandas.Series.reindex` Note: One difference from Pandas is that ``NA`` is used for rows that do not match, rather than ``NaN``. One side effect of this is @@ -1243,7 +1243,7 @@ def map(self, arg, na_action=None) -> "Series": dtype: int64 .. pandas-compat:: - **Series.map** + :meth:`pandas.Series.map` Please note map currently only supports fixed-width numeric type functions. @@ -2094,7 +2094,7 @@ def sort_values( dtype: int64 .. pandas-compat:: - **Series.sort_values** + :meth:`pandas.Series.sort_values` * Support axis='index' only. * The inplace and kind argument is currently unsupported @@ -2550,7 +2550,7 @@ def count(self): 5 .. pandas-compat:: - **Series.count** + :meth:`pandas.Series.count` Parameters currently not supported is `level`. """ @@ -2661,7 +2661,7 @@ def cov(self, other, min_periods=None): -0.015750000000000004 .. pandas-compat:: - **Series.cov** + :meth:`pandas.Series.cov` `min_periods` parameter is not yet supported. """ @@ -3422,7 +3422,7 @@ def rename(self, index=None, copy=True): 'numeric_series' .. pandas-compat:: - **Series.rename** + :meth:`pandas.Series.rename` - Supports scalar values only for changing name attribute - The ``inplace`` and ``level`` is not supported @@ -4702,7 +4702,7 @@ def strftime(self, date_format: str, *args, **kwargs) -> Series: dtype: object .. pandas-compat:: - **series.DatetimeProperties.strftime** + :meth:`pandas.DatetimeIndex.strftime` The following date format identifiers are not yet supported: ``%c``, ``%x``,``%X`` diff --git a/python/cudf/cudf/core/tools/numeric.py b/python/cudf/cudf/core/tools/numeric.py index 466d46f7dca..07158e4ee61 100644 --- a/python/cudf/cudf/core/tools/numeric.py +++ b/python/cudf/cudf/core/tools/numeric.py @@ -80,7 +80,7 @@ def to_numeric(arg, errors="raise", downcast=None): dtype: float64 .. pandas-compat:: - **cudf.to_numeric** + :func:`pandas.to_numeric` An important difference from pandas is that this function does not accept mixed numeric/non-numeric type sequences. diff --git a/python/cudf/cudf/core/window/ewm.py b/python/cudf/cudf/core/window/ewm.py index 21693e106bd..bb153d4b549 100644 --- a/python/cudf/cudf/core/window/ewm.py +++ b/python/cudf/cudf/core/window/ewm.py @@ -56,7 +56,7 @@ class ExponentialMovingWindow(_RollingBase): the equivalent pandas method. .. pandas-compat:: - **cudf.core.window.ExponentialMovingWindow** + :meth:`pandas.DataFrame.ewm` The parameters ``min_periods``, ``ignore_na``, ``axis``, and ``times`` are not yet supported. Behavior is defined only for data that begins From d5ab48d4f2586d2e45234463c1bbe877ce76afe8 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 19 Jul 2024 14:32:54 -0400 Subject: [PATCH 35/53] Use workflow branch 24.08 again (#16314) After updating everything to CUDA 12.5.1, use `shared-workflows@branch-24.08` again. Contributes to https://github.com/rapidsai/build-planning/issues/73 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) - https://github.com/jakirkham URL: https://github.com/rapidsai/cudf/pull/16314 --- .github/workflows/build.yaml | 20 ++++----- .github/workflows/pandas-tests.yaml | 2 +- .github/workflows/pr.yaml | 44 +++++++++---------- .../workflows/pr_issue_status_automation.yml | 6 +-- .github/workflows/test.yaml | 22 +++++----- 5 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 937080572ad..2e5959338b0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' needs: python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -101,7 +101,7 @@ jobs: wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -111,7 +111,7 @@ jobs: wheel-build-cudf-polars: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -123,7 +123,7 @@ jobs: wheel-publish-cudf-polars: needs: wheel-build-cudf-polars secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pandas-tests.yaml b/.github/workflows/pandas-tests.yaml index 1516cb09449..5a937b2f362 100644 --- a/.github/workflows/pandas-tests.yaml +++ b/.github/workflows/pandas-tests.yaml @@ -17,7 +17,7 @@ jobs: pandas-tests: # run the Pandas unit tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and (.CUDA_VER | startswith("12.5.")) )) build_type: nightly diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1fe64e7f318..d5dfc9e1ff5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,41 +34,41 @@ jobs: - pandas-tests - pandas-tests-diff secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 with: build_type: pull-request enable_check_symbols: true conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 with: build_type: pull-request conda-python-cudf-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -76,14 +76,14 @@ jobs: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 with: build_type: pull-request script: "ci/test_python_other.sh" conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -93,7 +93,7 @@ jobs: static-configure: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: pull-request # Use the wheel container so we can skip conda solves and since our @@ -103,7 +103,7 @@ jobs: conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -113,7 +113,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -123,21 +123,21 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: "ci/build_wheel_cudf.sh" wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-cudf-polars: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -146,7 +146,7 @@ jobs: wheel-tests-cudf-polars: needs: wheel-build-cudf-polars secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -157,7 +157,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -166,7 +166,7 @@ jobs: wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -174,7 +174,7 @@ jobs: script: ci/test_wheel_dask_cudf.sh devcontainer: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.08 with: arch: '["amd64"]' cuda: '["12.5"]' @@ -185,7 +185,7 @@ jobs: unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request @@ -194,7 +194,7 @@ jobs: # run the Pandas unit tests using PR branch needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and (.CUDA_VER | startswith("12.5.")) )) build_type: pull-request @@ -204,7 +204,7 @@ jobs: pandas-tests-diff: # diff the results of running the Pandas unit tests and publish a job summary needs: pandas-tests - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: node_type: cpu4 build_type: pull-request diff --git a/.github/workflows/pr_issue_status_automation.yml b/.github/workflows/pr_issue_status_automation.yml index 2a8ebd30993..8ca971dc28d 100644 --- a/.github/workflows/pr_issue_status_automation.yml +++ b/.github/workflows/pr_issue_status_automation.yml @@ -23,7 +23,7 @@ on: jobs: get-project-id: - uses: rapidsai/shared-workflows/.github/workflows/project-get-item-id.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/project-get-item-id.yaml@branch-24.08 if: github.event.pull_request.state == 'open' secrets: inherit permissions: @@ -34,7 +34,7 @@ jobs: update-status: # This job sets the PR and its linked issues to "In Progress" status - uses: rapidsai/shared-workflows/.github/workflows/project-get-set-single-select-field.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/project-get-set-single-select-field.yaml@branch-24.08 if: ${{ github.event.pull_request.state == 'open' && needs.get-project-id.outputs.ITEM_PROJECT_ID != '' }} needs: get-project-id with: @@ -50,7 +50,7 @@ jobs: update-sprint: # This job sets the PR and its linked issues to the current "Weekly Sprint" - uses: rapidsai/shared-workflows/.github/workflows/project-get-set-iteration-field.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/project-get-set-iteration-field.yaml@branch-24.08 if: ${{ github.event.pull_request.state == 'open' && needs.get-project-id.outputs.ITEM_PROJECT_ID != '' }} needs: get-project-id with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 73f8d726e77..36c9088d93c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -25,7 +25,7 @@ jobs: enable_check_symbols: true conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -33,7 +33,7 @@ jobs: sha: ${{ inputs.sha }} conda-cpp-memcheck-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -45,7 +45,7 @@ jobs: run_script: "ci/test_cpp_memcheck.sh" static-configure: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: pull-request # Use the wheel container so we can skip conda solves and since our @@ -54,7 +54,7 @@ jobs: run_script: "ci/configure_cpp_static.sh" conda-python-cudf-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -64,7 +64,7 @@ jobs: conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -73,7 +73,7 @@ jobs: script: "ci/test_python_other.sh" conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -85,7 +85,7 @@ jobs: run_script: "ci/test_java.sh" conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -97,7 +97,7 @@ jobs: run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -106,7 +106,7 @@ jobs: script: ci/test_wheel_cudf.sh wheel-tests-dask-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -117,7 +117,7 @@ jobs: script: ci/test_wheel_dask_cudf.sh unit-tests-cudf-pandas: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: nightly branch: ${{ inputs.branch }} From dc62177a64a5fb4d6521f346ff0f44c2ede740f6 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Fri, 19 Jul 2024 20:17:42 +0100 Subject: [PATCH 36/53] Preserve order in left join for cudf-polars (#16268) Unlike all other joins, polars provides an ordering guarantee for left joins. By default libcudf does not, so we need to order the gather maps in this case. While here, because it requires another hard-coding of `int32` for something that should be `size_type`, expose `type_to_id` in cython and plumb it through. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16268 --- python/cudf/cudf/_lib/pylibcudf/join.pyx | 15 +---- .../libcudf/utilities/type_dispatcher.pxd | 7 +++ python/cudf/cudf/_lib/pylibcudf/types.pyx | 7 ++- python/cudf/cudf/_lib/types.pyx | 4 +- .../cudf_polars/containers/column.py | 3 +- python/cudf_polars/cudf_polars/dsl/ir.py | 58 +++++++++++++++++++ python/cudf_polars/tests/test_join.py | 2 +- 7 files changed, 78 insertions(+), 18 deletions(-) create mode 100644 python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/type_dispatcher.pxd diff --git a/python/cudf/cudf/_lib/pylibcudf/join.pyx b/python/cudf/cudf/_lib/pylibcudf/join.pyx index 308b1b39291..2ded84d84d1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/join.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/join.pyx @@ -10,12 +10,7 @@ from rmm._lib.device_buffer cimport device_buffer from cudf._lib.pylibcudf.libcudf cimport join as cpp_join from cudf._lib.pylibcudf.libcudf.column.column cimport column from cudf._lib.pylibcudf.libcudf.table.table cimport table -from cudf._lib.pylibcudf.libcudf.types cimport ( - data_type, - null_equality, - size_type, - type_id, -) +from cudf._lib.pylibcudf.libcudf.types cimport null_equality from .column cimport Column from .table cimport Table @@ -23,15 +18,11 @@ from .table cimport Table cdef Column _column_from_gather_map(cpp_join.gather_map_type gather_map): # helper to convert a gather map to a Column - cdef device_buffer c_empty - cdef size_type size = dereference(gather_map.get()).size() return Column.from_libcudf( move( make_unique[column]( - data_type(type_id.INT32), - size, - dereference(gather_map.get()).release(), - move(c_empty), + move(dereference(gather_map.get())), + device_buffer(), 0 ) ) diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/type_dispatcher.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/type_dispatcher.pxd new file mode 100644 index 00000000000..890fca3a662 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/utilities/type_dispatcher.pxd @@ -0,0 +1,7 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cudf._lib.pylibcudf.libcudf.types cimport type_id + + +cdef extern from "cudf/utilities/type_dispatcher.hpp" namespace "cudf" nogil: + cdef type_id type_to_id[T]() diff --git a/python/cudf/cudf/_lib/pylibcudf/types.pyx b/python/cudf/cudf/_lib/pylibcudf/types.pyx index 6dbb287f3c4..c45c6071bb3 100644 --- a/python/cudf/cudf/_lib/pylibcudf/types.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/types.pyx @@ -2,7 +2,8 @@ from libc.stdint cimport int32_t -from cudf._lib.pylibcudf.libcudf.types cimport data_type, type_id +from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type, type_id +from cudf._lib.pylibcudf.libcudf.utilities.type_dispatcher cimport type_to_id from cudf._lib.pylibcudf.libcudf.types import type_id as TypeId # no-cython-lint, isort:skip from cudf._lib.pylibcudf.libcudf.types import nan_policy as NanPolicy # no-cython-lint, isort:skip @@ -67,3 +68,7 @@ cdef class DataType: cdef DataType ret = DataType.__new__(DataType, type_id.EMPTY) ret.c_obj = dt return ret + + +SIZE_TYPE = DataType(type_to_id[size_type]()) +SIZE_TYPE_ID = SIZE_TYPE.id() diff --git a/python/cudf/cudf/_lib/types.pyx b/python/cudf/cudf/_lib/types.pyx index fc672caa574..253fdf7b0d9 100644 --- a/python/cudf/cudf/_lib/types.pyx +++ b/python/cudf/cudf/_lib/types.pyx @@ -21,8 +21,6 @@ from cudf._lib.types cimport ( import cudf from cudf._lib import pylibcudf -size_type_dtype = np.dtype("int32") - class TypeId(IntEnum): EMPTY = libcudf_types.type_id.EMPTY @@ -150,6 +148,8 @@ datetime_unit_map = { TypeId.TIMESTAMP_NANOSECONDS: "ns", } +size_type_dtype = LIBCUDF_TO_SUPPORTED_NUMPY_TYPES[pylibcudf.types.SIZE_TYPE_ID] + class Interpolation(IntEnum): LINEAR = ( diff --git a/python/cudf_polars/cudf_polars/containers/column.py b/python/cudf_polars/cudf_polars/containers/column.py index 42aba0fcdc0..02018548b2c 100644 --- a/python/cudf_polars/cudf_polars/containers/column.py +++ b/python/cudf_polars/cudf_polars/containers/column.py @@ -185,8 +185,7 @@ def nan_count(self) -> int: plc.reduce.reduce( plc.unary.is_nan(self.obj), plc.aggregation.sum(), - # TODO: pylibcudf needs to have a SizeType DataType singleton - plc.DataType(plc.TypeId.INT32), + plc.types.SIZE_TYPE, ) ).as_py() return 0 diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 01834ab75a5..0b14530e0ed 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -653,6 +653,59 @@ def _joiners( else: assert_never(how) + def _reorder_maps( + self, + left_rows: int, + lg: plc.Column, + left_policy: plc.copying.OutOfBoundsPolicy, + right_rows: int, + rg: plc.Column, + right_policy: plc.copying.OutOfBoundsPolicy, + ) -> list[plc.Column]: + """ + Reorder gather maps to satisfy polars join order restrictions. + + Parameters + ---------- + left_rows + Number of rows in left table + lg + Left gather map + left_policy + Nullify policy for left map + right_rows + Number of rows in right table + rg + Right gather map + right_policy + Nullify policy for right map + + Returns + ------- + list of reordered left and right gather maps. + + Notes + ----- + For a left join, the polars result preserves the order of the + left keys, and is stable wrt the right keys. For all other + joins, there is no order obligation. + """ + dt = plc.interop.to_arrow(plc.types.SIZE_TYPE) + init = plc.interop.from_arrow(pa.scalar(0, type=dt)) + step = plc.interop.from_arrow(pa.scalar(1, type=dt)) + left_order = plc.copying.gather( + plc.Table([plc.filling.sequence(left_rows, init, step)]), lg, left_policy + ) + right_order = plc.copying.gather( + plc.Table([plc.filling.sequence(right_rows, init, step)]), rg, right_policy + ) + return plc.sorting.stable_sort_by_key( + plc.Table([lg, rg]), + plc.Table([*left_order.columns(), *right_order.columns()]), + [plc.types.Order.ASCENDING, plc.types.Order.ASCENDING], + [plc.types.NullOrder.AFTER, plc.types.NullOrder.AFTER], + ).columns() + def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: """Evaluate and return a dataframe.""" left = self.left.evaluate(cache=cache) @@ -693,6 +746,11 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: result = DataFrame.from_table(table, left.column_names) else: lg, rg = join_fn(left_on.table, right_on.table, null_equality) + if how == "left": + # Order of left table is preserved + lg, rg = self._reorder_maps( + left.num_rows, lg, left_policy, right.num_rows, rg, right_policy + ) if coalesce and how == "inner": right = right.discard_columns(right_on.column_names_set) left = DataFrame.from_table( diff --git a/python/cudf_polars/tests/test_join.py b/python/cudf_polars/tests/test_join.py index 89f6fd3455b..1ffbf3c0ef4 100644 --- a/python/cudf_polars/tests/test_join.py +++ b/python/cudf_polars/tests/test_join.py @@ -53,7 +53,7 @@ def test_join(how, coalesce, join_nulls, join_expr): query = left.join( right, on=join_expr, how=how, join_nulls=join_nulls, coalesce=coalesce ) - assert_gpu_result_equal(query, check_row_order=False) + assert_gpu_result_equal(query, check_row_order=how == "left") def test_cross_join(): From cb570fe6d7dc7ebdd6c8c030916ba27bef277b5e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:45:30 -1000 Subject: [PATCH 37/53] Deprecate dtype= parameter in reduction methods (#16313) In terms of pandas alignment, this argument doesn't exist in reduction ops. Additionally, the same result can be easily achieved by calling `astype` after the operation, and it appears libcudf does not support any arbitrary casting to an output type. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16313 --- python/cudf/cudf/_lib/reduce.pyx | 15 ++++++++++----- python/cudf/cudf/core/column/column.py | 11 ++++++++--- python/cudf/cudf/core/column/datetime.py | 9 +++------ python/cudf/cudf/core/column/numerical.py | 17 +++++++++-------- python/cudf/cudf/core/column/numerical_base.py | 11 +++-------- python/cudf/cudf/core/column/timedelta.py | 7 +++---- python/cudf/cudf/tests/test_reductions.py | 15 +++++++++------ 7 files changed, 45 insertions(+), 40 deletions(-) diff --git a/python/cudf/cudf/_lib/reduce.pyx b/python/cudf/cudf/_lib/reduce.pyx index 56bfa0ba332..64634b7a6f9 100644 --- a/python/cudf/cudf/_lib/reduce.pyx +++ b/python/cudf/cudf/_lib/reduce.pyx @@ -1,4 +1,5 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +import warnings import cudf from cudf.core.buffer import acquire_spill_lock @@ -26,11 +27,15 @@ def reduce(reduction_op, Column incol, dtype=None, **kwargs): A numpy data type to use for the output, defaults to the same type as the input column """ - - col_dtype = ( - dtype if dtype is not None - else incol._reduction_result_dtype(reduction_op) - ) + if dtype is not None: + warnings.warn( + "dtype is deprecated and will be remove in a future release. " + "Cast the result (e.g. .astype) after the operation instead.", + FutureWarning + ) + col_dtype = dtype + else: + col_dtype = incol._reduction_result_dtype(reduction_op) # check empty case if len(incol) <= incol.null_count: diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 9467bbeed15..5e77aa87e4e 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -261,7 +261,7 @@ def all(self, skipna: bool = True) -> bool: if self.null_count == self.size: return True - return libcudf.reduce.reduce("all", self, dtype=np.bool_) + return libcudf.reduce.reduce("all", self) def any(self, skipna: bool = True) -> bool: # Early exit for fast cases. @@ -271,7 +271,7 @@ def any(self, skipna: bool = True) -> bool: elif skipna and self.null_count == self.size: return False - return libcudf.reduce.reduce("any", self, dtype=np.bool_) + return libcudf.reduce.reduce("any", self) def dropna(self) -> Self: if self.has_nulls(): @@ -1305,7 +1305,10 @@ def _reduce( skipna=skipna, min_count=min_count ) if isinstance(preprocessed, ColumnBase): - return libcudf.reduce.reduce(op, preprocessed, **kwargs) + dtype = kwargs.pop("dtype", None) + return libcudf.reduce.reduce( + op, preprocessed, dtype=dtype, **kwargs + ) return preprocessed def _process_for_reduction( @@ -1336,6 +1339,8 @@ def _reduction_result_dtype(self, reduction_op: str) -> Dtype: Determine the correct dtype to pass to libcudf based on the input dtype, data dtype, and specific reduction op """ + if reduction_op in {"any", "all"}: + return np.dtype(np.bool_) return self.dtype def _with_type_metadata(self: ColumnBase, dtype: Dtype) -> ColumnBase: diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 004a059af95..a4538179415 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -485,13 +485,11 @@ def as_string_column(self) -> cudf.core.column.StringColumn: format = format.split(" ")[0] return self.strftime(format) - def mean( - self, skipna=None, min_count: int = 0, dtype=np.float64 - ) -> ScalarLike: + def mean(self, skipna=None, min_count: int = 0) -> ScalarLike: return pd.Timestamp( cast( "cudf.core.column.NumericalColumn", self.astype("int64") - ).mean(skipna=skipna, min_count=min_count, dtype=dtype), + ).mean(skipna=skipna, min_count=min_count), unit=self.time_unit, ).as_unit(self.time_unit) @@ -499,12 +497,11 @@ def std( self, skipna: bool | None = None, min_count: int = 0, - dtype: Dtype = np.float64, ddof: int = 1, ) -> pd.Timedelta: return pd.Timedelta( cast("cudf.core.column.NumericalColumn", self.astype("int64")).std( - skipna=skipna, min_count=min_count, dtype=dtype, ddof=ddof + skipna=skipna, min_count=min_count, ddof=ddof ) * _unit_to_nanoseconds_conversion[self.time_unit], ).as_unit(self.time_unit) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index cea68c88c90..ba080863722 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -395,7 +395,7 @@ def all(self, skipna: bool = True) -> bool: if result_col.null_count == result_col.size: return True - return libcudf.reduce.reduce("all", result_col, dtype=np.bool_) + return libcudf.reduce.reduce("all", result_col) def any(self, skipna: bool = True) -> bool: # Early exit for fast cases. @@ -406,7 +406,7 @@ def any(self, skipna: bool = True) -> bool: elif skipna and result_col.null_count == result_col.size: return False - return libcudf.reduce.reduce("any", result_col, dtype=np.bool_) + return libcudf.reduce.reduce("any", result_col) @functools.cached_property def nan_count(self) -> int: @@ -684,15 +684,16 @@ def to_pandas( return super().to_pandas(nullable=nullable, arrow_type=arrow_type) def _reduction_result_dtype(self, reduction_op: str) -> Dtype: - col_dtype = self.dtype if reduction_op in {"sum", "product"}: - col_dtype = ( - col_dtype if col_dtype.kind == "f" else np.dtype("int64") - ) + if self.dtype.kind == "f": + return self.dtype + return np.dtype("int64") elif reduction_op == "sum_of_squares": - col_dtype = np.result_dtype(col_dtype, np.dtype("uint64")) + return np.result_dtype(self.dtype, np.dtype("uint64")) + elif reduction_op in {"var", "std", "mean"}: + return np.dtype("float64") - return col_dtype + return super()._reduction_result_dtype(reduction_op) def _normalize_find_and_replace_input( diff --git a/python/cudf/cudf/core/column/numerical_base.py b/python/cudf/cudf/core/column/numerical_base.py index 95c78c5efcb..f41010062c8 100644 --- a/python/cudf/cudf/core/column/numerical_base.py +++ b/python/cudf/cudf/core/column/numerical_base.py @@ -144,32 +144,27 @@ def mean( self, skipna: bool | None = None, min_count: int = 0, - dtype=np.float64, ): - return self._reduce( - "mean", skipna=skipna, min_count=min_count, dtype=dtype - ) + return self._reduce("mean", skipna=skipna, min_count=min_count) def var( self, skipna: bool | None = None, min_count: int = 0, - dtype=np.float64, ddof=1, ): return self._reduce( - "var", skipna=skipna, min_count=min_count, dtype=dtype, ddof=ddof + "var", skipna=skipna, min_count=min_count, ddof=ddof ) def std( self, skipna: bool | None = None, min_count: int = 0, - dtype=np.float64, ddof=1, ): return self._reduce( - "std", skipna=skipna, min_count=min_count, dtype=dtype, ddof=ddof + "std", skipna=skipna, min_count=min_count, ddof=ddof ) def median(self, skipna: bool | None = None) -> NumericalBaseColumn: diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index 36d7d9f9614..59ea1cc002c 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -287,11 +287,11 @@ def as_timedelta_column(self, dtype: Dtype) -> TimeDeltaColumn: return self return libcudf.unary.cast(self, dtype=dtype) - def mean(self, skipna=None, dtype: Dtype = np.float64) -> pd.Timedelta: + def mean(self, skipna=None) -> pd.Timedelta: return pd.Timedelta( cast( "cudf.core.column.NumericalColumn", self.astype("int64") - ).mean(skipna=skipna, dtype=dtype), + ).mean(skipna=skipna), unit=self.time_unit, ).as_unit(self.time_unit) @@ -345,12 +345,11 @@ def std( self, skipna: bool | None = None, min_count: int = 0, - dtype: Dtype = np.float64, ddof: int = 1, ) -> pd.Timedelta: return pd.Timedelta( cast("cudf.core.column.NumericalColumn", self.astype("int64")).std( - skipna=skipna, min_count=min_count, ddof=ddof, dtype=dtype + skipna=skipna, min_count=min_count, ddof=ddof ), unit=self.time_unit, ).as_unit(self.time_unit) diff --git a/python/cudf/cudf/tests/test_reductions.py b/python/cudf/cudf/tests/test_reductions.py index 1247fa362ce..8be6463c699 100644 --- a/python/cudf/cudf/tests/test_reductions.py +++ b/python/cudf/cudf/tests/test_reductions.py @@ -248,16 +248,11 @@ def test_sum_masked(nelem): def test_sum_boolean(): s = Series(np.arange(100000)) - got = (s > 1).sum(dtype=np.int32) + got = (s > 1).sum() expect = 99998 assert expect == got - got = (s > 1).sum(dtype=np.bool_) - expect = True - - assert expect == got - def test_date_minmax(): np_data = np.random.normal(size=10**3) @@ -371,3 +366,11 @@ def test_reduction_column_multiindex(): result = df.mean() expected = df.to_pandas().mean() assert_eq(result, expected) + + +@pytest.mark.parametrize("op", ["sum", "product"]) +def test_dtype_deprecated(op): + ser = cudf.Series(range(5)) + with pytest.warns(FutureWarning): + result = getattr(ser, op)(dtype=np.dtype(np.int8)) + assert isinstance(result, np.int8) From 3df4ac28423b99e4dd88570da8d55e2e5af2e1bc Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:46:18 -1000 Subject: [PATCH 38/53] Remove squeeze argument from groupby (#16312) In pandas, this argument was deprecated in pandas 1.x and removed in pandas 2.x. xref https://github.com/pandas-dev/pandas/pull/33218 Looks like in cudf this argument was never implemented, so to align with pandas, I think it should be OK to just remove this argument Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16312 --- python/cudf/cudf/core/dataframe.py | 2 -- python/cudf/cudf/core/indexed_frame.py | 6 ------ python/cudf/cudf/core/series.py | 2 -- 3 files changed, 10 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index f06e45277e2..8f8baec0af4 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -4306,7 +4306,6 @@ def groupby( as_index=True, sort=no_default, group_keys=False, - squeeze=False, observed=True, dropna=True, ): @@ -4317,7 +4316,6 @@ def groupby( as_index, sort, group_keys, - squeeze, observed, dropna, ) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 77675edc0f0..576596f6f7d 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -5249,7 +5249,6 @@ def groupby( as_index=True, sort=no_default, group_keys=False, - squeeze=False, observed=True, dropna=True, ): @@ -5259,11 +5258,6 @@ def groupby( if axis not in (0, "index"): raise NotImplementedError("axis parameter is not yet implemented") - if squeeze is not False: - raise NotImplementedError( - "squeeze parameter is not yet implemented" - ) - if not observed: raise NotImplementedError( "observed parameter is not yet implemented" diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index c9d24890d15..baaa2eb46a1 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -3368,7 +3368,6 @@ def groupby( as_index=True, sort=no_default, group_keys=False, - squeeze=False, observed=True, dropna=True, ): @@ -3379,7 +3378,6 @@ def groupby( as_index, sort, group_keys, - squeeze, observed, dropna, ) From 18f5fe0010fd42f604a340cd025a9ca9e122c6f5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:41:39 -0700 Subject: [PATCH 39/53] Fix polars for 1.2.1 (#16316) I think Polars made a breaking change in a patch release. At least the error we're getting looks like the error from https://github.com/pola-rs/polars/pull/17606. Authors: - Thomas Li (https://github.com/lithomas1) Approvers: - Lawrence Mitchell (https://github.com/wence-) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16316 --- python/cudf_polars/cudf_polars/utils/versions.py | 1 + python/cudf_polars/tests/test_groupby.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index a9ac14c25aa..9807cffb384 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -15,6 +15,7 @@ POLARS_VERSION_GE_10 = POLARS_VERSION >= parse("1.0") POLARS_VERSION_GE_11 = POLARS_VERSION >= parse("1.1") POLARS_VERSION_GE_12 = POLARS_VERSION >= parse("1.2") +POLARS_VERSION_GE_121 = POLARS_VERSION >= parse("1.2.1") POLARS_VERSION_GT_10 = POLARS_VERSION > parse("1.0") POLARS_VERSION_GT_11 = POLARS_VERSION > parse("1.1") POLARS_VERSION_GT_12 = POLARS_VERSION > parse("1.2") diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index b650fee5079..a75825ef3d3 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -157,7 +157,18 @@ def test_groupby_nan_minmax_raises(op): assert_ir_translation_raises(q, NotImplementedError) -@pytest.mark.parametrize("key", [1, pl.col("key1")]) +@pytest.mark.parametrize( + "key", + [ + pytest.param( + 1, + marks=pytest.mark.xfail( + versions.POLARS_VERSION_GE_121, reason="polars 1.2.1 disallows this" + ), + ), + pl.col("key1"), + ], +) @pytest.mark.parametrize( "expr", [ From fa0d89d9b4b4152b919999b5f01b1e68407469c5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:46:28 -1000 Subject: [PATCH 40/53] Clean unneeded/redudant dtype utils (#16309) * Replace `min_scalar_type` with `min_signed_type` (the former just called the latter) * Replace `numeric_normalize_types` with `find_common_dtype` followed by a column `astype` * Removed `_NUMPY_SCTYPES` with just hardcoding the integer/floating types or using `np.integer`/`np.floating` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16309 --- python/cudf/cudf/core/column/column.py | 6 +++--- python/cudf/cudf/core/column/numerical.py | 12 +++++++---- python/cudf/cudf/core/dataframe.py | 22 ++++--------------- python/cudf/cudf/core/index.py | 22 +++++++++---------- python/cudf/cudf/utils/dtypes.py | 26 ++++++----------------- 5 files changed, 32 insertions(+), 56 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 5e77aa87e4e..89f0f79cb7c 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -71,7 +71,7 @@ get_time_unit, is_column_like, is_mixed_with_object_dtype, - min_scalar_type, + min_signed_type, min_unsigned_type, ) from cudf.utils.utils import _array_ufunc, mask_dtype @@ -1356,7 +1356,7 @@ def _label_encoding( self, cats: ColumnBase, dtype: Dtype | None = None, - na_sentinel: ScalarLike | None = None, + na_sentinel: cudf.Scalar | None = None, ): """ Convert each value in `self` into an integer code, with `cats` @@ -1396,7 +1396,7 @@ def _return_sentinel_column(): return as_column(na_sentinel, dtype=dtype, length=len(self)) if dtype is None: - dtype = min_scalar_type(max(len(cats), na_sentinel), 8) + dtype = min_signed_type(max(len(cats), na_sentinel.value), 8) if is_mixed_with_object_dtype(self, cats): return _return_sentinel_column() diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index ba080863722..b55284f1aff 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -29,10 +29,10 @@ from cudf.core.mixins import BinaryOperand from cudf.errors import MixedTypeError from cudf.utils.dtypes import ( + find_common_type, min_column_type, min_signed_type, np_dtypes_to_pandas_dtypes, - numeric_normalize_types, ) from .numerical_base import NumericalBaseColumn @@ -517,11 +517,15 @@ def find_and_replace( ) elif len(replacement_col) == 1 and len(to_replace_col) == 0: return self.copy() - to_replace_col, replacement_col, replaced = numeric_normalize_types( - to_replace_col, replacement_col, self + common_type = find_common_type( + (to_replace_col.dtype, replacement_col.dtype, self.dtype) ) + replaced = self.astype(common_type) df = cudf.DataFrame._from_data( - {"old": to_replace_col, "new": replacement_col} + { + "old": to_replace_col.astype(common_type), + "new": replacement_col.astype(common_type), + } ) df = df.drop_duplicates(subset=["old"], keep="last", ignore_index=True) if df._data["old"].null_count == 1: diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 8f8baec0af4..904bd4ccb2e 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -83,8 +83,7 @@ cudf_dtype_from_pydata_dtype, find_common_type, is_column_like, - min_scalar_type, - numeric_normalize_types, + min_signed_type, ) from cudf.utils.performance_tracking import _performance_tracking from cudf.utils.utils import GetAttrGetItemMixin, _external_only_api @@ -103,20 +102,6 @@ "var": "nanvar", } -_numeric_reduction_ops = ( - "mean", - "min", - "max", - "sum", - "product", - "prod", - "std", - "var", - "kurtosis", - "kurt", - "skew", -) - def _shape_mismatch_error(x, y): raise ValueError( @@ -923,7 +908,8 @@ def _init_from_series_list(self, data, columns, index): final_index = ensure_index(index) series_lengths = list(map(len, data)) - data = numeric_normalize_types(*data) + common_dtype = find_common_type([obj.dtype for obj in data]) + data = [obj.astype(common_dtype) for obj in data] if series_lengths.count(series_lengths[0]) == len(series_lengths): # Calculating the final dataframe columns by # getting union of all `index` of the Series objects. @@ -8304,7 +8290,7 @@ def _find_common_dtypes_and_categories(non_null_columns, dtypes): )._column.unique() # Set the column dtype to the codes' dtype. The categories # will be re-assigned at the end - dtypes[idx] = min_scalar_type(len(categories[idx])) + dtypes[idx] = min_signed_type(len(categories[idx])) # Otherwise raise an error if columns have different dtypes elif not all(is_dtype_equal(c.dtype, dtypes[idx]) for c in cols): raise ValueError("All columns must be the same type") diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 4164f981fca..cd52a34e35e 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -52,11 +52,9 @@ from cudf.core.single_column_frame import SingleColumnFrame from cudf.utils.docutils import copy_docstring from cudf.utils.dtypes import ( - _NUMPY_SCTYPES, _maybe_convert_to_default_type, find_common_type, is_mixed_with_object_dtype, - numeric_normalize_types, ) from cudf.utils.performance_tracking import _performance_tracking from cudf.utils.utils import _warn_no_dask_cudf, search_range @@ -357,12 +355,10 @@ def _data(self): @_performance_tracking def __contains__(self, item): hash(item) - if isinstance(item, bool) or not isinstance( - item, - tuple( - _NUMPY_SCTYPES["int"] + _NUMPY_SCTYPES["float"] + [int, float] - ), - ): + if not isinstance(item, (np.floating, np.integer, int, float)): + return False + elif isinstance(item, (np.timedelta64, np.datetime64, bool)): + # Cases that would pass the above check return False try: int_item = int(item) @@ -1601,9 +1597,13 @@ def append(self, other): f"either one of them to same dtypes." ) - if isinstance(self._values, cudf.core.column.NumericalColumn): - if self.dtype != other.dtype: - this, other = numeric_normalize_types(self, other) + if ( + isinstance(self._column, cudf.core.column.NumericalColumn) + and self.dtype != other.dtype + ): + common_type = find_common_type((self.dtype, other.dtype)) + this = this.astype(common_type) + other = other.astype(common_type) to_concat = [this, other] return self._concat(to_concat) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index af912bee342..69c268db149 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -89,10 +89,6 @@ BOOL_TYPES = {"bool"} ALL_TYPES = NUMERIC_TYPES | DATETIME_TYPES | TIMEDELTA_TYPES | OTHER_TYPES -# The NumPy scalar types are a bit of a mess as they align with the C types -# so for now we use the `sctypes` dict (although it was made private in 2.0) -_NUMPY_SCTYPES = np.sctypes if hasattr(np, "sctypes") else np._core.sctypes - def np_to_pa_dtype(dtype): """Util to convert numpy dtype to PyArrow dtype.""" @@ -114,12 +110,6 @@ def np_to_pa_dtype(dtype): return _np_pa_dtypes[cudf.dtype(dtype).type] -def numeric_normalize_types(*args): - """Cast all args to a common type using numpy promotion logic""" - dtype = np.result_type(*[a.dtype for a in args]) - return [a.astype(dtype) for a in args] - - def _find_common_type_decimal(dtypes): # Find the largest scale and the largest difference between # precision and scale of the columns to be concatenated @@ -330,32 +320,28 @@ def can_convert_to_column(obj): return is_column_like(obj) or cudf.api.types.is_list_like(obj) -def min_scalar_type(a, min_size=8): - return min_signed_type(a, min_size=min_size) - - -def min_signed_type(x, min_size=8): +def min_signed_type(x: int, min_size: int = 8) -> np.dtype: """ Return the smallest *signed* integer dtype that can represent the integer ``x`` """ - for int_dtype in _NUMPY_SCTYPES["int"]: + for int_dtype in (np.int8, np.int16, np.int32, np.int64): if (cudf.dtype(int_dtype).itemsize * 8) >= min_size: if np.iinfo(int_dtype).min <= x <= np.iinfo(int_dtype).max: - return int_dtype + return np.dtype(int_dtype) # resort to using `int64` and let numpy raise appropriate exception: return np.int64(x).dtype -def min_unsigned_type(x, min_size=8): +def min_unsigned_type(x: int, min_size: int = 8) -> np.dtype: """ Return the smallest *unsigned* integer dtype that can represent the integer ``x`` """ - for int_dtype in _NUMPY_SCTYPES["uint"]: + for int_dtype in (np.uint8, np.uint16, np.uint32, np.uint64): if (cudf.dtype(int_dtype).itemsize * 8) >= min_size: if 0 <= x <= np.iinfo(int_dtype).max: - return int_dtype + return np.dtype(int_dtype) # resort to using `uint64` and let numpy raise appropriate exception: return np.uint64(x).dtype From 910989eb8fb87b2e896aa032260705c27cce71e0 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 19 Jul 2024 15:48:37 -0600 Subject: [PATCH 41/53] Rename gather/scatter benchmarks to clarify coalesced behavior. (#16083) The benchmark names `coalesce_x` and `coalesce_o` are not very clear. This PR renames them to `coalesced` and `shuffled`. This was discussed with @GregoryKimball. Authors: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Karthikeyan (https://github.com/karthikeyann) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/16083 --- cpp/benchmarks/copying/gather.cu | 6 +++--- cpp/benchmarks/copying/scatter.cu | 6 +++--- cpp/benchmarks/lists/copying/scatter_lists.cu | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/benchmarks/copying/gather.cu b/cpp/benchmarks/copying/gather.cu index eeb0149fb3a..985166f7298 100644 --- a/cpp/benchmarks/copying/gather.cu +++ b/cpp/benchmarks/copying/gather.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,5 +71,5 @@ void BM_gather(benchmark::State& state) ->Ranges({{1 << 10, 1 << 26}, {1, 8}}) \ ->UseManualTime(); -GBM_BENCHMARK_DEFINE(double_coalesce_x, double, true); -GBM_BENCHMARK_DEFINE(double_coalesce_o, double, false); +GBM_BENCHMARK_DEFINE(double_coalesced, double, true); +GBM_BENCHMARK_DEFINE(double_shuffled, double, false); diff --git a/cpp/benchmarks/copying/scatter.cu b/cpp/benchmarks/copying/scatter.cu index a521dc82739..c27480b69f4 100644 --- a/cpp/benchmarks/copying/scatter.cu +++ b/cpp/benchmarks/copying/scatter.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -74,5 +74,5 @@ void BM_scatter(benchmark::State& state) ->Ranges({{1 << 10, 1 << 25}, {1, 8}}) \ ->UseManualTime(); -SBM_BENCHMARK_DEFINE(double_coalesce_x, double, true); -SBM_BENCHMARK_DEFINE(double_coalesce_o, double, false); +SBM_BENCHMARK_DEFINE(double_coalesced, double, true); +SBM_BENCHMARK_DEFINE(double_shuffled, double, false); diff --git a/cpp/benchmarks/lists/copying/scatter_lists.cu b/cpp/benchmarks/lists/copying/scatter_lists.cu index dbc3234dabf..570decf410f 100644 --- a/cpp/benchmarks/lists/copying/scatter_lists.cu +++ b/cpp/benchmarks/lists/copying/scatter_lists.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -143,5 +143,5 @@ void BM_lists_scatter(::benchmark::State& state) ->Ranges({{1 << 10, 1 << 25}, {64, 2048}}) /* 1K-1B rows, 64-2048 elements */ \ ->UseManualTime(); -SBM_BENCHMARK_DEFINE(double_type_colesce_o, double, true); -SBM_BENCHMARK_DEFINE(double_type_colesce_x, double, false); +SBM_BENCHMARK_DEFINE(double_coalesced, double, true); +SBM_BENCHMARK_DEFINE(double_shuffled, double, false); From 6e37afc7c9e177b307c41950e52453bd5906af44 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:52:27 -1000 Subject: [PATCH 42/53] Make __bool__ raise for more cudf objects (#16311) To match pandas, this PR makes `DataFrame`, `MultiIndex` and `RangeIndex` raise on `__bool__`. Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16311 --- python/cudf/cudf/core/_base_index.py | 6 ++++++ python/cudf/cudf/core/frame.py | 6 ++++++ python/cudf/cudf/core/single_column_frame.py | 6 ------ python/cudf/cudf/tests/test_csv.py | 2 +- python/cudf/cudf/tests/test_dataframe.py | 9 +++++++++ python/cudf/cudf/tests/test_index.py | 9 +++++++++ python/cudf/cudf/tests/test_multiindex.py | 9 +++++++++ 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 479f87bb78b..657acc41b18 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -62,6 +62,12 @@ def copy(self, deep: bool = True) -> Self: def __len__(self): raise NotImplementedError + def __bool__(self): + raise ValueError( + f"The truth value of a {type(self).__name__} is ambiguous. Use " + "a.empty, a.bool(), a.item(), a.any() or a.all()." + ) + @property def size(self): # The size of an index is always its length irrespective of dimension. diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 111225a5fc2..e3a2e840902 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1587,6 +1587,12 @@ def __pos__(self): def __abs__(self): return self._unaryop("abs") + def __bool__(self): + raise ValueError( + f"The truth value of a {type(self).__name__} is ambiguous. Use " + "a.empty, a.bool(), a.item(), a.any() or a.all()." + ) + # Reductions @classmethod @_performance_tracking diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index 04c7db7a53c..7efe13d9b45 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -91,12 +91,6 @@ def shape(self) -> tuple[int]: """Get a tuple representing the dimensionality of the Index.""" return (len(self),) - def __bool__(self): - raise TypeError( - f"The truth value of a {type(self)} is ambiguous. Use " - "a.empty, a.bool(), a.item(), a.any() or a.all()." - ) - @property # type: ignore @_performance_tracking def _num_columns(self) -> int: diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index a22a627523f..0525b02b698 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -1617,7 +1617,7 @@ def test_csv_reader_partial_dtype(dtype): StringIO('"A","B","C"\n0,1,2'), dtype=dtype, usecols=["A", "C"] ) - assert names_df == header_df + assert_eq(names_df, header_df) assert all(names_df.dtypes == ["int16", "int64"]) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 2009fc49ce5..53ed5d728cb 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -11100,3 +11100,12 @@ def test_from_records_with_index_no_shallow_copy(): data = np.array([(1.0, 2), (3.0, 4)], dtype=[("x", " Date: Fri, 19 Jul 2024 11:55:40 -1000 Subject: [PATCH 43/53] Align more DataFrame APIs with pandas (#16310) I have a script that did some signature comparisons between `pandas.DataFrame` and `cudf.DataFrame` API and it appears some signatures have changed between the pandas 1.x and 2.x release. The API changes in this PR are mostly adding implementations or adding missing keyword argument (although they might not be implemented). The APIs affected are: * `__init__` * `__array__` * `__arrow_c_stream__` * `to_dict` * `where` * `add_prefix` * `join` * `apply` * `to_records` * `from_records` * `unstack` * `pct_change` * `sort_values` Marking as breaking as I ensured some added keywords are in the same positions as pandas and therefore might break users who are using purely positional arguments. Authors: - Matthew Roeschke (https://github.com/mroeschke) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/16310 --- python/cudf/cudf/core/dataframe.py | 169 +++++++++++++++++++++++-- python/cudf/cudf/core/frame.py | 2 +- python/cudf/cudf/core/indexed_frame.py | 13 +- python/cudf/cudf/core/reshape.py | 7 +- python/cudf/cudf/core/series.py | 32 ++++- 5 files changed, 202 insertions(+), 21 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 904bd4ccb2e..7e07078c95b 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -594,6 +594,9 @@ class DataFrame(IndexedFrame, Serializable, GetAttrGetItemMixin): dtype : dtype, default None Data type to force. Only a single dtype is allowed. If None, infer. + copy : bool or None, default None + Copy data from inputs. + Currently not implemented. nan_as_null : bool, Default True If ``None``/``True``, converts ``np.nan`` values to ``null`` values. @@ -680,8 +683,11 @@ def __init__( index=None, columns=None, dtype=None, + copy=None, nan_as_null=no_default, ): + if copy is not None: + raise NotImplementedError("copy is not currently implemented.") super().__init__() if nan_as_null is no_default: nan_as_null = not cudf.get_option("mode.pandas_compatible") @@ -1524,6 +1530,25 @@ def __array_function__(self, func, types, args, kwargs): pass return NotImplemented + def __arrow_c_stream__(self, requested_schema=None): + """ + Export the cudf DataFrame as an Arrow C stream PyCapsule. + + Parameters + ---------- + requested_schema : PyCapsule, default None + The schema to which the dataframe should be casted, passed as a + PyCapsule containing a C ArrowSchema representation of the + requested schema. Currently not implemented. + + Returns + ------- + PyCapsule + """ + if requested_schema is not None: + raise NotImplementedError("requested_schema is not supported") + return self.to_arrow().__arrow_c_stream__() + # The _get_numeric_data method is necessary for dask compatibility. @_performance_tracking def _get_numeric_data(self): @@ -2235,6 +2260,7 @@ def to_dict( self, orient: str = "dict", into: type[dict] = dict, + index: bool = True, ) -> dict | list[dict]: """ Convert the DataFrame to a dictionary. @@ -2268,6 +2294,13 @@ def to_dict( instance of the mapping type you want. If you want a collections.defaultdict, you must pass it initialized. + index : bool, default True + Whether to include the index item (and index_names item if `orient` + is 'tight') in the returned dictionary. Can only be ``False`` + when `orient` is 'split' or 'tight'. Note that when `orient` is + 'records', this parameter does not take effect (index item always + not included). + Returns ------- dict, list or collections.abc.Mapping @@ -2349,7 +2382,7 @@ def to_dict( raise TypeError(f"unsupported type: {into}") return cons(self.items()) # type: ignore[misc] - return self.to_pandas().to_dict(orient=orient, into=into) + return self.to_pandas().to_dict(orient=orient, into=into, index=index) @_performance_tracking def scatter_by_map( @@ -3004,7 +3037,12 @@ def fillna( ) @_performance_tracking - def where(self, cond, other=None, inplace=False): + def where(self, cond, other=None, inplace=False, axis=None, level=None): + if axis is not None: + raise NotImplementedError("axis is not supported.") + elif level is not None: + raise NotImplementedError("level is not supported.") + from cudf.core._internals.where import ( _check_and_cast_columns_with_other, _make_categorical_like, @@ -3614,7 +3652,9 @@ def rename( return result @_performance_tracking - def add_prefix(self, prefix): + def add_prefix(self, prefix, axis=None): + if axis is not None: + raise NotImplementedError("axis is currently not implemented.") # TODO: Change to deep=False when copy-on-write is default out = self.copy(deep=True) out.columns = [ @@ -4230,6 +4270,7 @@ def join( lsuffix="", rsuffix="", sort=False, + validate: str | None = None, ): """Join columns with other DataFrame on index or on a key column. @@ -4243,6 +4284,16 @@ def join( column names when avoiding conflicts. sort : bool Set to True to ensure sorted ordering. + validate : str, optional + If specified, checks if join is of specified type. + + * "one_to_one" or "1:1": check if join keys are unique in both left + and right datasets. + * "one_to_many" or "1:m": check if join keys are unique in left dataset. + * "many_to_one" or "m:1": check if join keys are unique in right dataset. + * "many_to_many" or "m:m": allowed, but does not result in checks. + + Currently not supported. Returns ------- @@ -4256,6 +4307,10 @@ def join( """ if on is not None: raise NotImplementedError("The on parameter is not yet supported") + elif validate is not None: + raise NotImplementedError( + "The validate parameter is not yet supported" + ) df = self.merge( other, @@ -4404,7 +4459,16 @@ def query(self, expr, local_dict=None): @_performance_tracking def apply( - self, func, axis=1, raw=False, result_type=None, args=(), **kwargs + self, + func, + axis=1, + raw=False, + result_type=None, + args=(), + by_row: Literal[False, "compat"] = "compat", + engine: Literal["python", "numba"] = "python", + engine_kwargs: dict[str, bool] | None = None, + **kwargs, ): """ Apply a function along an axis of the DataFrame. @@ -4432,6 +4496,25 @@ def apply( Not yet supported args: tuple Positional arguments to pass to func in addition to the dataframe. + by_row : False or "compat", default "compat" + Only has an effect when ``func`` is a listlike or dictlike of funcs + and the func isn't a string. + If "compat", will if possible first translate the func into pandas + methods (e.g. ``Series().apply(np.sum)`` will be translated to + ``Series().sum()``). If that doesn't work, will try call to apply again with + ``by_row=True`` and if that fails, will call apply again with + ``by_row=False`` (backward compatible). + If False, the funcs will be passed the whole Series at once. + + Currently not supported. + + engine : {'python', 'numba'}, default 'python' + Unused. Added for compatibility with pandas. + engine_kwargs : dict + Unused. Added for compatibility with pandas. + **kwargs + Additional keyword arguments to pass as keywords arguments to + `func`. Examples -------- @@ -4582,13 +4665,17 @@ def apply( """ if axis != 1: - raise ValueError( + raise NotImplementedError( "DataFrame.apply currently only supports row wise ops" ) if raw: - raise ValueError("The `raw` kwarg is not yet supported.") + raise NotImplementedError("The `raw` kwarg is not yet supported.") if result_type is not None: - raise ValueError("The `result_type` kwarg is not yet supported.") + raise NotImplementedError( + "The `result_type` kwarg is not yet supported." + ) + if by_row != "compat": + raise NotImplementedError("by_row is currently not supported.") return self._apply(func, _get_row_kernel, *args, **kwargs) @@ -5489,7 +5576,7 @@ def from_arrow(cls, table): return out @_performance_tracking - def to_arrow(self, preserve_index=None): + def to_arrow(self, preserve_index=None) -> pa.Table: """ Convert to a PyArrow Table. @@ -5579,18 +5666,36 @@ def to_arrow(self, preserve_index=None): return out.replace_schema_metadata(metadata) @_performance_tracking - def to_records(self, index=True): + def to_records(self, index=True, column_dtypes=None, index_dtypes=None): """Convert to a numpy recarray Parameters ---------- index : bool Whether to include the index in the output. + column_dtypes : str, type, dict, default None + If a string or type, the data type to store all columns. If + a dictionary, a mapping of column names and indices (zero-indexed) + to specific data types. Currently not supported. + index_dtypes : str, type, dict, default None + If a string or type, the data type to store all index levels. If + a dictionary, a mapping of index level names and indices + (zero-indexed) to specific data types. + This mapping is applied only if `index=True`. + Currently not supported. Returns ------- numpy recarray """ + if column_dtypes is not None: + raise NotImplementedError( + "column_dtypes is currently not supported." + ) + elif index_dtypes is not None: + raise NotImplementedError( + "column_dtypes is currently not supported." + ) members = [("index", self.index.dtype)] if index else [] members += [(col, self[col].dtype) for col in self._data.names] dtype = np.dtype(members) @@ -5603,7 +5708,16 @@ def to_records(self, index=True): @classmethod @_performance_tracking - def from_records(cls, data, index=None, columns=None, nan_as_null=False): + def from_records( + cls, + data, + index=None, + exclude=None, + columns=None, + coerce_float: bool = False, + nrows: int | None = None, + nan_as_null=False, + ): """ Convert structured or record ndarray to DataFrame. @@ -5613,13 +5727,32 @@ def from_records(cls, data, index=None, columns=None, nan_as_null=False): index : str, array-like The name of the index column in *data*. If None, the default index is used. + exclude : sequence, default None + Columns or fields to exclude. + Currently not implemented. columns : list of str List of column names to include. + coerce_float : bool, default False + Attempt to convert values of non-string, non-numeric objects (like + decimal.Decimal) to floating point, useful for SQL result sets. + Currently not implemented. + nrows : int, default None + Number of rows to read if data is an iterator. + Currently not implemented. Returns ------- DataFrame """ + if exclude is not None: + raise NotImplementedError("exclude is currently not supported.") + if coerce_float is not False: + raise NotImplementedError( + "coerce_float is currently not supported." + ) + if nrows is not None: + raise NotImplementedError("nrows is currently not supported.") + if data.ndim != 1 and data.ndim != 2: raise ValueError( f"records dimension expected 1 or 2 but found {data.ndim}" @@ -7344,9 +7477,9 @@ def pivot_table( @_performance_tracking @copy_docstring(reshape.unstack) - def unstack(self, level=-1, fill_value=None): + def unstack(self, level=-1, fill_value=None, sort: bool = True): return cudf.core.reshape.unstack( - self, level=level, fill_value=fill_value + self, level=level, fill_value=fill_value, sort=sort ) @_performance_tracking @@ -7392,7 +7525,12 @@ def explode(self, column, ignore_index=False): return super()._explode(column, ignore_index) def pct_change( - self, periods=1, fill_method=no_default, limit=no_default, freq=None + self, + periods=1, + fill_method=no_default, + limit=no_default, + freq=None, + **kwargs, ): """ Calculates the percent change between sequential elements @@ -7417,6 +7555,9 @@ def pct_change( freq : str, optional Increment to use from time series API. Not yet implemented. + **kwargs + Additional keyword arguments are passed into + `DataFrame.shift`. Returns ------- @@ -7462,7 +7603,7 @@ def pct_change( data = self.fillna(method=fill_method, limit=limit) return data.diff(periods=periods) / data.shift( - periods=periods, freq=freq + periods=periods, freq=freq, **kwargs ) def __dataframe__( diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index e3a2e840902..c82e073d7b7 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -389,7 +389,7 @@ def values_host(self) -> np.ndarray: return self.to_numpy() @_performance_tracking - def __array__(self, dtype=None): + def __array__(self, dtype=None, copy=None): raise TypeError( "Implicit conversion to a host NumPy array via __array__ is not " "allowed, To explicitly construct a GPU matrix, consider using " diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 576596f6f7d..60cd142db4b 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3302,7 +3302,7 @@ def pad(self, value=None, axis=None, inplace=None, limit=None): ) return self.ffill(value=value, axis=axis, inplace=inplace, limit=limit) - def add_prefix(self, prefix): + def add_prefix(self, prefix, axis=None): """ Prefix labels with string `prefix`. @@ -3464,6 +3464,7 @@ def sort_values( kind="quicksort", na_position="last", ignore_index=False, + key=None, ): """Sort by the values along either axis. @@ -3479,6 +3480,14 @@ def sort_values( 'first' puts nulls at the beginning, 'last' puts nulls at the end ignore_index : bool, default False If True, index will not be sorted. + key : callable, optional + Apply the key function to the values + before sorting. This is similar to the ``key`` argument in the + builtin ``sorted`` function, with the notable difference that + this ``key`` function should be *vectorized*. It should expect a + ``Series`` and return a Series with the same shape as the input. + It will be applied to each column in `by` independently. + Currently not supported. Returns ------- @@ -3518,6 +3527,8 @@ def sort_values( ) if axis != 0: raise NotImplementedError("`axis` not currently implemented.") + if key is not None: + raise NotImplementedError("key is not currently supported.") if len(self) == 0: return self diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 1120642947b..b538ae34b6f 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -1060,7 +1060,7 @@ def pivot(data, columns=None, index=no_default, values=no_default): return result -def unstack(df, level, fill_value=None): +def unstack(df, level, fill_value=None, sort: bool = True): """ Pivot one or more levels of the (necessarily hierarchical) index labels. @@ -1080,6 +1080,9 @@ def unstack(df, level, fill_value=None): levels of the index to pivot fill_value Non-functional argument provided for compatibility with Pandas. + sort : bool, default True + Sort the level(s) in the resulting MultiIndex columns. + Returns ------- @@ -1156,6 +1159,8 @@ def unstack(df, level, fill_value=None): if fill_value is not None: raise NotImplementedError("fill_value is not supported.") + elif sort is False: + raise NotImplementedError(f"{sort=} is not supported.") if pd.api.types.is_list_like(level): if not level: return df diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index baaa2eb46a1..b1e63806934 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -2063,6 +2063,7 @@ def sort_values( kind="quicksort", na_position="last", ignore_index=False, + key=None, ): """Sort by the values along either axis. @@ -2076,6 +2077,14 @@ def sort_values( 'first' puts nulls at the beginning, 'last' puts nulls at the end ignore_index : bool, default False If True, index will not be sorted. + key : callable, optional + Apply the key function to the values + before sorting. This is similar to the ``key`` argument in the + builtin ``sorted`` function, with the notable difference that + this ``key`` function should be *vectorized*. It should expect a + ``Series`` and return a Series with the same shape as the input. + It will be applied to each column in `by` independently. + Currently not supported. Returns ------- @@ -2107,6 +2116,7 @@ def sort_values( kind=kind, na_position=na_position, ignore_index=ignore_index, + key=key, ) @_performance_tracking @@ -3429,7 +3439,9 @@ def rename(self, index=None, copy=True): return Series._from_data(out_data, self.index, name=index) @_performance_tracking - def add_prefix(self, prefix): + def add_prefix(self, prefix, axis=None): + if axis is not None: + raise NotImplementedError("axis is currently not implemented.") return Series._from_data( # TODO: Change to deep=False when copy-on-write is default data=self._data.copy(deep=True), @@ -3527,7 +3539,12 @@ def explode(self, ignore_index=False): @_performance_tracking def pct_change( - self, periods=1, fill_method=no_default, limit=no_default, freq=None + self, + periods=1, + fill_method=no_default, + limit=no_default, + freq=None, + **kwargs, ): """ Calculates the percent change between sequential elements @@ -3552,6 +3569,9 @@ def pct_change( freq : str, optional Increment to use from time series API. Not yet implemented. + **kwargs + Additional keyword arguments are passed into + `Series.shift`. Returns ------- @@ -3596,11 +3616,15 @@ def pct_change( warnings.simplefilter("ignore") data = self.fillna(method=fill_method, limit=limit) diff = data.diff(periods=periods) - change = diff / data.shift(periods=periods, freq=freq) + change = diff / data.shift(periods=periods, freq=freq, **kwargs) return change @_performance_tracking - def where(self, cond, other=None, inplace=False): + def where(self, cond, other=None, inplace=False, axis=None, level=None): + if axis is not None: + raise NotImplementedError("axis is not supported.") + elif level is not None: + raise NotImplementedError("level is not supported.") result_col = super().where(cond, other, inplace) return self._mimic_inplace( self._from_data_like_self( From 57ed7fce6742abc96a8fd65216f032bad5937a2f Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:24:55 -0500 Subject: [PATCH 44/53] Add tests for `pylibcudf` binaryops (#15470) This PR implements a more general approach to testing binaryops that originally came up in https://github.com/rapidsai/cudf/pull/15279. This PR can possibly supersede that one. Authors: - https://github.com/brandon-b-miller Approvers: - Lawrence Mitchell (https://github.com/wence-) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/15470 --- cpp/include/cudf/binaryop.hpp | 11 + cpp/src/binaryop/binaryop.cpp | 7 +- .../binaryop/binop-verify-input-test.cpp | 4 +- python/cudf/cudf/_lib/pylibcudf/binaryop.pxd | 9 + python/cudf/cudf/_lib/pylibcudf/binaryop.pyx | 35 + .../cudf/_lib/pylibcudf/libcudf/binaryop.pxd | 39 +- .../cudf/cudf/pylibcudf_tests/common/utils.py | 10 + .../cudf/pylibcudf_tests/test_binaryops.py | 786 ++++++++++++++++++ 8 files changed, 889 insertions(+), 12 deletions(-) create mode 100644 python/cudf/cudf/pylibcudf_tests/test_binaryops.py diff --git a/cpp/include/cudf/binaryop.hpp b/cpp/include/cudf/binaryop.hpp index 22dad11e109..c74c91e39c2 100644 --- a/cpp/include/cudf/binaryop.hpp +++ b/cpp/include/cudf/binaryop.hpp @@ -290,6 +290,17 @@ cudf::data_type binary_operation_fixed_point_output_type(binary_operator op, namespace binops { +/** + * @brief Returns true if the binary operator is supported for the given input types. + * + * @param out The output data type + * @param lhs The left-hand cudf::data_type + * @param rhs The right-hand cudf::data_type + * @param op The binary operator + * @return true if the binary operator is supported for the given input types + */ +bool is_supported_operation(data_type out, data_type lhs, data_type rhs, binary_operator op); + /** * @brief Computes output valid mask for op between a column and a scalar * diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 8ac1491547d..3ac8547baad 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -50,6 +50,11 @@ namespace cudf { namespace binops { +bool is_supported_operation(data_type out, data_type lhs, data_type rhs, binary_operator op) +{ + return cudf::binops::compiled::is_supported_operation(out, lhs, rhs, op); +} + /** * @brief Computes output valid mask for op between a column and a scalar */ @@ -194,7 +199,7 @@ std::unique_ptr binary_operation(LhsType const& lhs, rmm::device_async_resource_ref mr) { if constexpr (std::is_same_v and std::is_same_v) - CUDF_EXPECTS(lhs.size() == rhs.size(), "Column sizes don't match"); + CUDF_EXPECTS(lhs.size() == rhs.size(), "Column sizes don't match", std::invalid_argument); if (lhs.type().id() == type_id::STRING and rhs.type().id() == type_id::STRING and output_type.id() == type_id::STRING and diff --git a/cpp/tests/binaryop/binop-verify-input-test.cpp b/cpp/tests/binaryop/binop-verify-input-test.cpp index 1346dcd4666..def6e94452e 100644 --- a/cpp/tests/binaryop/binop-verify-input-test.cpp +++ b/cpp/tests/binaryop/binop-verify-input-test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Copyright 2018-2019 BlazingDB, Inc. * Copyright 2018 Christian Noboa Mardini @@ -42,5 +42,5 @@ TEST_F(BinopVerifyInputTest, Vector_Vector_ErrorSecondOperandVectorZeroSize) EXPECT_THROW(cudf::binary_operation( lhs, rhs, cudf::binary_operator::ADD, cudf::data_type(cudf::type_id::INT64)), - cudf::logic_error); + std::invalid_argument); } diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd b/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd index 9a8c8e49dcf..2411e28ac66 100644 --- a/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/binaryop.pxd @@ -1,5 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. +from libcpp cimport bool + from cudf._lib.pylibcudf.libcudf.binaryop cimport binary_operator from .column cimport Column @@ -22,3 +24,10 @@ cpdef Column binary_operation( binary_operator op, DataType output_type ) + +cpdef bool is_supported_operation( + DataType out, + DataType lhs, + DataType rhs, + binary_operator op +) diff --git a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx b/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx index c1d669c3c1c..44d9f4ad04a 100644 --- a/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/binaryop.pyx @@ -2,6 +2,7 @@ from cython.operator import dereference +from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move @@ -84,3 +85,37 @@ cpdef Column binary_operation( raise ValueError(f"Invalid arguments {lhs} and {rhs}") return Column.from_libcudf(move(result)) + + +cpdef bool is_supported_operation( + DataType out, + DataType lhs, + DataType rhs, + binary_operator op +): + """Check if an operation is supported for the given data types. + + For details, see :cpp:func::is_supported_operation`. + + Parameters + ---------- + out : DataType + The output data type. + lhs : DataType + The left hand side data type. + rhs : DataType + The right hand side data type. + op : BinaryOperator + The operation to check. + Returns + ------- + bool + True if the operation is supported, False otherwise + """ + + return cpp_binaryop.is_supported_operation( + out.c_obj, + lhs.c_obj, + rhs.c_obj, + op + ) diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd index 0eda7d34ff9..b34fea6a775 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/binaryop.pxd @@ -1,9 +1,11 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. from libc.stdint cimport int32_t +from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.string cimport string +from cudf._lib.exception_handler cimport cudf_exception_handler from cudf._lib.pylibcudf.libcudf.column.column cimport column from cudf._lib.pylibcudf.libcudf.column.column_view cimport column_view from cudf._lib.pylibcudf.libcudf.scalar.scalar cimport scalar @@ -19,9 +21,20 @@ cdef extern from "cudf/binaryop.hpp" namespace "cudf" nogil: TRUE_DIV FLOOR_DIV MOD + PMOD PYMOD POW INT_POW + LOG_BASE + ATAN2 + SHIFT_LEFT + SHIFT_RIGHT + SHIFT_RIGHT_UNSIGNED + BITWISE_AND + BITWISE_OR + BITWISE_XOR + LOGICAL_AND + LOGICAL_OR EQUAL NOT_EQUAL LESS @@ -29,38 +42,46 @@ cdef extern from "cudf/binaryop.hpp" namespace "cudf" nogil: LESS_EQUAL GREATER_EQUAL NULL_EQUALS + NULL_MAX + NULL_MIN NULL_NOT_EQUALS - BITWISE_AND - BITWISE_OR - BITWISE_XOR - LOGICAL_AND - LOGICAL_OR GENERIC_BINARY + NULL_LOGICAL_AND + NULL_LOGICAL_OR + INVALID_BINARY cdef unique_ptr[column] binary_operation ( const scalar& lhs, const column_view& rhs, binary_operator op, data_type output_type - ) except + + ) except +cudf_exception_handler cdef unique_ptr[column] binary_operation ( const column_view& lhs, const scalar& rhs, binary_operator op, data_type output_type - ) except + + ) except +cudf_exception_handler cdef unique_ptr[column] binary_operation ( const column_view& lhs, const column_view& rhs, binary_operator op, data_type output_type - ) except + + ) except +cudf_exception_handler cdef unique_ptr[column] binary_operation ( const column_view& lhs, const column_view& rhs, const string& op, data_type output_type - ) except + + ) except +cudf_exception_handler + +cdef extern from "cudf/binaryop.hpp" namespace "cudf::binops" nogil: + cdef bool is_supported_operation( + data_type output_type, + data_type lhs_type, + data_type rhs_type, + binary_operator op + ) except +cudf_exception_handler diff --git a/python/cudf/cudf/pylibcudf_tests/common/utils.py b/python/cudf/cudf/pylibcudf_tests/common/utils.py index e029edfa2ed..ed2c5ca06c9 100644 --- a/python/cudf/cudf/pylibcudf_tests/common/utils.py +++ b/python/cudf/cudf/pylibcudf_tests/common/utils.py @@ -111,6 +111,16 @@ def _make_fields_nullable(typ): lhs = rhs.cast(lhs_type) if pa.types.is_floating(lhs.type) and pa.types.is_floating(rhs.type): + lhs_nans = pa.compute.is_nan(lhs) + rhs_nans = pa.compute.is_nan(rhs) + assert lhs_nans.equals(rhs_nans) + + if pa.compute.any(lhs_nans) or pa.compute.any(rhs_nans): + # masks must be equal at this point + mask = pa.compute.fill_null(pa.compute.invert(lhs_nans), True) + lhs = lhs.filter(mask) + rhs = rhs.filter(mask) + np.testing.assert_array_almost_equal(lhs, rhs) else: assert lhs.equals(rhs) diff --git a/python/cudf/cudf/pylibcudf_tests/test_binaryops.py b/python/cudf/cudf/pylibcudf_tests/test_binaryops.py new file mode 100644 index 00000000000..a83caf39ead --- /dev/null +++ b/python/cudf/cudf/pylibcudf_tests/test_binaryops.py @@ -0,0 +1,786 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import math + +import numpy as np +import pyarrow as pa +import pytest +from utils import assert_column_eq + +from cudf._lib import pylibcudf as plc + + +def idfn(param): + ltype, rtype, outtype, plc_op, _ = param + params = (plc_op.name, ltype, rtype, outtype) + return "-".join(map(str, params)) + + +@pytest.fixture(params=[True, False], ids=["nulls", "no_nulls"]) +def nulls(request): + return request.param + + +def make_col(dtype, nulls): + if dtype == "int64": + data = [1, 2, 3, 4, 5] + pa_type = pa.int64() + elif dtype == "uint64": + data = [1, 2, 3, 4, 5] + pa_type = pa.uint64() + elif dtype == "float64": + data = [1.0, 2.0, 3.0, 4.0, 5.0] + pa_type = pa.float64() + elif dtype == "bool": + data = [True, False, True, False, True] + pa_type = pa.bool_() + elif dtype == "timestamp64[ns]": + data = [ + np.datetime64("2022-01-01"), + np.datetime64("2022-01-02"), + np.datetime64("2022-01-03"), + np.datetime64("2022-01-04"), + np.datetime64("2022-01-05"), + ] + pa_type = pa.timestamp("ns") + elif dtype == "timedelta64[ns]": + data = [ + np.timedelta64(1, "ns"), + np.timedelta64(2, "ns"), + np.timedelta64(3, "ns"), + np.timedelta64(4, "ns"), + np.timedelta64(5, "ns"), + ] + pa_type = pa.duration("ns") + else: + raise ValueError("Unsupported dtype") + + if nulls: + data[3] = None + + return pa.array(data, type=pa_type) + + +@pytest.fixture +def pa_data(request, nulls): + ltype, rtype, outtype = request.param + values = make_col(ltype, nulls), make_col(rtype, nulls), outtype + return values + + +@pytest.fixture +def plc_data(pa_data): + lhs, rhs, outtype = pa_data + return ( + plc.interop.from_arrow(lhs), + plc.interop.from_arrow(rhs), + plc.interop.from_arrow(pa.from_numpy_dtype(np.dtype(outtype))), + ) + + +@pytest.fixture +def tests(request, nulls): + ltype, rtype, py_outtype, plc_op, py_op = request.param + pa_lhs, pa_rhs = make_col(ltype, nulls), make_col(rtype, nulls) + plc_lhs, plc_rhs = ( + plc.interop.from_arrow(pa_lhs), + plc.interop.from_arrow(pa_rhs), + ) + plc_dtype = plc.interop.from_arrow( + pa.from_numpy_dtype(np.dtype(py_outtype)) + ) + return ( + pa_lhs, + pa_rhs, + py_outtype, + plc_lhs, + plc_rhs, + plc_dtype, + py_op, + plc_op, + ) + + +def custom_pyop(func): + def wrapper(x, y): + x = x.to_pylist() + y = y.to_pylist() + + def inner(x, y): + if x is None or y is None: + return None + return func(x, y) + + return pa.array([inner(x, y) for x, y in zip(x, y)]) + + return wrapper + + +@custom_pyop +def py_floordiv(x, y): + return x // y + + +@custom_pyop +def py_pmod(x, y): + return (x % y + y) % y + + +@custom_pyop +def py_mod(x, y): + return x % y + + +@custom_pyop +def py_atan2(x, y): + return math.atan2(x, y) + + +@custom_pyop +def py_shift_right_unsigned(x, y): + unsigned_x = np.uint32(x) + result = unsigned_x >> y + return result + + +@pytest.mark.parametrize( + "tests", + [ + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.ADD, + pa.compute.add, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.ADD, + pa.compute.add, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.ADD, + pa.compute.add, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.SUB, + pa.compute.subtract, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.SUB, + pa.compute.subtract, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.SUB, + pa.compute.subtract, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.MUL, + pa.compute.multiply, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.MUL, + pa.compute.multiply, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.MUL, + pa.compute.multiply, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.DIV, + pa.compute.divide, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.DIV, + pa.compute.divide, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.DIV, + pa.compute.divide, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.TRUE_DIV, + pa.compute.divide, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.TRUE_DIV, + pa.compute.divide, + ), + ( + "int64", + "int64", + "timedelta64[ns]", + plc.binaryop.BinaryOperator.TRUE_DIV, + pa.compute.divide, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.FLOOR_DIV, + py_floordiv, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.FLOOR_DIV, + py_floordiv, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.FLOOR_DIV, + py_floordiv, + ), + ("int64", "int64", "int64", plc.binaryop.BinaryOperator.MOD, py_mod), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.MOD, + py_mod, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.MOD, + py_mod, + ), + ("int64", "int64", "int64", plc.binaryop.BinaryOperator.PMOD, py_pmod), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.PMOD, + py_pmod, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.PMOD, + py_pmod, + ), + ("int64", "int64", "int64", plc.binaryop.BinaryOperator.PYMOD, py_mod), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.PYMOD, + py_mod, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.PYMOD, + py_mod, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.POW, + pa.compute.power, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.POW, + pa.compute.power, + ), + ( + "int64", + "int64", + "timedelta64[ns]", + plc.binaryop.BinaryOperator.POW, + pa.compute.power, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.INT_POW, + pa.compute.power, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.INT_POW, + pa.compute.power, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.INT_POW, + pa.compute.power, + ), + ( + "float64", + "float64", + "float64", + plc.binaryop.BinaryOperator.LOG_BASE, + pa.compute.logb, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.LOG_BASE, + pa.compute.logb, + ), + ( + "int64", + "int64", + "timedelta64[ns]", + plc.binaryop.BinaryOperator.LOG_BASE, + pa.compute.logb, + ), + ( + "float64", + "float64", + "float64", + plc.binaryop.BinaryOperator.ATAN2, + py_atan2, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.ATAN2, + py_atan2, + ), + ( + "int64", + "int64", + "timedelta64[ns]", + plc.binaryop.BinaryOperator.ATAN2, + py_atan2, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.SHIFT_LEFT, + pa.compute.shift_left, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.SHIFT_LEFT, + pa.compute.shift_left, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.SHIFT_LEFT, + pa.compute.shift_left, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.SHIFT_RIGHT, + pa.compute.shift_right, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.SHIFT_RIGHT, + pa.compute.shift_right, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.SHIFT_RIGHT, + pa.compute.shift_right, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.SHIFT_RIGHT_UNSIGNED, + py_shift_right_unsigned, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.SHIFT_RIGHT_UNSIGNED, + py_shift_right_unsigned, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.SHIFT_RIGHT_UNSIGNED, + py_shift_right_unsigned, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.BITWISE_AND, + pa.compute.bit_wise_and, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.BITWISE_AND, + pa.compute.bit_wise_and, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.BITWISE_AND, + pa.compute.bit_wise_and, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.BITWISE_OR, + pa.compute.bit_wise_or, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.BITWISE_OR, + pa.compute.bit_wise_or, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.BITWISE_OR, + pa.compute.bit_wise_or, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.BITWISE_XOR, + pa.compute.bit_wise_xor, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.BITWISE_XOR, + pa.compute.bit_wise_xor, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.BITWISE_XOR, + pa.compute.bit_wise_xor, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.LOGICAL_AND, + pa.compute.and_, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.LOGICAL_AND, + pa.compute.and_, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.LOGICAL_AND, + pa.compute.and_, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.LOGICAL_OR, + pa.compute.or_, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.LOGICAL_OR, + pa.compute.or_, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.LOGICAL_OR, + pa.compute.or_, + ), + ( + "int64", + "int64", + "bool", + plc.binaryop.BinaryOperator.EQUAL, + pa.compute.equal, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.EQUAL, + pa.compute.equal, + ), + ( + "int64", + "int64", + "bool", + plc.binaryop.BinaryOperator.NOT_EQUAL, + pa.compute.not_equal, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.NOT_EQUAL, + pa.compute.not_equal, + ), + ( + "int64", + "int64", + "bool", + plc.binaryop.BinaryOperator.LESS, + pa.compute.less, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.LESS, + pa.compute.less, + ), + ( + "int64", + "int64", + "bool", + plc.binaryop.BinaryOperator.GREATER, + pa.compute.greater, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.GREATER, + pa.compute.greater, + ), + ( + "int64", + "int64", + "bool", + plc.binaryop.BinaryOperator.LESS_EQUAL, + pa.compute.less_equal, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.LESS_EQUAL, + pa.compute.less_equal, + ), + ( + "int64", + "int64", + "bool", + plc.binaryop.BinaryOperator.GREATER_EQUAL, + pa.compute.greater_equal, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.GREATER_EQUAL, + pa.compute.greater_equal, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.NULL_EQUALS, + pa.compute.equal, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.NULL_EQUALS, + pa.compute.equal, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.NULL_MAX, + pa.compute.max_element_wise, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.NULL_MAX, + pa.compute.max_element_wise, + ), + ( + "int64", + "int64", + "datetime64[ns]", + plc.binaryop.BinaryOperator.NULL_MIN, + pa.compute.min_element_wise, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.NULL_MIN, + pa.compute.min_element_wise, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.NULL_NOT_EQUALS, + pa.compute.not_equal, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.NULL_NOT_EQUALS, + pa.compute.not_equal, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.NULL_LOGICAL_AND, + pa.compute.and_, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.NULL_LOGICAL_AND, + pa.compute.and_, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.NULL_LOGICAL_OR, + pa.compute.or_, + ), + ( + "int64", + "float64", + "float64", + plc.binaryop.BinaryOperator.NULL_LOGICAL_OR, + pa.compute.or_, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.GENERIC_BINARY, + None, + ), + ( + "int64", + "int64", + "int64", + plc.binaryop.BinaryOperator.INVALID_BINARY, + None, + ), + ], + indirect=True, + ids=idfn, +) +def test_binaryops(tests): + ( + pa_lhs, + pa_rhs, + py_outtype, + plc_lhs, + plc_rhs, + plc_outtype, + py_op, + plc_op, + ) = tests + + def get_result(): + return plc.binaryop.binary_operation( + plc_lhs, + plc_rhs, + plc_op, + plc_outtype, + ) + + if not plc.binaryop.is_supported_operation( + plc_outtype, plc_lhs.type(), plc_rhs.type(), plc_op + ): + with pytest.raises(TypeError): + get_result() + else: + expect = py_op(pa_lhs, pa_rhs).cast(py_outtype) + got = get_result() + assert_column_eq(expect, got) From 7d3083254c0503b07f82af32188120f42acef860 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:48:39 -1000 Subject: [PATCH 45/53] Replace np.isscalar/issubdtype checks with is_scalar/.kind checks (#16275) * `is_scalar` also handles cudf.Scalars which should be handled internally * `issubdtype` can largely be replaced by checking the `.kind` attribute on the dtype Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16275 --- python/cudf/cudf/core/_internals/where.py | 2 +- python/cudf/cudf/core/column/column.py | 10 +++---- python/cudf/cudf/core/column/datetime.py | 2 +- python/cudf/cudf/core/column/lists.py | 9 ++++--- python/cudf/cudf/core/column/numerical.py | 28 +++++++------------- python/cudf/cudf/core/join/_join_helpers.py | 29 ++++++--------------- python/cudf/cudf/core/series.py | 2 +- python/cudf/cudf/testing/testing.py | 10 +++---- python/cudf/cudf/utils/dtypes.py | 4 +-- 9 files changed, 37 insertions(+), 59 deletions(-) diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 6003a0f6aea..18ab32d2c9e 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -47,7 +47,7 @@ def _check_and_cast_columns_with_other( other_is_scalar = is_scalar(other) if other_is_scalar: - if isinstance(other, float) and not np.isnan(other): + if isinstance(other, (float, np.floating)) and not np.isnan(other): try: is_safe = source_dtype.type(other) == other except OverflowError: diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 89f0f79cb7c..da735c22c52 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1458,9 +1458,10 @@ def column_empty_like( return column_empty(row_count, dtype, masked) -def _has_any_nan(arbitrary): +def _has_any_nan(arbitrary: pd.Series | np.ndarray) -> bool: + """Check if an object dtype Series or array contains NaN.""" return any( - ((isinstance(x, float) or isinstance(x, np.floating)) and np.isnan(x)) + isinstance(x, (float, np.floating)) and np.isnan(x) for x in np.asarray(arbitrary) ) @@ -2312,9 +2313,8 @@ def concat_columns(objs: "MutableSequence[ColumnBase]") -> ColumnBase: # Notice, we can always cast pure null columns not_null_col_dtypes = [o.dtype for o in objs if o.null_count != len(o)] if len(not_null_col_dtypes) and all( - _is_non_decimal_numeric_dtype(dtyp) - and np.issubdtype(dtyp, np.datetime64) - for dtyp in not_null_col_dtypes + _is_non_decimal_numeric_dtype(dtype) and dtype.kind == "M" + for dtype in not_null_col_dtypes ): common_dtype = find_common_type(not_null_col_dtypes) # Cast all columns to the common dtype diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index a4538179415..73902789c11 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -639,7 +639,7 @@ def isin(self, values: Sequence) -> ColumnBase: return cudf.core.tools.datetimes._isin_datetimelike(self, values) def can_cast_safely(self, to_dtype: Dtype) -> bool: - if np.issubdtype(to_dtype, np.datetime64): + if to_dtype.kind == "M": # type: ignore[union-attr] to_res, _ = np.datetime_data(to_dtype) self_res, _ = np.datetime_data(self.dtype) diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index 46b844413f7..1b7cd95b3d0 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -564,10 +564,11 @@ def take(self, lists_indices: ColumnLike) -> ParentType: raise ValueError( "lists_indices and list column is of different " "size." ) - if not _is_non_decimal_numeric_dtype( - lists_indices_col.children[1].dtype - ) or not np.issubdtype( - lists_indices_col.children[1].dtype, np.integer + if ( + not _is_non_decimal_numeric_dtype( + lists_indices_col.children[1].dtype + ) + or lists_indices_col.children[1].dtype.kind not in "iu" ): raise TypeError( "lists_indices should be column of values of index types." diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index b55284f1aff..5e07bbab40c 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -225,25 +225,17 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: tmp = self if reflect else other # Guard against division by zero for integers. if ( - (tmp.dtype.type in int_float_dtype_mapping) - and (tmp.dtype.type != np.bool_) - and ( - ( - ( - np.isscalar(tmp) - or ( - isinstance(tmp, cudf.Scalar) - # host to device copy - and tmp.is_valid() - ) - ) - and (0 == tmp) - ) - or ((isinstance(tmp, NumericalColumn)) and (0 in tmp)) - ) + tmp.dtype.type in int_float_dtype_mapping + and tmp.dtype.kind != "b" ): - out_dtype = cudf.dtype("float64") - + if isinstance(tmp, NumericalColumn) and 0 in tmp: + out_dtype = cudf.dtype("float64") + elif isinstance(tmp, cudf.Scalar): + if tmp.is_valid() and tmp == 0: + # tmp == 0 can return NA + out_dtype = cudf.dtype("float64") + elif is_scalar(tmp) and tmp == 0: + out_dtype = cudf.dtype("float64") if op in { "__lt__", "__gt__", diff --git a/python/cudf/cudf/core/join/_join_helpers.py b/python/cudf/cudf/core/join/_join_helpers.py index dd0a4f666a1..32c84763401 100644 --- a/python/cudf/cudf/core/join/_join_helpers.py +++ b/python/cudf/cudf/core/join/_join_helpers.py @@ -9,7 +9,7 @@ import numpy as np import cudf -from cudf.api.types import is_decimal_dtype, is_dtype_equal +from cudf.api.types import is_decimal_dtype, is_dtype_equal, is_numeric_dtype from cudf.core.column import CategoricalColumn from cudf.core.dtypes import CategoricalDtype @@ -88,38 +88,25 @@ def _match_join_keys( ) if ( - np.issubdtype(ltype, np.number) - and np.issubdtype(rtype, np.number) - and not ( - np.issubdtype(ltype, np.timedelta64) - or np.issubdtype(rtype, np.timedelta64) - ) + is_numeric_dtype(ltype) + and is_numeric_dtype(rtype) + and not (ltype.kind == "m" or rtype.kind == "m") ): common_type = ( max(ltype, rtype) if ltype.kind == rtype.kind else np.result_type(ltype, rtype) ) - elif ( - np.issubdtype(ltype, np.datetime64) - and np.issubdtype(rtype, np.datetime64) - ) or ( - np.issubdtype(ltype, np.timedelta64) - and np.issubdtype(rtype, np.timedelta64) + elif (ltype.kind == "M" and rtype.kind == "M") or ( + ltype.kind == "m" and rtype.kind == "m" ): common_type = max(ltype, rtype) - elif ( - np.issubdtype(ltype, np.datetime64) - or np.issubdtype(ltype, np.timedelta64) - ) and not rcol.fillna(0).can_cast_safely(ltype): + elif ltype.kind in "mM" and not rcol.fillna(0).can_cast_safely(ltype): raise TypeError( f"Cannot join between {ltype} and {rtype}, please type-cast both " "columns to the same type." ) - elif ( - np.issubdtype(rtype, np.datetime64) - or np.issubdtype(rtype, np.timedelta64) - ) and not lcol.fillna(0).can_cast_safely(rtype): + elif rtype.kind in "mM" and not lcol.fillna(0).can_cast_safely(rtype): raise TypeError( f"Cannot join between {rtype} and {ltype}, please type-cast both " "columns to the same type." diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index b1e63806934..eb077179562 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -213,7 +213,7 @@ def __setitem__(self, key, value): and self._frame.dtype.categories.dtype.kind == "f" ) ) - and isinstance(value, (np.float32, np.float64)) + and isinstance(value, np.floating) and np.isnan(value) ): raise MixedTypeError( diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index e56c8d867cb..c2072d90e98 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -158,12 +158,12 @@ def assert_column_equal( return True if check_datetimelike_compat: - if np.issubdtype(left.dtype, np.datetime64): + if left.dtype.kind == "M": right = right.astype(left.dtype) - elif np.issubdtype(right.dtype, np.datetime64): + elif right.dtype.kind == "M": left = left.astype(right.dtype) - if np.issubdtype(left.dtype, np.datetime64): + if left.dtype.kind == "M": if not left.equals(right): raise AssertionError( f"[datetimelike_compat=True] {left.values} " @@ -779,9 +779,7 @@ def assert_eq(left, right, **kwargs): tm.assert_index_equal(left, right, **kwargs) elif isinstance(left, np.ndarray) and isinstance(right, np.ndarray): - if np.issubdtype(left.dtype, np.floating) and np.issubdtype( - right.dtype, np.floating - ): + if left.dtype.kind == "f" and right.dtype.kind == "f": assert np.allclose(left, right, equal_nan=True) else: assert np.array_equal(left, right) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 69c268db149..c0de5274742 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -359,10 +359,10 @@ def min_column_type(x, expected_type): if x.null_count == len(x): return x.dtype - if np.issubdtype(x.dtype, np.floating): + if x.dtype.kind == "f": return get_min_float_dtype(x) - elif np.issubdtype(expected_type, np.integer): + elif cudf.dtype(expected_type).kind in "iu": max_bound_dtype = np.min_scalar_type(x.max()) min_bound_dtype = np.min_scalar_type(x.min()) result_type = np.promote_types(max_bound_dtype, min_bound_dtype) From 4c46628eaf7ba16a2a181ceb3311f315cd4932dc Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:51:07 -1000 Subject: [PATCH 46/53] Mark cudf._typing as a typing module in ruff (#16318) Additionally breaks up the prior, single-line of `select` rules that are enabled. Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Thomas Li (https://github.com/lithomas1) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16318 --- pyproject.toml | 64 ++++++++++++++++++++++++++++++- python/cudf/cudf/core/resample.py | 6 ++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2f59864894b..e15cb7b3cdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,69 @@ quiet-level = 3 line-length = 79 [tool.ruff.lint] -select = ["E", "F", "W", "D201", "D204", "D206", "D207", "D208", "D209", "D210", "D211", "D214", "D215", "D300", "D301", "D403", "D405", "D406", "D407", "D408", "D409", "D410", "D411", "D412", "D414", "D418", "TCH", "FA", "UP006", "UP007"] +typing-modules = ["cudf._typing"] +select = [ + # pycodestyle Error + "E", + # Pyflakes + "F", + # pycodestyle Warning + "W", + # no-blank-line-before-function + "D201", + # one-blank-line-after-class + "D204", + # indent-with-spaces + "D206", + # under-indentation + "D207", + # over-indentation + "D208", + # new-line-after-last-paragraph + "D209", + # surrounding-whitespace + "D210", + # blank-line-before-class + "D211", + # section-not-over-indented + "D214", + # section-underline-not-over-indented + "D215", + # triple-single-quotes + "D300", + # escape-sequence-in-docstring + "D301", + # first-line-capitalized + "D403", + # capitalize-section-name + "D405", + # new-line-after-section-name + "D406", + # dashed-underline-after-section + "D407", + # section-underline-after-name + "D408", + # section-underline-matches-section-length + "D409", + # no-blank-line-after-section + "D410", + # no-blank-line-before-section + "D411", + # blank-lines-between-header-and-content + "D412", + # empty-docstring-section + "D414", + # overload-with-docstring + "D418", + # flake8-type-checking + "TCH", + # flake8-future-annotations + "FA", + # non-pep585-annotation + "UP006", + # non-pep604-annotation + "UP007" +] ignore = [ # whitespace before : "E203", diff --git a/python/cudf/cudf/core/resample.py b/python/cudf/cudf/core/resample.py index cdd4ec6f8e5..4e0c5bd86b9 100644 --- a/python/cudf/cudf/core/resample.py +++ b/python/cudf/cudf/core/resample.py @@ -13,9 +13,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations import pickle import warnings +from typing import TYPE_CHECKING import numpy as np import pandas as pd @@ -23,7 +25,6 @@ import cudf import cudf._lib.labeling import cudf.core.index -from cudf._typing import DataFrameOrSeries from cudf.core.groupby.groupby import ( DataFrameGroupBy, GroupBy, @@ -31,6 +32,9 @@ _Grouping, ) +if TYPE_CHECKING: + from cudf._typing import DataFrameOrSeries + class _Resampler(GroupBy): grouping: "_ResampleGrouping" From 5dde41d7f7533180ecd355bac248a7ed18adcc10 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:08:36 -1000 Subject: [PATCH 47/53] Replace is_float/integer_dtype checks with .kind checks (#16261) It appears this was called when we already had a dtype object so can instead just simply check the .kind attribute Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16261 --- python/cudf/cudf/api/types.py | 2 +- python/cudf/cudf/core/_base_index.py | 19 +++---------- python/cudf/cudf/core/column/column.py | 29 ++++++++++---------- python/cudf/cudf/core/column/decimal.py | 4 +-- python/cudf/cudf/core/column/numerical.py | 13 +++------ python/cudf/cudf/core/index.py | 13 +++++---- python/cudf/cudf/core/indexing_utils.py | 8 ++---- python/cudf/cudf/core/series.py | 7 ++--- python/cudf/cudf/core/single_column_frame.py | 3 +- python/cudf/cudf/tests/test_dataframe.py | 2 +- python/cudf/cudf/utils/dtypes.py | 28 +++++++++---------- 11 files changed, 52 insertions(+), 76 deletions(-) diff --git a/python/cudf/cudf/api/types.py b/python/cudf/cudf/api/types.py index d97e9c815b6..294ae2fd985 100644 --- a/python/cudf/cudf/api/types.py +++ b/python/cudf/cudf/api/types.py @@ -90,7 +90,7 @@ def is_integer(obj): bool """ if isinstance(obj, cudf.Scalar): - return pd.api.types.is_integer_dtype(obj.dtype) + return obj.dtype.kind in "iu" return pd.api.types.is_integer(obj) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 657acc41b18..c38352009de 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -19,14 +19,7 @@ ) from cudf._lib.types import size_type_dtype from cudf.api.extensions import no_default -from cudf.api.types import ( - is_integer, - is_integer_dtype, - is_list_like, - is_scalar, - is_signed_integer_dtype, - is_unsigned_integer_dtype, -) +from cudf.api.types import is_integer, is_list_like, is_scalar from cudf.core.abc import Serializable from cudf.core.column import ColumnBase, column from cudf.errors import MixedTypeError @@ -621,12 +614,8 @@ def union(self, other, sort=None): # Bools + other types will result in mixed type. # This is not yet consistent in pandas and specific to APIs. raise MixedTypeError("Cannot perform union with mixed types") - if ( - is_signed_integer_dtype(self.dtype) - and is_unsigned_integer_dtype(other.dtype) - ) or ( - is_unsigned_integer_dtype(self.dtype) - and is_signed_integer_dtype(other.dtype) + if (self.dtype.kind == "i" and other.dtype.kind == "u") or ( + self.dtype.kind == "u" and other.dtype.kind == "i" ): # signed + unsigned types will result in # mixed type for union in pandas. @@ -2103,7 +2092,7 @@ def _gather(self, gather_map, nullify=False, check_bounds=True): # TODO: For performance, the check and conversion of gather map should # be done by the caller. This check will be removed in future release. - if not is_integer_dtype(gather_map.dtype): + if gather_map.dtype.kind not in "iu": gather_map = gather_map.astype(size_type_dtype) if not _gather_map_is_valid( diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index da735c22c52..32e6aade65b 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -2219,25 +2219,26 @@ def as_column( and arbitrary.null_count > 0 ): arbitrary = arbitrary.cast(pa.float64()) - if cudf.get_option( - "default_integer_bitwidth" - ) and pa.types.is_integer(arbitrary.type): - dtype = _maybe_convert_to_default_type("int") - elif cudf.get_option( - "default_float_bitwidth" - ) and pa.types.is_floating(arbitrary.type): - dtype = _maybe_convert_to_default_type("float") + if ( + cudf.get_option("default_integer_bitwidth") + and pa.types.is_integer(arbitrary.type) + ) or ( + cudf.get_option("default_float_bitwidth") + and pa.types.is_floating(arbitrary.type) + ): + dtype = _maybe_convert_to_default_type( + cudf.dtype(arbitrary.type.to_pandas_dtype()) + ) except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError): arbitrary = pd.Series(arbitrary) - if cudf.get_option( - "default_integer_bitwidth" - ) and arbitrary.dtype.kind in set("iu"): - dtype = _maybe_convert_to_default_type("int") - elif ( + if ( + cudf.get_option("default_integer_bitwidth") + and arbitrary.dtype.kind in set("iu") + ) or ( cudf.get_option("default_float_bitwidth") and arbitrary.dtype.kind == "f" ): - dtype = _maybe_convert_to_default_type("float") + dtype = _maybe_convert_to_default_type(arbitrary.dtype) return as_column(arbitrary, nan_as_null=nan_as_null, dtype=dtype) diff --git a/python/cudf/cudf/core/column/decimal.py b/python/cudf/cudf/core/column/decimal.py index a63055ed527..6a7f338b065 100644 --- a/python/cudf/cudf/core/column/decimal.py +++ b/python/cudf/cudf/core/column/decimal.py @@ -15,7 +15,7 @@ from cudf._lib.strings.convert.convert_fixed_point import ( from_decimal as cpp_from_decimal, ) -from cudf.api.types import is_integer_dtype, is_scalar +from cudf.api.types import is_scalar from cudf.core.buffer import as_buffer from cudf.core.column import ColumnBase from cudf.core.dtypes import ( @@ -150,7 +150,7 @@ def _validate_fillna_value( def normalize_binop_value(self, other): if isinstance(other, ColumnBase): if isinstance(other, cudf.core.column.NumericalColumn): - if not is_integer_dtype(other.dtype): + if other.dtype.kind not in "iu": raise TypeError( "Decimal columns only support binary operations with " "integer numerical columns." diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 5e07bbab40c..f9404eb3b40 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -12,12 +12,7 @@ import cudf from cudf import _lib as libcudf from cudf._lib import pylibcudf -from cudf.api.types import ( - is_float_dtype, - is_integer, - is_integer_dtype, - is_scalar, -) +from cudf.api.types import is_integer, is_scalar from cudf.core.column import ( ColumnBase, as_column, @@ -249,7 +244,7 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: out_dtype = "bool" if op in {"__and__", "__or__", "__xor__"}: - if is_float_dtype(self.dtype) or is_float_dtype(other.dtype): + if self.dtype.kind == "f" or other.dtype.kind == "f": raise TypeError( f"Operation 'bitwise {op[2:-2]}' not supported between " f"{self.dtype.type.__name__} and " @@ -260,8 +255,8 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase: if ( op == "__pow__" - and is_integer_dtype(self.dtype) - and (is_integer(other) or is_integer_dtype(other.dtype)) + and self.dtype.kind in "iu" + and (is_integer(other) or other.dtype.kind in "iu") ): op = "INT_POW" diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index cd52a34e35e..ae20fcd5d9c 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -1456,18 +1456,19 @@ def notna(self): notnull = notna def _is_numeric(self): - return isinstance( - self._values, cudf.core.column.NumericalColumn - ) and self.dtype != cudf.dtype("bool") + return ( + isinstance(self._values, cudf.core.column.NumericalColumn) + and self.dtype.kind != "b" + ) def _is_boolean(self): - return self.dtype == cudf.dtype("bool") + return self.dtype.kind == "b" def _is_integer(self): - return cudf.api.types.is_integer_dtype(self.dtype) + return self.dtype.kind in "iu" def _is_floating(self): - return cudf.api.types.is_float_dtype(self.dtype) + return self.dtype.kind == "f" def _is_object(self): return isinstance(self._values, cudf.core.column.StringColumn) diff --git a/python/cudf/cudf/core/indexing_utils.py b/python/cudf/cudf/core/indexing_utils.py index 9c81b0eb607..a0089242909 100644 --- a/python/cudf/cudf/core/indexing_utils.py +++ b/python/cudf/cudf/core/indexing_utils.py @@ -8,11 +8,7 @@ from typing_extensions import TypeAlias import cudf -from cudf.api.types import ( - _is_scalar_or_zero_d_array, - is_integer, - is_integer_dtype, -) +from cudf.api.types import _is_scalar_or_zero_d_array, is_integer from cudf.core.copy_types import BooleanMask, GatherMap @@ -233,7 +229,7 @@ def parse_row_iloc_indexer(key: Any, n: int) -> IndexingSpec: return MaskIndexer(BooleanMask(key, n)) elif len(key) == 0: return EmptyIndexer() - elif is_integer_dtype(key.dtype): + elif key.dtype.kind in "iu": return MapIndexer(GatherMap(key, n, nullify=False)) else: raise TypeError( diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index eb077179562..d8dbaa897e7 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -24,7 +24,6 @@ _is_scalar_or_zero_d_array, is_dict_like, is_integer, - is_integer_dtype, is_scalar, ) from cudf.core import indexing_utils @@ -356,12 +355,10 @@ def _loc_to_iloc(self, arg): ) if not _is_non_decimal_numeric_dtype(index_dtype) and not ( isinstance(index_dtype, cudf.CategoricalDtype) - and is_integer_dtype(index_dtype.categories.dtype) + and index_dtype.categories.dtype.kind in "iu" ): # TODO: switch to cudf.utils.dtypes.is_integer(arg) - if isinstance(arg, cudf.Scalar) and is_integer_dtype( - arg.dtype - ): + if isinstance(arg, cudf.Scalar) and arg.dtype.kind in "iu": # Do not remove until pandas 3.0 support is added. assert ( PANDAS_LT_300 diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index 7efe13d9b45..b93528f9693 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -12,7 +12,6 @@ from cudf.api.types import ( _is_scalar_or_zero_d_array, is_integer, - is_integer_dtype, is_numeric_dtype, ) from cudf.core.column import ColumnBase, as_column @@ -352,7 +351,7 @@ def _get_elements_from_column(self, arg) -> ScalarLike | ColumnBase: arg = as_column(arg) if len(arg) == 0: arg = cudf.core.column.column_empty(0, dtype="int32") - if is_integer_dtype(arg.dtype): + if arg.dtype.kind in "iu": return self._column.take(arg) if arg.dtype.kind == "b": if (bn := len(arg)) != (n := len(self)): diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 53ed5d728cb..e2ce5c03b70 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -10833,7 +10833,7 @@ def test_dataframe_contains(name, contains, other_names): expectation = contains is cudf.NA and name is cudf.NA assert (contains in pdf) == expectation assert (contains in gdf) == expectation - elif pd.api.types.is_float_dtype(gdf.columns.dtype): + elif gdf.columns.dtype.kind == "f": # In some cases, the columns are converted to an Index[float] based on # the other column names. That casts name values from None to np.nan. expectation = contains is np.nan and (name is None or name is np.nan) diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index c0de5274742..b0788bcc0fc 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -1,7 +1,9 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. +from __future__ import annotations import datetime from decimal import Decimal +from typing import TYPE_CHECKING import cupy as cp import numpy as np @@ -11,6 +13,9 @@ import cudf +if TYPE_CHECKING: + from cudf._typing import DtypeObj + """Map numpy dtype to pyarrow types. Note that np.bool_ bitwidth (8) is different from pa.bool_ (1). Special handling is required when converting a Boolean column into arrow. @@ -568,25 +573,18 @@ def _dtype_pandas_compatible(dtype): return dtype -def _maybe_convert_to_default_type(dtype): +def _maybe_convert_to_default_type(dtype: DtypeObj) -> DtypeObj: """Convert `dtype` to default if specified by user. If not specified, return as is. """ - if cudf.get_option("default_integer_bitwidth"): - if cudf.api.types.is_signed_integer_dtype(dtype): - return cudf.dtype( - f'i{cudf.get_option("default_integer_bitwidth")//8}' - ) - elif cudf.api.types.is_unsigned_integer_dtype(dtype): - return cudf.dtype( - f'u{cudf.get_option("default_integer_bitwidth")//8}' - ) - if cudf.get_option( - "default_float_bitwidth" - ) and cudf.api.types.is_float_dtype(dtype): - return cudf.dtype(f'f{cudf.get_option("default_float_bitwidth")//8}') - + if ib := cudf.get_option("default_integer_bitwidth"): + if dtype.kind == "i": + return cudf.dtype(f"i{ib//8}") + elif dtype.kind == "u": + return cudf.dtype(f"u{ib//8}") + if (fb := cudf.get_option("default_float_bitwidth")) and dtype.kind == "f": + return cudf.dtype(f"f{fb//8}") return dtype From e169e8e4273e4d317e3f27c810c5b137dd75adb3 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:36:03 -0700 Subject: [PATCH 48/53] Implement read_csv in cudf-polars using pylibcudf (#16307) Replace cudf-classic with pylibcudf for CSV reading in cudf-polars Authors: - Thomas Li (https://github.com/lithomas1) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16307 --- python/cudf_polars/cudf_polars/dsl/ir.py | 50 ++++++++++++------------ python/cudf_polars/tests/test_scan.py | 38 ++++++++++++++++++ 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 0b14530e0ed..a84fe73810e 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -242,10 +242,6 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: with_columns = options.with_columns row_index = options.row_index if self.typ == "csv": - dtype_map = { - name: cudf._lib.types.PYLIBCUDF_TO_SUPPORTED_NUMPY_TYPES[typ.id()] - for name, typ in self.schema.items() - } parse_options = self.reader_options["parse_options"] sep = chr(parse_options["separator"]) quote = chr(parse_options["quote_char"]) @@ -280,31 +276,37 @@ def evaluate(self, *, cache: MutableMapping[int, DataFrame]) -> DataFrame: pieces = [] for p in self.paths: skiprows = self.reader_options["skip_rows"] - # TODO: read_csv expands globs which we should not do, - # because polars will already have handled them. path = Path(p) with path.open() as f: while f.readline() == "\n": skiprows += 1 - pieces.append( - cudf.read_csv( - path, - sep=sep, - quotechar=quote, - lineterminator=eol, - names=column_names, - header=header, - usecols=usecols, - na_filter=True, - na_values=null_values, - keep_default_na=False, - skiprows=skiprows, - comment=comment, - decimal=decimal, - dtype=dtype_map, - ) + tbl_w_meta = plc.io.csv.read_csv( + plc.io.SourceInfo([path]), + delimiter=sep, + quotechar=quote, + lineterminator=eol, + col_names=column_names, + header=header, + usecols=usecols, + na_filter=True, + na_values=null_values, + keep_default_na=False, + skiprows=skiprows, + comment=comment, + decimal=decimal, + dtypes=self.schema, + ) + pieces.append(tbl_w_meta) + tables, colnames = zip( + *( + (piece.tbl, piece.column_names(include_children=False)) + for piece in pieces ) - df = DataFrame.from_cudf(cudf.concat(pieces)) + ) + df = DataFrame.from_table( + plc.concatenate.concatenate(list(tables)), + colnames[0], + ) elif self.typ == "parquet": cdf = cudf.read_parquet(self.paths, columns=with_columns) assert isinstance(cdf, cudf.DataFrame) diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index d0c41090433..0981a96a34a 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +import os + import pytest import polars as pl @@ -129,6 +131,42 @@ def test_scan_csv_column_renames_projection_schema(tmp_path): assert_gpu_result_equal(q) +@pytest.mark.parametrize( + "filename,glob", + [ + (["test1.csv", "test2.csv"], True), + ("test*.csv", True), + # Make sure we don't expand glob when + # trying to read a file like test*.csv + # when glob=False + ("test*.csv", False), + ], +) +def test_scan_csv_multi(tmp_path, filename, glob): + with (tmp_path / "test1.csv").open("w") as f: + f.write("""foo,bar,baz\n1,2\n3,4,5""") + with (tmp_path / "test2.csv").open("w") as f: + f.write("""foo,bar,baz\n1,2\n3,4,5""") + with (tmp_path / "test*.csv").open("w") as f: + f.write("""foo,bar,baz\n1,2\n3,4,5""") + os.chdir(tmp_path) + q = pl.scan_csv(filename, glob=glob) + + assert_gpu_result_equal(q) + + +def test_scan_csv_multi_differing_colnames(tmp_path): + with (tmp_path / "test1.csv").open("w") as f: + f.write("""foo,bar,baz\n1,2\n3,4,5""") + with (tmp_path / "test2.csv").open("w") as f: + f.write("""abc,def,ghi\n1,2\n3,4,5""") + q = pl.scan_csv( + [tmp_path / "test1.csv", tmp_path / "test2.csv"], + ) + with pytest.raises(pl.exceptions.ComputeError): + q.explain() + + def test_scan_csv_skip_after_header_not_implemented(tmp_path): with (tmp_path / "test.csv").open("w") as f: f.write("""foo,bar,baz\n1,2,3\n3,4,5""") From 535db9b26ed1a57e4275f4a6f11b04ebeee21248 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:28:14 -0700 Subject: [PATCH 49/53] Deprecate Arrow support in I/O (#16132) Contributes to https://github.com/rapidsai/cudf/issues/15193 Authors: - Thomas Li (https://github.com/lithomas1) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Richard (Rick) Zamora (https://github.com/rjzamora) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16132 --- .../cudf/_lib/pylibcudf/io/datasource.pyx | 10 +- python/cudf/cudf/io/csv.py | 2 +- python/cudf/cudf/io/orc.py | 33 +++-- python/cudf/cudf/io/parquet.py | 40 ++++-- .../io/test_source_sink_info.py | 21 +-- python/cudf/cudf/tests/test_csv.py | 5 +- python/cudf/cudf/tests/test_gcs.py | 3 +- python/cudf/cudf/tests/test_parquet.py | 19 +-- python/cudf/cudf/tests/test_s3.py | 136 ++++++++++-------- python/cudf/cudf/utils/ioutils.py | 78 ++++++++-- python/cudf/cudf/utils/utils.py | 26 ++++ .../dask_cudf/dask_cudf/io/tests/test_s3.py | 6 +- 12 files changed, 247 insertions(+), 132 deletions(-) diff --git a/python/cudf/cudf/_lib/pylibcudf/io/datasource.pyx b/python/cudf/cudf/_lib/pylibcudf/io/datasource.pyx index aa7fa0efdaf..8f265f585de 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/datasource.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/io/datasource.pyx @@ -7,6 +7,8 @@ from pyarrow.lib cimport NativeFile from cudf._lib.pylibcudf.libcudf.io.arrow_io_source cimport arrow_io_source from cudf._lib.pylibcudf.libcudf.io.datasource cimport datasource +import warnings + cdef class Datasource: cdef datasource* get_datasource(self) except * nogil: @@ -16,10 +18,16 @@ cdef class Datasource: cdef class NativeFileDatasource(Datasource): - def __cinit__(self, NativeFile native_file,): + def __cinit__(self, NativeFile native_file): cdef shared_ptr[CRandomAccessFile] ra_src + warnings.warn( + "Support for reading pyarrow's NativeFile is deprecated " + "and will be removed in a future release of cudf.", + FutureWarning, + ) + ra_src = native_file.get_random_access_file() self.c_datasource.reset(new arrow_io_source(ra_src)) diff --git a/python/cudf/cudf/io/csv.py b/python/cudf/cudf/io/csv.py index e909d96309e..0f2820a01e9 100644 --- a/python/cudf/cudf/io/csv.py +++ b/python/cudf/cudf/io/csv.py @@ -50,7 +50,7 @@ def read_csv( comment=None, delim_whitespace=False, byte_range=None, - use_python_file_object=True, + use_python_file_object=None, storage_options=None, bytes_per_thread=None, ): diff --git a/python/cudf/cudf/io/orc.py b/python/cudf/cudf/io/orc.py index 7082a85237a..289292b5182 100644 --- a/python/cudf/cudf/io/orc.py +++ b/python/cudf/cudf/io/orc.py @@ -10,6 +10,7 @@ from cudf._lib import orc as liborc from cudf.api.types import is_list_like from cudf.utils import ioutils +from cudf.utils.utils import maybe_filter_deprecation def _make_empty_df(filepath_or_buffer, columns): @@ -280,7 +281,7 @@ def read_orc( num_rows=None, use_index=True, timestamp_type=None, - use_python_file_object=True, + use_python_file_object=None, storage_options=None, bytes_per_thread=None, ): @@ -320,6 +321,9 @@ def read_orc( ) filepaths_or_buffers = [] + have_nativefile = any( + isinstance(source, pa.NativeFile) for source in filepath_or_buffer + ) for source in filepath_or_buffer: if ioutils.is_directory( path_or_data=source, storage_options=storage_options @@ -360,17 +364,24 @@ def read_orc( stripes = selected_stripes if engine == "cudf": - return DataFrame._from_data( - *liborc.read_orc( - filepaths_or_buffers, - columns, - stripes, - skiprows, - num_rows, - use_index, - timestamp_type, + # Don't want to warn if use_python_file_object causes us to get + # a NativeFile (there is a separate deprecation warning for that) + with maybe_filter_deprecation( + not have_nativefile, + message="Support for reading pyarrow's NativeFile is deprecated", + category=FutureWarning, + ): + return DataFrame._from_data( + *liborc.read_orc( + filepaths_or_buffers, + columns, + stripes, + skiprows, + num_rows, + use_index, + timestamp_type, + ) ) - ) else: from pyarrow import orc diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 02b26ea1c01..0f0a240b5d0 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -15,6 +15,7 @@ import numpy as np import pandas as pd +import pyarrow as pa from pyarrow import dataset as ds import cudf @@ -23,6 +24,7 @@ from cudf.core.column import as_column, build_categorical_column, column_empty from cudf.utils import ioutils from cudf.utils.performance_tracking import _performance_tracking +from cudf.utils.utils import maybe_filter_deprecation BYTE_SIZES = { "kb": 1000, @@ -350,7 +352,7 @@ def read_parquet_metadata(filepath_or_buffer): path_or_data=source, compression=None, fs=fs, - use_python_file_object=True, + use_python_file_object=None, open_file_options=None, storage_options=None, bytes_per_thread=None, @@ -532,7 +534,7 @@ def read_parquet( filters=None, row_groups=None, use_pandas_metadata=True, - use_python_file_object=True, + use_python_file_object=None, categorical_partitions=True, open_file_options=None, bytes_per_thread=None, @@ -615,6 +617,9 @@ def read_parquet( row_groups=row_groups, fs=fs, ) + have_nativefile = any( + isinstance(source, pa.NativeFile) for source in filepath_or_buffer + ) for source in filepath_or_buffer: tmp_source, compression = ioutils.get_reader_filepath_or_buffer( path_or_data=source, @@ -662,19 +667,26 @@ def read_parquet( ) # Convert parquet data to a cudf.DataFrame - df = _parquet_to_frame( - filepaths_or_buffers, - engine, - *args, - columns=columns, - row_groups=row_groups, - use_pandas_metadata=use_pandas_metadata, - partition_keys=partition_keys, - partition_categories=partition_categories, - dataset_kwargs=dataset_kwargs, - **kwargs, - ) + # Don't want to warn if use_python_file_object causes us to get + # a NativeFile (there is a separate deprecation warning for that) + with maybe_filter_deprecation( + not have_nativefile, + message="Support for reading pyarrow's NativeFile is deprecated", + category=FutureWarning, + ): + df = _parquet_to_frame( + filepaths_or_buffers, + engine, + *args, + columns=columns, + row_groups=row_groups, + use_pandas_metadata=use_pandas_metadata, + partition_keys=partition_keys, + partition_categories=partition_categories, + dataset_kwargs=dataset_kwargs, + **kwargs, + ) # Apply filters row-wise (if any are defined), and return df = _apply_post_filters(df, filters) if projected_columns: diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_source_sink_info.py b/python/cudf/cudf/pylibcudf_tests/io/test_source_sink_info.py index 287dd8f21c8..438c482b77a 100644 --- a/python/cudf/cudf/pylibcudf_tests/io/test_source_sink_info.py +++ b/python/cudf/cudf/pylibcudf_tests/io/test_source_sink_info.py @@ -2,11 +2,9 @@ import io -import pyarrow as pa import pytest import cudf._lib.pylibcudf as plc -from cudf._lib.pylibcudf.io.datasource import NativeFileDatasource @pytest.fixture(params=[plc.io.SourceInfo, plc.io.SinkInfo]) @@ -18,10 +16,8 @@ def _skip_invalid_sinks(io_class, sink): """ Skip invalid sinks for SinkInfo """ - if io_class is plc.io.SinkInfo and isinstance( - sink, (bytes, NativeFileDatasource) - ): - pytest.skip(f"{sink} is not a valid input for SinkInfo") + if io_class is plc.io.SinkInfo and isinstance(sink, bytes): + pytest.skip("bytes is not a valid input for SinkInfo") @pytest.mark.parametrize( @@ -30,7 +26,6 @@ def _skip_invalid_sinks(io_class, sink): "a.txt", b"hello world", io.BytesIO(b"hello world"), - NativeFileDatasource(pa.PythonFile(io.BytesIO(), mode="r")), ], ) def test_source_info_ctor(io_class, source, tmp_path): @@ -47,13 +42,12 @@ def test_source_info_ctor(io_class, source, tmp_path): @pytest.mark.parametrize( "sources", [ + ["a.txt"], + [b"hello world"], + [io.BytesIO(b"hello world")], ["a.txt", "a.txt"], [b"hello world", b"hello there"], [io.BytesIO(b"hello world"), io.BytesIO(b"hello there")], - [ - NativeFileDatasource(pa.PythonFile(io.BytesIO(), mode="r")), - NativeFileDatasource(pa.PythonFile(io.BytesIO(), mode="r")), - ], ], ) def test_source_info_ctor_multiple(io_class, sources, tmp_path): @@ -79,11 +73,6 @@ def test_source_info_ctor_multiple(io_class, sources, tmp_path): io.BytesIO(b"hello there"), b"hello world", ], - [ - NativeFileDatasource(pa.PythonFile(io.BytesIO(), mode="r")), - "awef.txt", - b"hello world", - ], ], ) def test_source_info_ctor_mixing_invalid(io_class, sources, tmp_path): diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py index 0525b02b698..6a21cb1b9d7 100644 --- a/python/cudf/cudf/tests/test_csv.py +++ b/python/cudf/cudf/tests/test_csv.py @@ -1085,8 +1085,9 @@ def test_csv_reader_arrow_nativefile(path_or_buf): # Arrow FileSystem interface expect = cudf.read_csv(path_or_buf("filepath")) fs, path = pa_fs.FileSystem.from_uri(path_or_buf("filepath")) - with fs.open_input_file(path) as fil: - got = cudf.read_csv(fil) + with pytest.warns(FutureWarning): + with fs.open_input_file(path) as fil: + got = cudf.read_csv(fil) assert_eq(expect, got) diff --git a/python/cudf/cudf/tests/test_gcs.py b/python/cudf/cudf/tests/test_gcs.py index fc22d8bc0ea..28fdfb5c2f1 100644 --- a/python/cudf/cudf/tests/test_gcs.py +++ b/python/cudf/cudf/tests/test_gcs.py @@ -46,7 +46,8 @@ def mock_size(*args): # use_python_file_object=True, because the pyarrow # `open_input_file` command will fail (since it doesn't # use the monkey-patched `open` definition) - got = cudf.read_csv(f"gcs://{fpath}", use_python_file_object=False) + with pytest.warns(FutureWarning): + got = cudf.read_csv(f"gcs://{fpath}", use_python_file_object=False) assert_eq(pdf, got) # AbstractBufferedFile -> PythonFile conversion diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index ecb7fd44422..f2820d9c112 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -711,7 +711,8 @@ def test_parquet_reader_arrow_nativefile(parquet_path_or_buf): expect = cudf.read_parquet(parquet_path_or_buf("filepath")) fs, path = pa_fs.FileSystem.from_uri(parquet_path_or_buf("filepath")) with fs.open_input_file(path) as fil: - got = cudf.read_parquet(fil) + with pytest.warns(FutureWarning): + got = cudf.read_parquet(fil) assert_eq(expect, got) @@ -726,16 +727,18 @@ def test_parquet_reader_use_python_file_object( fs, _, paths = get_fs_token_paths(parquet_path_or_buf("filepath")) # Pass open fsspec file - with fs.open(paths[0], mode="rb") as fil: - got1 = cudf.read_parquet( - fil, use_python_file_object=use_python_file_object - ) + with pytest.warns(FutureWarning): + with fs.open(paths[0], mode="rb") as fil: + got1 = cudf.read_parquet( + fil, use_python_file_object=use_python_file_object + ) assert_eq(expect, got1) # Pass path only - got2 = cudf.read_parquet( - paths[0], use_python_file_object=use_python_file_object - ) + with pytest.warns(FutureWarning): + got2 = cudf.read_parquet( + paths[0], use_python_file_object=use_python_file_object + ) assert_eq(expect, got2) diff --git a/python/cudf/cudf/tests/test_s3.py b/python/cudf/cudf/tests/test_s3.py index a44bf791767..3ae318d3bf5 100644 --- a/python/cudf/cudf/tests/test_s3.py +++ b/python/cudf/cudf/tests/test_s3.py @@ -138,22 +138,24 @@ def test_read_csv(s3_base, s3so, pdf, bytes_per_thread): buffer = pdf.to_csv(index=False) # Use fsspec file object - with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - got = cudf.read_csv( - f"s3://{bucket}/{fname}", - storage_options=s3so, - bytes_per_thread=bytes_per_thread, - use_python_file_object=False, - ) + with pytest.warns(FutureWarning): + with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): + got = cudf.read_csv( + f"s3://{bucket}/{fname}", + storage_options=s3so, + bytes_per_thread=bytes_per_thread, + use_python_file_object=False, + ) assert_eq(pdf, got) # Use Arrow PythonFile object - with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - got = cudf.read_csv( - f"s3://{bucket}/{fname}", - storage_options=s3so, - use_python_file_object=True, - ) + with pytest.warns(FutureWarning): + with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): + got = cudf.read_csv( + f"s3://{bucket}/{fname}", + storage_options=s3so, + use_python_file_object=True, + ) assert_eq(pdf, got) @@ -166,8 +168,9 @@ def test_read_csv_arrow_nativefile(s3_base, s3so, pdf): fs = pa_fs.S3FileSystem( endpoint_override=s3so["client_kwargs"]["endpoint_url"], ) - with fs.open_input_file(f"{bucket}/{fname}") as fil: - got = cudf.read_csv(fil) + with pytest.warns(FutureWarning): + with fs.open_input_file(f"{bucket}/{fname}") as fil: + got = cudf.read_csv(fil) assert_eq(pdf, got) @@ -184,17 +187,18 @@ def test_read_csv_byte_range( # Use fsspec file object with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - got = cudf.read_csv( - f"s3://{bucket}/{fname}", - storage_options=s3so, - byte_range=(74, 73), - bytes_per_thread=bytes_per_thread - if not use_python_file_object - else None, - header=None, - names=["Integer", "Float", "Integer2", "String", "Boolean"], - use_python_file_object=use_python_file_object, - ) + with pytest.warns(FutureWarning): + got = cudf.read_csv( + f"s3://{bucket}/{fname}", + storage_options=s3so, + byte_range=(74, 73), + bytes_per_thread=bytes_per_thread + if not use_python_file_object + else None, + header=None, + names=["Integer", "Float", "Integer2", "String", "Boolean"], + use_python_file_object=use_python_file_object, + ) assert_eq(pdf.iloc[-2:].reset_index(drop=True), got) @@ -241,18 +245,19 @@ def test_read_parquet( # Check direct path handling buffer.seek(0) with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - got1 = cudf.read_parquet( - f"s3://{bucket}/{fname}", - open_file_options=( - {"precache_options": {"method": precache}} - if use_python_file_object - else None - ), - storage_options=s3so, - bytes_per_thread=bytes_per_thread, - columns=columns, - use_python_file_object=use_python_file_object, - ) + with pytest.warns(FutureWarning): + got1 = cudf.read_parquet( + f"s3://{bucket}/{fname}", + open_file_options=( + {"precache_options": {"method": precache}} + if use_python_file_object + else None + ), + storage_options=s3so, + bytes_per_thread=bytes_per_thread, + columns=columns, + use_python_file_object=use_python_file_object, + ) expect = pdf[columns] if columns else pdf assert_eq(expect, got1) @@ -263,12 +268,13 @@ def test_read_parquet( f"s3://{bucket}/{fname}", storage_options=s3so )[0] with fs.open(f"s3://{bucket}/{fname}", mode="rb") as f: - got2 = cudf.read_parquet( - f, - bytes_per_thread=bytes_per_thread, - columns=columns, - use_python_file_object=use_python_file_object, - ) + with pytest.warns(FutureWarning): + got2 = cudf.read_parquet( + f, + bytes_per_thread=bytes_per_thread, + columns=columns, + use_python_file_object=use_python_file_object, + ) assert_eq(expect, got2) @@ -353,11 +359,12 @@ def test_read_parquet_arrow_nativefile(s3_base, s3so, pdf, columns): pdf.to_parquet(path=buffer) buffer.seek(0) with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - fs = pa_fs.S3FileSystem( - endpoint_override=s3so["client_kwargs"]["endpoint_url"], - ) - with fs.open_input_file(f"{bucket}/{fname}") as fil: - got = cudf.read_parquet(fil, columns=columns) + with pytest.warns(FutureWarning): + fs = pa_fs.S3FileSystem( + endpoint_override=s3so["client_kwargs"]["endpoint_url"], + ) + with fs.open_input_file(f"{bucket}/{fname}") as fil: + got = cudf.read_parquet(fil, columns=columns) expect = pdf[columns] if columns else pdf assert_eq(expect, got) @@ -372,12 +379,13 @@ def test_read_parquet_filters(s3_base, s3so, pdf_ext, precache): buffer.seek(0) filters = [("String", "==", "Omega")] with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - got = cudf.read_parquet( - f"s3://{bucket}/{fname}", - storage_options=s3so, - filters=filters, - open_file_options={"precache_options": {"method": precache}}, - ) + with pytest.warns(FutureWarning): + got = cudf.read_parquet( + f"s3://{bucket}/{fname}", + storage_options=s3so, + filters=filters, + open_file_options={"precache_options": {"method": precache}}, + ) # All row-groups should be filtered out assert_eq(pdf_ext.iloc[:0], got.reset_index(drop=True)) @@ -449,12 +457,13 @@ def test_read_orc(s3_base, s3so, datadir, use_python_file_object, columns): buffer = f.read() with s3_context(s3_base=s3_base, bucket=bucket, files={fname: buffer}): - got = cudf.read_orc( - f"s3://{bucket}/{fname}", - columns=columns, - storage_options=s3so, - use_python_file_object=use_python_file_object, - ) + with pytest.warns(FutureWarning): + got = cudf.read_orc( + f"s3://{bucket}/{fname}", + columns=columns, + storage_options=s3so, + use_python_file_object=use_python_file_object, + ) if columns: expect = expect[columns] @@ -475,8 +484,9 @@ def test_read_orc_arrow_nativefile(s3_base, s3so, datadir, columns): fs = pa_fs.S3FileSystem( endpoint_override=s3so["client_kwargs"]["endpoint_url"], ) - with fs.open_input_file(f"{bucket}/{fname}") as fil: - got = cudf.read_orc(fil, columns=columns) + with pytest.warns(FutureWarning): + with fs.open_input_file(f"{bucket}/{fname}") as fil: + got = cudf.read_orc(fil, columns=columns) if columns: expect = expect[columns] diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 76c7f2bfdb8..80555750b3a 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -6,6 +6,7 @@ import warnings from io import BufferedWriter, BytesIO, IOBase, TextIOWrapper from threading import Thread +from typing import Callable import fsspec import fsspec.implementations.local @@ -15,6 +16,7 @@ from pyarrow import PythonFile as ArrowPythonFile from pyarrow.lib import NativeFile +from cudf.api.extensions import no_default from cudf.core._compat import PANDAS_LT_300 from cudf.utils.docutils import docfmt_partial @@ -24,7 +26,6 @@ except ImportError: fsspec_parquet = None - _BYTES_PER_THREAD_DEFAULT = 256 * 1024 * 1024 _ROW_GROUP_SIZE_BYTES_DEFAULT = 128 * 1024 * 1024 @@ -86,7 +87,7 @@ 1 20 rapids 2 30 ai """.format(remote_data_sources=_docstring_remote_sources) -doc_read_avro = docfmt_partial(docstring=_docstring_read_avro) +doc_read_avro: Callable = docfmt_partial(docstring=_docstring_read_avro) _docstring_read_parquet_metadata = """ Read a Parquet file's metadata and schema @@ -174,15 +175,23 @@ columns are also loaded. use_python_file_object : boolean, default True If True, Arrow-backed PythonFile objects will be used in place of fsspec - AbstractBufferedFile objects at IO time. Setting this argument to `False` - will require the entire file to be copied to host memory, and is highly - discouraged. + AbstractBufferedFile objects at IO time. + + .. deprecated:: 24.08 + `use_python_file_object` is deprecated and will be removed in a future + version of cudf, as PyArrow NativeFiles will no longer be accepted as + input/output in cudf readers/writers in the future. open_file_options : dict, optional Dictionary of key-value pairs to pass to the function used to open remote files. By default, this will be `fsspec.parquet.open_parquet_file`. To deactivate optimized precaching, set the "method" to `None` under the "precache_options" key. Note that the `open_file_func` key can also be used to specify a custom file-open function. + + .. deprecated:: 24.08 + `open_file_options` is deprecated as it was intended for + pyarrow file inputs, which will no longer be accepted as + input/output cudf readers/writers in the future. bytes_per_thread : int, default None Determines the number of bytes to be allocated per thread to read the files in parallel. When there is a file of large size, we get slightly @@ -468,8 +477,12 @@ If True, use row index if available for faster seeking. use_python_file_object : boolean, default True If True, Arrow-backed PythonFile objects will be used in place of fsspec - AbstractBufferedFile objects at IO time. This option is likely to improve - performance when making small reads from larger ORC files. + AbstractBufferedFile objects at IO time. + + .. deprecated:: 24.08 + `use_python_file_object` is deprecated and will be removed in a future + version of cudf, as PyArrow NativeFiles will no longer be accepted as + input/output in cudf readers/writers in the future. storage_options : dict, optional, default None Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value @@ -934,7 +947,7 @@ -------- cudf.DataFrame.to_hdf : Write a HDF file from a DataFrame. """ -doc_read_hdf = docfmt_partial(docstring=_docstring_read_hdf) +doc_read_hdf: Callable = docfmt_partial(docstring=_docstring_read_hdf) _docstring_to_hdf = """ Write the contained data to an HDF5 file using HDFStore. @@ -1006,7 +1019,7 @@ cudf.DataFrame.to_parquet : Write a DataFrame to the binary parquet format. cudf.DataFrame.to_feather : Write out feather-format for DataFrames. """ -doc_to_hdf = docfmt_partial(docstring=_docstring_to_hdf) +doc_to_hdf: Callable = docfmt_partial(docstring=_docstring_to_hdf) _docstring_read_feather = """ Load an feather object from the file path, returning a DataFrame. @@ -1188,8 +1201,12 @@ the end of the range. use_python_file_object : boolean, default True If True, Arrow-backed PythonFile objects will be used in place of fsspec - AbstractBufferedFile objects at IO time. This option is likely to improve - performance when making small reads from larger CSV files. + AbstractBufferedFile objects at IO time. + + .. deprecated:: 24.08 + `use_python_file_object` is deprecated and will be removed in a future + version of cudf, as PyArrow NativeFiles will no longer be accepted as + input/output in cudf readers/writers in the future. storage_options : dict, optional, default None Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value @@ -1409,7 +1426,7 @@ result : Series """ -doc_read_text = docfmt_partial(docstring=_docstring_text_datasource) +doc_read_text: Callable = docfmt_partial(docstring=_docstring_text_datasource) _docstring_get_reader_filepath_or_buffer = """ @@ -1430,9 +1447,19 @@ use_python_file_object : boolean, default False If True, Arrow-backed PythonFile objects will be used in place of fsspec AbstractBufferedFile objects. + + .. deprecated:: 24.08 + `use_python_file_object` is deprecated and will be removed in a future + version of cudf, as PyArrow NativeFiles will no longer be accepted as + input/output in cudf readers/writers. open_file_options : dict, optional Optional dictionary of keyword arguments to pass to `_open_remote_files` (used for remote storage only). + + .. deprecated:: 24.08 + `open_file_options` is deprecated as it was intended for + pyarrow file inputs, which will no longer be accepted as + input/output cudf readers/writers in the future. allow_raw_text_input : boolean, default False If True, this indicates the input `path_or_data` could be a raw text input and will not check for its existence in the filesystem. If False, @@ -1708,7 +1735,8 @@ def get_reader_filepath_or_buffer( mode="rb", fs=None, iotypes=(BytesIO, NativeFile), - use_python_file_object=False, + # no_default aliases to False + use_python_file_object=no_default, open_file_options=None, allow_raw_text_input=False, storage_options=None, @@ -1720,6 +1748,30 @@ def get_reader_filepath_or_buffer( path_or_data = stringify_pathlike(path_or_data) + if use_python_file_object is no_default: + use_python_file_object = False + elif use_python_file_object is not None: + warnings.warn( + "The 'use_python_file_object' keyword is deprecated and " + "will be removed in a future version.", + FutureWarning, + ) + else: + # Preserve the readers (e.g. read_csv) default of True + # if no use_python_file_object option is specified by the user + # for now (note: this is different from the default for this + # function of False) + # TODO: when non-pyarrow file reading perf is good enough + # we can default this to False + use_python_file_object = True + + if open_file_options is not None: + warnings.warn( + "The 'open_file_options' keyword is deprecated and " + "will be removed in a future version.", + FutureWarning, + ) + if isinstance(path_or_data, str): # Get a filesystem object if one isn't already available paths = [path_or_data] diff --git a/python/cudf/cudf/utils/utils.py b/python/cudf/cudf/utils/utils.py index 7347ec7866a..c9b343e0f9f 100644 --- a/python/cudf/cudf/utils/utils.py +++ b/python/cudf/cudf/utils/utils.py @@ -6,6 +6,7 @@ import os import traceback import warnings +from contextlib import contextmanager import numpy as np import pandas as pd @@ -403,3 +404,28 @@ def _all_bools_with_nulls(lhs, rhs, bool_fill_value): if result_mask is not None: result_col = result_col.set_mask(result_mask.as_mask()) return result_col + + +@contextmanager +def maybe_filter_deprecation( + condition: bool, message: str, category: type[Warning] +): + """Conditionally filter a warning category. + + Parameters + ---------- + condition + If true, filter the warning + message + Message to match, passed to :func:`warnings.filterwarnings` + category + Category of warning, passed to :func:`warnings.filterwarnings` + """ + with warnings.catch_warnings(): + if condition: + warnings.filterwarnings( + "ignore", + message, + category=category, + ) + yield diff --git a/python/dask_cudf/dask_cudf/io/tests/test_s3.py b/python/dask_cudf/dask_cudf/io/tests/test_s3.py index a67404da4fe..3947c69aaa5 100644 --- a/python/dask_cudf/dask_cudf/io/tests/test_s3.py +++ b/python/dask_cudf/dask_cudf/io/tests/test_s3.py @@ -138,5 +138,7 @@ def test_read_parquet(s3_base, s3so, open_file_options): storage_options=s3so, open_file_options=open_file_options, ) - assert df.a.sum().compute() == 10 - assert df.b.sum().compute() == 9 + with pytest.warns(FutureWarning): + assert df.a.sum().compute() == 10 + with pytest.warns(FutureWarning): + assert df.b.sum().compute() == 9 From 75335f6af51bde6be68c1fb0a6caa8030b9eda3e Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Fri, 19 Jul 2024 18:21:27 -0700 Subject: [PATCH 50/53] Report number of rows per file read by PQ reader when no row selection and fix segfault in chunked PQ reader when skip_rows > 0 (#16195) Closes #15389 Closes #16186 This PR adds the capability to calculate and report the number of rows read from each data source into the table returned by the Parquet reader (both chunked and normal). The returned vector of counts is only valid (non-empty) when row selection (AST filter) is not being used. This PR also fixes a segfault in chunked parquet reader when skip_rows > 0 and the number of passes > 1. This segfault was being caused by a couple of arithmetic errors when computing the (start_row, num_row) for row_group_info, pass, column chunk descriptor structs. Both changes were added to this PR as changes and the gtests from the former work were needed to implement the segfault fix. Authors: - Muhammad Haseeb (https://github.com/mhaseeb123) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/16195 --- cpp/include/cudf/io/types.hpp | 3 + cpp/src/io/parquet/reader_impl.cpp | 86 +++- cpp/src/io/parquet/reader_impl.hpp | 31 +- cpp/src/io/parquet/reader_impl_chunking.cu | 53 ++- cpp/src/io/parquet/reader_impl_chunking.hpp | 6 + cpp/src/io/parquet/reader_impl_helpers.cpp | 32 +- cpp/src/io/parquet/reader_impl_helpers.hpp | 20 +- cpp/src/io/parquet/reader_impl_preprocess.cu | 19 +- cpp/tests/io/parquet_chunked_reader_test.cu | 385 ++++++++++++++++++ cpp/tests/io/parquet_reader_test.cpp | 203 +++++++++ .../cudf/_lib/pylibcudf/libcudf/io/types.pxd | 1 + 11 files changed, 796 insertions(+), 43 deletions(-) diff --git a/cpp/include/cudf/io/types.hpp b/cpp/include/cudf/io/types.hpp index 0c96268f6c7..431a5e7be83 100644 --- a/cpp/include/cudf/io/types.hpp +++ b/cpp/include/cudf/io/types.hpp @@ -277,6 +277,9 @@ struct column_name_info { struct table_metadata { std::vector schema_info; //!< Detailed name information for the entire output hierarchy + std::vector num_rows_per_source; //!< Number of rows read from each data source. + //!< Currently only computed for Parquet readers if no + //!< AST filters being used. Empty vector otherwise. std::map user_data; //!< Format-dependent metadata of the first input //!< file as key-values pairs (deprecated) std::vector> diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index f705f6626e7..68ec61ead0a 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -26,6 +26,7 @@ #include +#include #include #include @@ -549,7 +550,17 @@ table_with_metadata reader::impl::read_chunk_internal(read_mode mode) out_columns.reserve(_output_buffers.size()); // no work to do (this can happen on the first pass if we have no rows to read) - if (!has_more_work()) { return finalize_output(out_metadata, out_columns); } + if (!has_more_work()) { + // Check if number of rows per source should be included in output metadata. + if (include_output_num_rows_per_source()) { + // Empty dataframe case: Simply initialize to a list of zeros + out_metadata.num_rows_per_source = + std::vector(_file_itm_data.num_rows_per_source.size(), 0); + } + + // Finalize output + return finalize_output(mode, out_metadata, out_columns); + } auto& pass = *_pass_itm_data; auto& subpass = *pass.subpass; @@ -585,11 +596,80 @@ table_with_metadata reader::impl::read_chunk_internal(read_mode mode) } } + // Check if number of rows per source should be included in output metadata. + if (include_output_num_rows_per_source()) { + // For chunked reading, compute the output number of rows per source + if (mode == read_mode::CHUNKED_READ) { + out_metadata.num_rows_per_source = + calculate_output_num_rows_per_source(read_info.skip_rows, read_info.num_rows); + } + // Simply move the number of rows per file if reading all at once + else { + // Move is okay here as we are reading in one go. + out_metadata.num_rows_per_source = std::move(_file_itm_data.num_rows_per_source); + } + } + // Add empty columns if needed. Filter output columns based on filter. - return finalize_output(out_metadata, out_columns); + return finalize_output(mode, out_metadata, out_columns); +} + +std::vector reader::impl::calculate_output_num_rows_per_source(size_t const chunk_start_row, + size_t const chunk_num_rows) +{ + // Handle base cases. + if (_file_itm_data.num_rows_per_source.size() == 0) { + return {}; + } else if (_file_itm_data.num_rows_per_source.size() == 1) { + return {chunk_num_rows}; + } + + std::vector num_rows_per_source(_file_itm_data.num_rows_per_source.size(), 0); + + // Subtract global skip rows from the start_row as we took care of that when computing + // _file_itm_data.num_rows_per_source + auto const start_row = chunk_start_row - _file_itm_data.global_skip_rows; + auto const end_row = start_row + chunk_num_rows; + CUDF_EXPECTS(start_row <= end_row and end_row <= _file_itm_data.global_num_rows, + "Encountered invalid output chunk row bounds."); + + // Copy reference to a const local variable for better readability + auto const& partial_sum_nrows_source = _file_itm_data.exclusive_sum_num_rows_per_source; + + // Binary search start_row and end_row in exclusive_sum_num_rows_per_source vector + auto const start_iter = + std::upper_bound(partial_sum_nrows_source.cbegin(), partial_sum_nrows_source.cend(), start_row); + auto const end_iter = + (end_row == _file_itm_data.global_skip_rows + _file_itm_data.global_num_rows) + ? partial_sum_nrows_source.cend() - 1 + : std::upper_bound(start_iter, partial_sum_nrows_source.cend(), end_row); + + // Compute the array offset index for both iterators + auto const start_idx = std::distance(partial_sum_nrows_source.cbegin(), start_iter); + auto const end_idx = std::distance(partial_sum_nrows_source.cbegin(), end_iter); + + CUDF_EXPECTS(start_idx <= end_idx, + "Encountered invalid source files indexes for output chunk row bounds"); + + // If the entire chunk is from the same source file, then the count is simply num_rows + if (start_idx == end_idx) { + num_rows_per_source[start_idx] = chunk_num_rows; + } else { + // Compute the number of rows from the first source file + num_rows_per_source[start_idx] = partial_sum_nrows_source[start_idx] - start_row; + // Compute the number of rows from the last source file + num_rows_per_source[end_idx] = end_row - partial_sum_nrows_source[end_idx - 1]; + // Simply copy the number of rows for each source in range: (start_idx, end_idx) + std::copy(_file_itm_data.num_rows_per_source.cbegin() + start_idx + 1, + _file_itm_data.num_rows_per_source.cbegin() + end_idx, + num_rows_per_source.begin() + start_idx + 1); + } + + return num_rows_per_source; } -table_with_metadata reader::impl::finalize_output(table_metadata& out_metadata, +table_with_metadata reader::impl::finalize_output(read_mode mode, + table_metadata& out_metadata, std::vector>& out_columns) { // Create empty columns as needed (this can happen if we've ended up with no actual data to read) diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 3b8e80a29e6..5e3cc4301f9 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -262,11 +262,13 @@ class reader::impl { * @brief Finalize the output table by adding empty columns for the non-selected columns in * schema. * + * @param read_mode Value indicating if the data sources are read all at once or chunk by chunk * @param out_metadata The output table metadata * @param out_columns The columns for building the output table * @return The output table along with columns' metadata */ - table_with_metadata finalize_output(table_metadata& out_metadata, + table_with_metadata finalize_output(read_mode mode, + table_metadata& out_metadata, std::vector>& out_columns); /** @@ -336,11 +338,36 @@ class reader::impl { : true; } + /** + * @brief Check if this is the first output chunk + * + * @return True if this is the first output chunk + */ [[nodiscard]] bool is_first_output_chunk() const { return _file_itm_data._output_chunk_count == 0; } + /** + * @brief Check if number of rows per source should be included in output metadata. + * + * @return True if AST filter is not present + */ + [[nodiscard]] bool include_output_num_rows_per_source() const + { + return not _expr_conv.get_converted_expr().has_value(); + } + + /** + * @brief Calculate the number of rows read from each source in the output chunk + * + * @param chunk_start_row The offset of the first row in the output chunk + * @param chunk_num_rows The number of rows in the the output chunk + * @return Vector of number of rows from each respective data source in the output chunk + */ + [[nodiscard]] std::vector calculate_output_num_rows_per_source(size_t chunk_start_row, + size_t chunk_num_rows); + rmm::cuda_stream_view _stream; rmm::device_async_resource_ref _mr{rmm::mr::get_current_device_resource()}; @@ -387,7 +414,7 @@ class reader::impl { // chunked reading happens in 2 parts: // - // At the top level, the entire file is divided up into "passes" omn which we try and limit the + // At the top level, the entire file is divided up into "passes" on which we try and limit the // total amount of temporary memory (compressed data, decompressed data) in use // via _input_pass_read_limit. // diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index 3da303e6928..05e0d8c0111 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -1232,22 +1232,22 @@ void reader::impl::setup_next_pass(read_mode mode) pass.skip_rows = _file_itm_data.global_skip_rows; pass.num_rows = _file_itm_data.global_num_rows; } else { - auto const global_start_row = _file_itm_data.global_skip_rows; - auto const global_end_row = global_start_row + _file_itm_data.global_num_rows; - auto const start_row = - std::max(_file_itm_data.input_pass_start_row_count[_file_itm_data._current_input_pass], - global_start_row); - auto const end_row = - std::min(_file_itm_data.input_pass_start_row_count[_file_itm_data._current_input_pass + 1], - global_end_row); - - // skip_rows is always global in the sense that it is relative to the first row of - // everything we will be reading, regardless of what pass we are on. - // num_rows is how many rows we are reading this pass. - pass.skip_rows = - global_start_row + + // pass_start_row and pass_end_row are computed from the selected row groups relative to the + // global_skip_rows. + auto const pass_start_row = _file_itm_data.input_pass_start_row_count[_file_itm_data._current_input_pass]; - pass.num_rows = end_row - start_row; + auto const pass_end_row = + std::min(_file_itm_data.input_pass_start_row_count[_file_itm_data._current_input_pass + 1], + _file_itm_data.global_num_rows); + + // pass.skip_rows is always global in the sense that it is relative to the first row of + // the data source (global row number 0), regardless of what pass we are on. Therefore, + // we must re-add global_skip_rows to the pass_start_row which is relative to the + // global_skip_rows. + pass.skip_rows = _file_itm_data.global_skip_rows + pass_start_row; + // num_rows is how many rows we are reading this pass. Since this is a difference, adding + // global_skip_rows to both variables is redundant. + pass.num_rows = pass_end_row - pass_start_row; } // load page information for the chunk. this retrieves the compressed bytes for all the @@ -1509,6 +1509,7 @@ void reader::impl::create_global_chunk_info() // Initialize column chunk information auto remaining_rows = num_rows; + auto skip_rows = _file_itm_data.global_skip_rows; for (auto const& rg : row_groups_info) { auto const& row_group = _metadata->get_row_group(rg.index, rg.source_index); auto const row_group_start = rg.start_row; @@ -1561,7 +1562,12 @@ void reader::impl::create_global_chunk_info() schema.type == BYTE_ARRAY and _strings_to_categorical)); } - remaining_rows -= row_group_rows; + // Adjust for skip_rows when updating the remaining rows after the first group + remaining_rows -= + (skip_rows) ? std::min(rg.start_row + row_group.num_rows - skip_rows, remaining_rows) + : row_group_rows; + // Set skip_rows = 0 as it is no longer needed for subsequent row_groups + skip_rows = 0; } } @@ -1598,6 +1604,9 @@ void reader::impl::compute_input_passes() _file_itm_data.input_pass_row_group_offsets.push_back(0); _file_itm_data.input_pass_start_row_count.push_back(0); + // To handle global_skip_rows when computing input passes + int skip_rows = _file_itm_data.global_skip_rows; + for (size_t cur_rg_index = 0; cur_rg_index < row_groups_info.size(); cur_rg_index++) { auto const& rgi = row_groups_info[cur_rg_index]; auto const& row_group = _metadata->get_row_group(rgi.index, rgi.source_index); @@ -1606,6 +1615,14 @@ void reader::impl::compute_input_passes() auto const [compressed_rg_size, _ /*compressed + uncompressed*/] = get_row_group_size(row_group); + // We must use the effective size of the first row group we are reading to accurately calculate + // the first non-zero input_pass_start_row_count. + auto const row_group_rows = + (skip_rows) ? rgi.start_row + row_group.num_rows - skip_rows : row_group.num_rows; + + // Set skip_rows = 0 as it is no longer needed for subsequent row_groups + skip_rows = 0; + // can we add this row group if (cur_pass_byte_size + compressed_rg_size >= comp_read_limit) { // A single row group (the current one) is larger than the read limit: @@ -1613,7 +1630,7 @@ void reader::impl::compute_input_passes() // row group if (cur_rg_start == cur_rg_index) { _file_itm_data.input_pass_row_group_offsets.push_back(cur_rg_index + 1); - _file_itm_data.input_pass_start_row_count.push_back(cur_row_count + row_group.num_rows); + _file_itm_data.input_pass_start_row_count.push_back(cur_row_count + row_group_rows); cur_rg_start = cur_rg_index + 1; cur_pass_byte_size = 0; } @@ -1627,7 +1644,7 @@ void reader::impl::compute_input_passes() } else { cur_pass_byte_size += compressed_rg_size; } - cur_row_count += row_group.num_rows; + cur_row_count += row_group_rows; } // add the last pass if necessary diff --git a/cpp/src/io/parquet/reader_impl_chunking.hpp b/cpp/src/io/parquet/reader_impl_chunking.hpp index b959c793011..3a3cdd34a58 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.hpp +++ b/cpp/src/io/parquet/reader_impl_chunking.hpp @@ -41,6 +41,12 @@ struct file_intermediate_data { // is not capped by global_skip_rows and global_num_rows. std::vector input_pass_start_row_count{}; + // number of rows to be read from each data source + std::vector num_rows_per_source{}; + + // partial sum of the number of rows per data source + std::vector exclusive_sum_num_rows_per_source{}; + size_t _current_input_pass{0}; // current input pass index size_t _output_chunk_count{0}; // how many output chunks we have produced diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index d1e9a823d3b..581c44d024b 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -945,7 +945,7 @@ std::vector aggregate_reader_metadata::get_pandas_index_names() con return names; } -std::tuple> +std::tuple, std::vector> aggregate_reader_metadata::select_row_groups( host_span const> row_group_indices, int64_t skip_rows_opt, @@ -976,6 +976,9 @@ aggregate_reader_metadata::select_row_groups( static_cast(from_opts.second)}; }(); + // Get number of rows in each data source + std::vector num_rows_per_source(per_file_metadata.size(), 0); + if (!row_group_indices.empty()) { CUDF_EXPECTS(row_group_indices.size() == per_file_metadata.size(), "Must specify row groups for each source"); @@ -989,28 +992,45 @@ aggregate_reader_metadata::select_row_groups( selection.emplace_back(rowgroup_idx, rows_to_read, src_idx); // if page-level indexes are present, then collect extra chunk and page info. column_info_for_row_group(selection.back(), 0); - rows_to_read += get_row_group(rowgroup_idx, src_idx).num_rows; + auto const rows_this_rg = get_row_group(rowgroup_idx, src_idx).num_rows; + rows_to_read += rows_this_rg; + num_rows_per_source[src_idx] += rows_this_rg; } } } else { size_type count = 0; for (size_t src_idx = 0; src_idx < per_file_metadata.size(); ++src_idx) { auto const& fmd = per_file_metadata[src_idx]; - for (size_t rg_idx = 0; rg_idx < fmd.row_groups.size(); ++rg_idx) { + for (size_t rg_idx = 0; + rg_idx < fmd.row_groups.size() and count < rows_to_skip + rows_to_read; + ++rg_idx) { auto const& rg = fmd.row_groups[rg_idx]; auto const chunk_start_row = count; count += rg.num_rows; if (count > rows_to_skip || count == 0) { + // start row of this row group adjusted with rows_to_skip + num_rows_per_source[src_idx] += count; + num_rows_per_source[src_idx] -= + (chunk_start_row <= rows_to_skip) ? rows_to_skip : chunk_start_row; + + // We need the unadjusted start index of this row group to correctly initialize + // ColumnChunkDesc for this row group in create_global_chunk_info() and calculate + // the row offset for the first pass in compute_input_passes(). selection.emplace_back(rg_idx, chunk_start_row, src_idx); - // if page-level indexes are present, then collect extra chunk and page info. + + // If page-level indexes are present, then collect extra chunk and page info. + // The page indexes rely on absolute row numbers, not adjusted for skip_rows. column_info_for_row_group(selection.back(), chunk_start_row); } - if (count >= rows_to_skip + rows_to_read) { break; } + // Adjust the number of rows for the last source file. + if (count >= rows_to_skip + rows_to_read) { + num_rows_per_source[src_idx] -= count - rows_to_skip - rows_to_read; + } } } } - return {rows_to_skip, rows_to_read, std::move(selection)}; + return {rows_to_skip, rows_to_read, std::move(selection), std::move(num_rows_per_source)}; } std::tuple, diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index 6bfa8519c76..309132a5347 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -282,17 +282,17 @@ class aggregate_reader_metadata { * @param output_column_schemas schema indices of output columns * @param filter Optional AST expression to filter row groups based on Column chunk statistics * @param stream CUDA stream used for device memory operations and kernel launches - * @return A tuple of corrected row_start, row_count and list of row group indexes and its - * starting row + * @return A tuple of corrected row_start, row_count, list of row group indexes and its + * starting row, and list of number of rows per source. */ - [[nodiscard]] std::tuple> select_row_groups( - host_span const> row_group_indices, - int64_t row_start, - std::optional const& row_count, - host_span output_dtypes, - host_span output_column_schemas, - std::optional> filter, - rmm::cuda_stream_view stream) const; + [[nodiscard]] std::tuple, std::vector> + select_row_groups(host_span const> row_group_indices, + int64_t row_start, + std::optional const& row_count, + host_span output_dtypes, + host_span output_column_schemas, + std::optional> filter, + rmm::cuda_stream_view stream) const; /** * @brief Filters and reduces down to a selection of columns diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index f28a7311ccb..ff47dfc4cf3 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -1235,8 +1235,10 @@ void reader::impl::preprocess_file(read_mode mode) [](auto const& col) { return col.type; }); } - std::tie( - _file_itm_data.global_skip_rows, _file_itm_data.global_num_rows, _file_itm_data.row_groups) = + std::tie(_file_itm_data.global_skip_rows, + _file_itm_data.global_num_rows, + _file_itm_data.row_groups, + _file_itm_data.num_rows_per_source) = _metadata->select_row_groups(_options.row_group_indices, _options.skip_rows, _options.num_rows, @@ -1245,9 +1247,18 @@ void reader::impl::preprocess_file(read_mode mode) _expr_conv.get_converted_expr(), _stream); + // Inclusive scan the number of rows per source + if (not _expr_conv.get_converted_expr().has_value() and mode == read_mode::CHUNKED_READ) { + _file_itm_data.exclusive_sum_num_rows_per_source.resize( + _file_itm_data.num_rows_per_source.size()); + thrust::inclusive_scan(_file_itm_data.num_rows_per_source.cbegin(), + _file_itm_data.num_rows_per_source.cend(), + _file_itm_data.exclusive_sum_num_rows_per_source.begin()); + } + // check for page indexes - _has_page_index = std::all_of(_file_itm_data.row_groups.begin(), - _file_itm_data.row_groups.end(), + _has_page_index = std::all_of(_file_itm_data.row_groups.cbegin(), + _file_itm_data.row_groups.cend(), [](auto const& row_group) { return row_group.has_page_index(); }); if (_file_itm_data.global_num_rows > 0 && not _file_itm_data.row_groups.empty() && diff --git a/cpp/tests/io/parquet_chunked_reader_test.cu b/cpp/tests/io/parquet_chunked_reader_test.cu index cff85647725..2917852235c 100644 --- a/cpp/tests/io/parquet_chunked_reader_test.cu +++ b/cpp/tests/io/parquet_chunked_reader_test.cu @@ -149,6 +149,33 @@ auto chunked_read(std::string const& filepath, return chunked_read(vpath, output_limit, input_limit); } +auto const read_table_and_nrows_per_source(cudf::io::chunked_parquet_reader const& reader) +{ + auto out_tables = std::vector>{}; + int num_chunks = 0; + auto nrows_per_source = std::vector{}; + while (reader.has_next()) { + auto chunk = reader.read_chunk(); + out_tables.emplace_back(std::move(chunk.tbl)); + num_chunks++; + if (nrows_per_source.empty()) { + nrows_per_source = std::move(chunk.metadata.num_rows_per_source); + } else { + std::transform(chunk.metadata.num_rows_per_source.cbegin(), + chunk.metadata.num_rows_per_source.cend(), + nrows_per_source.begin(), + nrows_per_source.begin(), + std::plus()); + } + } + auto out_tviews = std::vector{}; + for (auto const& tbl : out_tables) { + out_tviews.emplace_back(tbl->view()); + } + + return std::tuple(cudf::concatenate(out_tviews), num_chunks, nrows_per_source); +} + } // namespace struct ParquetChunkedReaderTest : public cudf::test::BaseFixture {}; @@ -1477,3 +1504,361 @@ TEST_F(ParquetChunkedReaderTest, TestChunkedReadOutOfBoundChunks) CUDF_TEST_EXPECT_TABLES_EQUAL(*expected, *result); } } + +TEST_F(ParquetChunkedReaderTest, TestNumRowsPerSource) +{ + constexpr int num_rows = 10'723; // A prime number + constexpr int rows_in_row_group = 500; + + // Table with single col of random int64 values + auto const int64_data = random_values(num_rows); + auto int64_col = int64s_col(int64_data.begin(), int64_data.end()).release(); + + std::vector> input_columns; + input_columns.emplace_back(std::move(int64_col)); + + // Write to Parquet + auto const [expected, filepath] = write_file(input_columns, + "num_rows_per_source", + false, + false, + cudf::io::default_max_page_size_bytes, + rows_in_row_group); + + // Chunked-read single data source entirely + { + auto constexpr output_read_limit = 1'500; + auto constexpr pass_read_limit = 3'500; + + auto const options = + cudf::io::parquet_reader_options_builder(cudf::io::source_info{filepath}).build(); + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected->view(), result->view()); + EXPECT_EQ(num_rows_per_source.size(), 1); + EXPECT_EQ(num_rows_per_source[0], num_rows); + } + + // Chunked-read rows_to_read rows skipping rows_to_skip from single data source + { + auto const rows_to_skip = 1'237; + auto const rows_to_read = 7'232; + auto constexpr output_read_limit = 1'500; + auto constexpr pass_read_limit = 3'500; + + auto const options = cudf::io::parquet_reader_options_builder(cudf::io::source_info{filepath}) + .skip_rows(rows_to_skip) + .num_rows(rows_to_read) + .build(); + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + auto int64_col_selected = int64s_col(int64_data.begin() + rows_to_skip, + int64_data.begin() + rows_to_skip + rows_to_read) + .release(); + + cudf::table_view const expected_selected({int64_col_selected->view()}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_selected, result->view()); + EXPECT_EQ(num_rows_per_source.size(), 1); + EXPECT_EQ(num_rows_per_source[0], rows_to_read); + } + + // Chunked-read two data sources skipping the first entire file completely + { + auto constexpr rows_to_skip = 15'723; + auto constexpr output_read_limit = 1'024'000; + auto constexpr pass_read_limit = 1'024'000; + + auto constexpr nsources = 2; + std::vector const datasources(nsources, filepath); + + auto const options = + cudf::io::parquet_reader_options_builder(cudf::io::source_info{datasources}) + .skip_rows(rows_to_skip) + .build(); + + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + auto int64_col_selected = + int64s_col(int64_data.begin() + rows_to_skip - num_rows, int64_data.end()).release(); + + cudf::table_view const expected_selected({int64_col_selected->view()}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_selected, result->view()); + EXPECT_EQ(num_rows_per_source.size(), 2); + EXPECT_EQ(num_rows_per_source[0], 0); + EXPECT_EQ(num_rows_per_source[1], nsources * num_rows - rows_to_skip); + } + + // Chunked-read from single data source skipping rows_to_skip + { + auto const rows_to_skip = 1'237; + auto constexpr output_read_limit = 1'500; + auto constexpr pass_read_limit = 1'800; + + auto const options = cudf::io::parquet_reader_options_builder(cudf::io::source_info{filepath}) + .skip_rows(rows_to_skip) + .build(); + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + auto int64_col_selected = + int64s_col(int64_data.begin() + rows_to_skip, int64_data.end()).release(); + + cudf::table_view const expected_selected({int64_col_selected->view()}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_selected, result->view()); + EXPECT_EQ(num_rows_per_source.size(), 1); + EXPECT_EQ(num_rows_per_source[0], num_rows - rows_to_skip); + } + + // Filtered chunked-read from single data source + { + int64_t const max_value = int64_data[int64_data.size() / 2]; + auto constexpr output_read_limit = 1'500; + auto constexpr pass_read_limit = 3'500; + auto literal_value = cudf::numeric_scalar{max_value}; + auto literal = cudf::ast::literal{literal_value}; + auto col_ref = cudf::ast::column_reference(0); + auto filter_expression = + cudf::ast::operation(cudf::ast::ast_operator::LESS_EQUAL, col_ref, literal); + + auto const options = cudf::io::parquet_reader_options_builder(cudf::io::source_info{filepath}) + .filter(filter_expression) + .build(); + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + std::vector int64_data_filtered; + int64_data_filtered.reserve(num_rows); + std::copy_if( + int64_data.begin(), int64_data.end(), std::back_inserter(int64_data_filtered), [=](auto val) { + return val <= max_value; + }); + + auto int64_col_filtered = + int64s_col(int64_data_filtered.begin(), int64_data_filtered.end()).release(); + + cudf::table_view expected_filtered({int64_col_filtered->view()}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_filtered, result->view()); + EXPECT_TRUE(num_rows_per_source.empty()); + } +} + +TEST_F(ParquetChunkedReaderTest, TestNumRowsPerSourceMultipleSources) +{ + constexpr int num_rows = 10'723; // A prime number + constexpr int rows_in_row_group = 500; + + // Table with single col of random int64 values + auto const int64_data = random_values(num_rows); + auto int64_col = int64s_col(int64_data.begin(), int64_data.end()).release(); + + std::vector> input_columns; + input_columns.emplace_back(std::move(int64_col)); + + // Write to Parquet + auto const [expected, filepath] = write_file(input_columns, + "num_rows_per_source", + false, + false, + cudf::io::default_max_page_size_bytes, + rows_in_row_group); + + // Function to initialize a vector of expected counts per source + auto initialize_expected_counts = + [](int const nsources, int const num_rows, int const rows_to_skip, int const rows_to_read) { + // Initialize expected_counts + std::vector expected_counts(nsources, num_rows); + + // Adjust expected_counts for rows_to_skip + int64_t counter = 0; + for (auto& nrows : expected_counts) { + if (counter < rows_to_skip) { + counter += nrows; + nrows = (counter >= rows_to_skip) ? counter - rows_to_skip : 0; + } else { + break; + } + } + + // Reset the counter + counter = 0; + + // Adjust expected_counts for rows_to_read + for (auto& nrows : expected_counts) { + if (counter < rows_to_read) { + counter += nrows; + nrows = (counter >= rows_to_read) ? rows_to_read - counter + nrows : nrows; + } else if (counter > rows_to_read) { + nrows = 0; + } + } + + return expected_counts; + }; + + // Chunked-read six data sources entirely + { + auto const nsources = 6; + auto constexpr output_read_limit = 15'000; + auto constexpr pass_read_limit = 35'000; + std::vector const datasources(nsources, filepath); + + auto const options = + cudf::io::parquet_reader_options_builder(cudf::io::source_info{datasources}).build(); + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + // Initialize expected_counts + std::vector const expected_counts(nsources, num_rows); + + EXPECT_EQ(num_rows_per_source.size(), nsources); + EXPECT_TRUE( + std::equal(expected_counts.cbegin(), expected_counts.cend(), num_rows_per_source.cbegin())); + } + + // Chunked-read rows_to_read rows skipping rows_to_skip from eight data sources + { + auto const rows_to_skip = 25'571; + auto const rows_to_read = 41'232; + auto constexpr output_read_limit = 15'000; + auto constexpr pass_read_limit = 35'000; + auto const nsources = 8; + std::vector int64_selected_data{}; + int64_selected_data.reserve(nsources * num_rows); + + std::for_each( + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(nsources), + [&](auto const i) { + std::copy(int64_data.begin(), int64_data.end(), std::back_inserter(int64_selected_data)); + }); + + std::vector const datasources(nsources, filepath); + + auto const options = + cudf::io::parquet_reader_options_builder(cudf::io::source_info{datasources}) + .skip_rows(rows_to_skip) + .num_rows(rows_to_read) + .build(); + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + // Initialize expected_counts + auto const expected_counts = + initialize_expected_counts(nsources, num_rows, rows_to_skip, rows_to_read); + + // Initialize expected table + auto int64_col_selected = int64s_col(int64_selected_data.begin() + rows_to_skip, + int64_selected_data.begin() + +rows_to_skip + rows_to_read) + .release(); + + cudf::table_view const expected_selected({int64_col_selected->view()}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_selected, result->view()); + EXPECT_EQ(num_rows_per_source.size(), nsources); + EXPECT_TRUE( + std::equal(expected_counts.cbegin(), expected_counts.cend(), num_rows_per_source.cbegin())); + } + + // Chunked-read four data sources skipping three files completely + { + auto const nsources = 4; + int constexpr rows_to_skip = num_rows * 3 + 1; + auto constexpr output_read_limit = 15'000; + auto constexpr pass_read_limit = 35'000; + std::vector int64_selected_data{}; + int64_selected_data.reserve(nsources * num_rows); + + std::for_each( + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(nsources), + [&](auto const i) { + std::copy(int64_data.begin(), int64_data.end(), std::back_inserter(int64_selected_data)); + }); + + std::vector const datasources(nsources, filepath); + auto const options = + cudf::io::parquet_reader_options_builder(cudf::io::source_info{datasources}) + .skip_rows(rows_to_skip) + .build(); + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + // Initialize expected_counts + auto const expected_counts = + initialize_expected_counts(nsources, num_rows, rows_to_skip, num_rows * nsources); + + // Initialize expected table + auto int64_col_selected = + int64s_col(int64_selected_data.begin() + rows_to_skip, int64_selected_data.end()).release(); + + cudf::table_view const expected_selected({int64_col_selected->view()}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_selected, result->view()); + EXPECT_EQ(num_rows_per_source.size(), nsources); + EXPECT_TRUE( + std::equal(expected_counts.cbegin(), expected_counts.cend(), num_rows_per_source.cbegin())); + } +} + +TEST_F(ParquetChunkedReaderTest, TestNumRowsPerSourceEmptyTable) +{ + auto constexpr output_read_limit = 4'500; + auto constexpr pass_read_limit = 8'500; + auto const nsources = 10; + + // Table with single col of random int64 values + auto int64_empty_col = int64s_col{}.release(); + + std::vector> input_empty_columns; + input_empty_columns.emplace_back(std::move(int64_empty_col)); + + // Write to Parquet + auto const [expected_empty, filepath_empty] = write_file(input_empty_columns, + "num_rows_per_source_empty", + false, + false, + cudf::io::default_max_page_size_bytes, + 500); + + std::vector const datasources(nsources, filepath_empty); + + auto const options = + cudf::io::parquet_reader_options_builder(cudf::io::source_info{datasources}).build(); + auto const reader = cudf::io::chunked_parquet_reader( + output_read_limit, pass_read_limit, options, cudf::get_default_stream()); + + auto const [result, num_chunks, num_rows_per_source] = read_table_and_nrows_per_source(reader); + + // Initialize expected_counts + std::vector const expected_counts(nsources, 0); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_empty->view(), result->view()); + + EXPECT_EQ(num_chunks, 1); + EXPECT_EQ(num_rows_per_source.size(), nsources); + EXPECT_TRUE( + std::equal(expected_counts.cbegin(), expected_counts.cend(), num_rows_per_source.cbegin())); +} diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 2edf9e0aee6..6c61535359f 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -2243,6 +2243,209 @@ TEST_F(ParquetReaderTest, StringsWithPageStats) } } +TEST_F(ParquetReaderTest, NumRowsPerSource) +{ + int constexpr num_rows = 10'723; // A prime number + int constexpr rows_in_row_group = 500; + + // Table with single col of random int64 values + auto const int64_data = random_values(num_rows); + column_wrapper const int64_col{ + int64_data.begin(), int64_data.end(), cudf::test::iterators::no_nulls()}; + cudf::table_view const expected({int64_col}); + + // Write to Parquet + auto const filepath = temp_env->get_temp_filepath("NumRowsPerSource.parquet"); + auto const out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .row_group_size_rows(rows_in_row_group) + .build(); + cudf::io::write_parquet(out_opts); + + // Read single data source entirely + { + auto const in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}).build(); + auto const result = cudf::io::read_parquet(in_opts); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + EXPECT_EQ(result.metadata.num_rows_per_source.size(), 1); + EXPECT_EQ(result.metadata.num_rows_per_source[0], num_rows); + } + + // Read rows_to_read rows skipping rows_to_skip from single data source + { + auto constexpr rows_to_skip = 557; // a prime number != rows_in_row_group + auto constexpr rows_to_read = 7'232; + auto const in_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .skip_rows(rows_to_skip) + .num_rows(rows_to_read) + .build(); + auto const result = cudf::io::read_parquet(in_opts); + column_wrapper int64_col_selected{int64_data.begin() + rows_to_skip, + int64_data.begin() + rows_to_skip + rows_to_read, + cudf::test::iterators::no_nulls()}; + + cudf::table_view const expected_selected({int64_col_selected}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_selected, result.tbl->view()); + EXPECT_EQ(result.metadata.num_rows_per_source.size(), 1); + EXPECT_EQ(result.metadata.num_rows_per_source[0], rows_to_read); + } + + // Filtered read from single data source + { + auto constexpr max_value = 100; + auto literal_value = cudf::numeric_scalar{max_value}; + auto literal = cudf::ast::literal{literal_value}; + auto col_ref = cudf::ast::column_reference(0); + auto filter_expression = + cudf::ast::operation(cudf::ast::ast_operator::LESS_EQUAL, col_ref, literal); + + auto const in_opts = cudf::io::parquet_reader_options::builder(cudf::io::source_info{filepath}) + .filter(filter_expression) + .build(); + + std::vector int64_data_filtered; + int64_data_filtered.reserve(num_rows); + std::copy_if( + int64_data.begin(), int64_data.end(), std::back_inserter(int64_data_filtered), [=](auto val) { + return val <= max_value; + }); + column_wrapper int64_col_filtered{ + int64_data_filtered.begin(), int64_data_filtered.end(), cudf::test::iterators::no_nulls()}; + + cudf::table_view expected_filtered({int64_col_filtered}); + + auto const result = cudf::io::read_parquet(in_opts); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_filtered, result.tbl->view()); + EXPECT_EQ(result.metadata.num_rows_per_source.size(), 0); + } + + // Read two data sources skipping the first entire file completely + { + auto constexpr rows_to_skip = 15'723; + auto constexpr nsources = 2; + std::vector const datasources(nsources, filepath); + + auto const in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{datasources}) + .skip_rows(rows_to_skip) + .build(); + + auto const result = cudf::io::read_parquet(in_opts); + + column_wrapper int64_col_selected{int64_data.begin() + rows_to_skip - num_rows, + int64_data.end(), + cudf::test::iterators::no_nulls()}; + + cudf::table_view const expected_selected({int64_col_selected}); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected_selected, result.tbl->view()); + EXPECT_EQ(result.metadata.num_rows_per_source.size(), 2); + EXPECT_EQ(result.metadata.num_rows_per_source[0], 0); + EXPECT_EQ(result.metadata.num_rows_per_source[1], nsources * num_rows - rows_to_skip); + } + + // Read ten data sources entirely + { + auto constexpr nsources = 10; + std::vector const datasources(nsources, filepath); + + auto const in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{datasources}).build(); + auto const result = cudf::io::read_parquet(in_opts); + + // Initialize expected_counts + std::vector const expected_counts(nsources, num_rows); + + EXPECT_EQ(result.metadata.num_rows_per_source.size(), nsources); + EXPECT_TRUE(std::equal(expected_counts.cbegin(), + expected_counts.cend(), + result.metadata.num_rows_per_source.cbegin())); + } + + // Read rows_to_read rows skipping rows_to_skip (> two sources) from ten data sources + { + auto constexpr rows_to_skip = 25'999; + auto constexpr rows_to_read = 47'232; + + auto constexpr nsources = 10; + std::vector const datasources(nsources, filepath); + + auto const in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{datasources}) + .skip_rows(rows_to_skip) + .num_rows(rows_to_read) + .build(); + + auto const result = cudf::io::read_parquet(in_opts); + + // Initialize expected_counts + std::vector expected_counts(nsources, num_rows); + + // Adjust expected_counts for rows_to_skip + int64_t counter = 0; + for (auto& nrows : expected_counts) { + if (counter < rows_to_skip) { + counter += nrows; + nrows = (counter >= rows_to_skip) ? counter - rows_to_skip : 0; + } else { + break; + } + } + + // Reset the counter + counter = 0; + + // Adjust expected_counts for rows_to_read + for (auto& nrows : expected_counts) { + if (counter < rows_to_read) { + counter += nrows; + nrows = (counter >= rows_to_read) ? rows_to_read - counter + nrows : nrows; + } else if (counter > rows_to_read) { + nrows = 0; + } + } + + EXPECT_EQ(result.metadata.num_rows_per_source.size(), nsources); + EXPECT_TRUE(std::equal(expected_counts.cbegin(), + expected_counts.cend(), + result.metadata.num_rows_per_source.cbegin())); + } +} + +TEST_F(ParquetReaderTest, NumRowsPerSourceEmptyTable) +{ + auto const nsources = 10; + + column_wrapper const int64_empty_col{}; + cudf::table_view const expected_empty({int64_empty_col}); + + // Write to Parquet + auto const filepath_empty = temp_env->get_temp_filepath("NumRowsPerSourceEmpty.parquet"); + auto const out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath_empty}, expected_empty) + .build(); + cudf::io::write_parquet(out_opts); + + // Read from Parquet + std::vector const datasources(nsources, filepath_empty); + + auto const in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{datasources}).build(); + auto const result = cudf::io::read_parquet(in_opts); + + // Initialize expected_counts + std::vector const expected_counts(nsources, 0); + + EXPECT_EQ(result.metadata.num_rows_per_source.size(), nsources); + EXPECT_TRUE(std::equal(expected_counts.cbegin(), + expected_counts.cend(), + result.metadata.num_rows_per_source.cbegin())); +} + /////////////////// // metadata tests diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pxd index 8d87deb1472..0a6bddcd907 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/io/types.pxd @@ -81,6 +81,7 @@ cdef extern from "cudf/io/types.hpp" \ map[string, string] user_data vector[unordered_map[string, string]] per_file_user_data vector[column_name_info] schema_info + vector[size_t] num_rows_per_source cdef cppclass table_with_metadata: unique_ptr[table] tbl From 26a3799d2ff9ffb2aa72d63bb388b4bee70b3440 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:49:01 -1000 Subject: [PATCH 51/53] Make ColumnAccessor strictly require a mapping of columns (#16285) `ColumnAccessor` had a default `data=None` argument and initialized an empty dict in the `__init__` if `data` was not passed. This PR now makes `data` a required argument. Additionally if `verify=True`, the `__init__` would call `as_column` on each `data.values()` allowing non-`ColumnBase` inputs. This PR now avoids this call and makes the caller responsible for ensuring the inputs are `ColumnBase`s Also, adds a few `verify=False` internally where we know we are passing columns from a libcudf op or reconstructing from another `ColumnAccessor` Authors: - Matthew Roeschke (https://github.com/mroeschke) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16285 --- python/cudf/cudf/core/_base_index.py | 4 +- python/cudf/cudf/core/column_accessor.py | 64 +++--- python/cudf/cudf/core/dataframe.py | 24 ++- python/cudf/cudf/core/frame.py | 2 +- python/cudf/cudf/core/groupby/groupby.py | 4 +- python/cudf/cudf/core/index.py | 4 +- python/cudf/cudf/core/indexed_frame.py | 1 + python/cudf/cudf/core/reshape.py | 12 +- python/cudf/cudf/core/series.py | 27 ++- .../cudf/cudf/tests/test_column_accessor.py | 190 ++++++++++++------ 10 files changed, 211 insertions(+), 121 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index c38352009de..8fad82c5c46 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -98,7 +98,7 @@ def astype(self, dtype, copy: bool = True): """ raise NotImplementedError - def argsort(self, *args, **kwargs): + def argsort(self, *args, **kwargs) -> cupy.ndarray: """Return the integer indices that would sort the index. Parameters vary by subclass. @@ -1520,7 +1520,7 @@ def sort_values( ascending=True, na_position="last", key=None, - ): + ) -> Self | tuple[Self, cupy.ndarray]: """ Return a sorted copy of the index, and optionally return the indices that sorted the index itself. diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index f30a557efb0..819d351b2c4 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -16,6 +16,8 @@ from cudf.core import column if TYPE_CHECKING: + from typing_extensions import Self + from cudf._typing import Dtype from cudf.core.column import ColumnBase @@ -86,58 +88,58 @@ class ColumnAccessor(abc.MutableMapping): (default=None). verify : bool, optional For non ColumnAccessor inputs, whether to verify - column length and type + column length and data.values() are all Columns """ - _data: "dict[Any, ColumnBase]" - multiindex: bool + _data: dict[Any, ColumnBase] _level_names: tuple[Any, ...] def __init__( self, - data: abc.MutableMapping | ColumnAccessor | None = None, + data: abc.MutableMapping[Any, ColumnBase] | Self, multiindex: bool = False, level_names=None, rangeindex: bool = False, label_dtype: Dtype | None = None, verify: bool = True, ): - self.rangeindex = rangeindex - self.label_dtype = label_dtype - if data is None: - data = {} - # TODO: we should validate the keys of `data` if isinstance(data, ColumnAccessor): - multiindex = multiindex or data.multiindex - level_names = level_names or data.level_names self._data = data._data - self.multiindex = multiindex - self._level_names = level_names - self.rangeindex = data.rangeindex - self.label_dtype = data.label_dtype - else: + self._level_names = data.level_names + self.multiindex: bool = data.multiindex + self.rangeindex: bool = data.rangeindex + self.label_dtype: Dtype | None = data.label_dtype + elif isinstance(data, abc.MutableMapping): # This code path is performance-critical for copies and should be # modified with care. - data = dict(data) if data and verify: - result = {} # Faster than next(iter(data.values())) column_length = len(data[next(iter(data))]) - for k, v in data.items(): - # Much faster to avoid the function call if possible; the - # extra isinstance is negligible if we do have to make a - # column from something else. - if not isinstance(v, column.ColumnBase): - v = column.as_column(v) - if len(v) != column_length: + # TODO: we should validate the keys of `data` + for col in data.values(): + if not isinstance(col, column.ColumnBase): + raise ValueError( + f"All data.values() must be Column, not {type(col).__name__}" + ) + if len(col) != column_length: raise ValueError("All columns must be of equal length") - result[k] = v - self._data = result - else: - self._data = data + if not isinstance(data, dict): + data = dict(data) + self._data = data + + if rangeindex and multiindex: + raise ValueError( + f"{rangeindex=} and {multiindex=} cannot both be True." + ) + self.rangeindex = rangeindex self.multiindex = multiindex + self.label_dtype = label_dtype self._level_names = level_names + else: + raise ValueError( + f"data must be a ColumnAccessor or MutableMapping, not {type(data).__name__}" + ) def __iter__(self): return iter(self._data) @@ -161,7 +163,9 @@ def __repr__(self) -> str: type_info = ( f"{self.__class__.__name__}(" f"multiindex={self.multiindex}, " - f"level_names={self.level_names})" + f"level_names={self.level_names}, " + f"rangeindex={self.rangeindex}, " + f"label_dtype={self.label_dtype})" ) column_info = "\n".join( [f"{name}: {col.dtype}" for name, col in self.items()] diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 7e07078c95b..dbc7f10b569 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -475,6 +475,7 @@ def __getitem__(self, arg): {key: ca._data[key] for key in column_names}, multiindex=ca.multiindex, level_names=ca.level_names, + verify=False, ), index=index, ) @@ -485,6 +486,7 @@ def __getitem__(self, arg): {key: ca._data[key] for key in column_names}, multiindex=ca.multiindex, level_names=ca.level_names, + verify=False, ), index=index, ) @@ -771,6 +773,7 @@ def __init__( else None, rangeindex=rangeindex, label_dtype=label_dtype, + verify=False, ) elif isinstance(data, ColumnAccessor): raise TypeError( @@ -931,7 +934,7 @@ def _init_from_series_list(self, data, columns, index): ) if not series.index.equals(final_columns): series = series.reindex(final_columns) - self._data[idx] = column.as_column(series._column) + self._data[idx] = series._column # Setting `final_columns` to self._index so # that the resulting `transpose` will be have @@ -2958,7 +2961,7 @@ def set_index( # label-like if is_scalar(col) or isinstance(col, tuple): if col in self._column_names: - data_to_add.append(self[col]) + data_to_add.append(self[col]._column) names.append(col) if drop: to_drop.append(col) @@ -2973,7 +2976,7 @@ def set_index( elif isinstance( col, (cudf.Series, cudf.Index, pd.Series, pd.Index) ): - data_to_add.append(col) + data_to_add.append(as_column(col)) names.append(col.name) else: try: @@ -4769,7 +4772,7 @@ def _func(x): # pragma: no cover result = {} for name, col in self._data.items(): apply_sr = Series._from_data({None: col}) - result[name] = apply_sr.apply(_func) + result[name] = apply_sr.apply(_func)._column return DataFrame._from_data(result, index=self.index) @@ -5806,6 +5809,7 @@ def from_records( ), level_names=level_names, label_dtype=getattr(columns, "dtype", None), + verify=False, ), index=new_index, ) @@ -5892,6 +5896,7 @@ def _from_arrays( ), level_names=level_names, label_dtype=getattr(columns, "dtype", None), + verify=False, ), index=index, ) @@ -6302,10 +6307,9 @@ def count(self, axis=0, numeric_only=False): length = len(self) return Series._from_data( { - None: [ - length - self._data[col].null_count - for col in self._data.names - ] + None: as_column( + [length - col.null_count for col in self._columns] + ) }, cudf.Index(self._data.names), ) @@ -7374,7 +7378,9 @@ def to_struct(self, name=None): offset=0, ) return cudf.Series._from_data( - cudf.core.column_accessor.ColumnAccessor({name: col}), + cudf.core.column_accessor.ColumnAccessor( + {name: col}, verify=False + ), index=self.index, name=name, ) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index c82e073d7b7..04ecae4ba85 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -1305,7 +1305,7 @@ def argsort( order=None, ascending=True, na_position="last", - ): + ) -> cupy.ndarray: """Return the integer indices that would sort the Series values. Parameters diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 3f91be71f29..1646c5042fd 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1360,7 +1360,9 @@ def _post_process_chunk_results( if isinstance(chunk_results, ColumnBase) or cudf.api.types.is_scalar( chunk_results[0] ): - data = {None: chunk_results} + data = ColumnAccessor( + {None: as_column(chunk_results)}, verify=False + ) ty = cudf.Series if self._as_index else cudf.DataFrame result = ty._from_data(data, index=group_names) result.index.names = self.grouping.names diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index ae20fcd5d9c..73b7298410a 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -349,7 +349,7 @@ def hasnans(self) -> bool: @_performance_tracking def _data(self): return cudf.core.column_accessor.ColumnAccessor( - {self.name: self._values} + {self.name: self._values}, verify=False ) @_performance_tracking @@ -1492,7 +1492,7 @@ def argsort( order=None, ascending=True, na_position="last", - ): + ) -> cupy.ndarray: """Return the integer indices that would sort the index. Parameters diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 60cd142db4b..e75b51e0d43 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -6229,6 +6229,7 @@ def rank( multiindex=self._data.multiindex, level_names=self._data.level_names, label_dtype=self._data.label_dtype, + verify=False, ), ) else: diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index b538ae34b6f..a542c5f5969 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -932,14 +932,10 @@ def _pivot(df, index, columns): index_labels, index_idx = index._encode() column_labels = columns_labels.to_pandas().to_flat_index() - # the result of pivot always has a multicolumn - result = cudf.core.column_accessor.ColumnAccessor( - multiindex=True, level_names=(None,) + columns._data.names - ) - def as_tuple(x): return x if isinstance(x, tuple) else (x,) + result = {} for v in df: names = [as_tuple(v) + as_tuple(name) for name in column_labels] nrows = len(index_labels) @@ -964,8 +960,12 @@ def as_tuple(x): } ) + # the result of pivot always has a multicolumn + ca = cudf.core.column_accessor.ColumnAccessor( + result, multiindex=True, level_names=(None,) + columns._data.names + ) return cudf.DataFrame._from_data( - result, index=cudf.Index(index_labels, name=index.name) + ca, index=cudf.Index(index_labels, name=index.name) ) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index d8dbaa897e7..94c33eed37a 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -2263,20 +2263,19 @@ def argsort( order=None, ascending=True, na_position="last", - ): - obj = self.__class__._from_data( - { - None: super().argsort( - axis=axis, - kind=kind, - order=order, - ascending=ascending, - na_position=na_position, - ) - } + ) -> Self: + col = as_column( + super().argsort( + axis=axis, + kind=kind, + order=order, + ascending=ascending, + na_position=na_position, + ) + ) + return self._from_data_like_self( + self._data._from_columns_like_self([col]) ) - obj.name = self.name - return obj @_performance_tracking def replace(self, to_replace=None, value=no_default, *args, **kwargs): @@ -2631,7 +2630,7 @@ def mode(self, dropna=True): val_counts = val_counts[val_counts == val_counts.iloc[0]] return Series._from_data( - {self.name: val_counts.index.sort_values()}, name=self.name + {self.name: val_counts.index.sort_values()._column}, name=self.name ) @_performance_tracking diff --git a/python/cudf/cudf/tests/test_column_accessor.py b/python/cudf/cudf/tests/test_column_accessor.py index f3343c37d1d..e84e1433c10 100644 --- a/python/cudf/cudf/tests/test_column_accessor.py +++ b/python/cudf/cudf/tests/test_column_accessor.py @@ -5,28 +5,35 @@ import pytest import cudf +from cudf.core.column import as_column from cudf.core.column_accessor import ColumnAccessor from cudf.testing import assert_eq simple_test_data = [ {}, - {"a": []}, - {"a": [1]}, - {"a": ["a"]}, - {"a": [1, 2, 3], "b": ["a", "b", "c"]}, + {"a": as_column([])}, + {"a": as_column([1])}, + {"a": as_column(["a"])}, + {"a": as_column([1, 2, 3]), "b": as_column(["a", "b", "c"])}, ] mi_test_data = [ - {("a", "b"): [1, 2, 4], ("a", "c"): [2, 3, 4]}, - {("a", "b"): [1, 2, 3], ("a", ""): [2, 3, 4]}, - {("a", "b"): [1, 2, 4], ("c", "d"): [2, 3, 4]}, - {("a", "b"): [1, 2, 3], ("a", "c"): [2, 3, 4], ("b", ""): [4, 5, 6]}, + {("a", "b"): as_column([1, 2, 4]), ("a", "c"): as_column([2, 3, 4])}, + {("a", "b"): as_column([1, 2, 3]), ("a", ""): as_column([2, 3, 4])}, + {("a", "b"): as_column([1, 2, 4]), ("c", "d"): as_column([2, 3, 4])}, + { + ("a", "b"): as_column([1, 2, 3]), + ("a", "c"): as_column([2, 3, 4]), + ("b", ""): as_column([4, 5, 6]), + }, ] def check_ca_equal(lhs, rhs): assert lhs.level_names == rhs.level_names assert lhs.multiindex == rhs.multiindex + assert lhs.rangeindex == rhs.rangeindex + assert lhs.label_dtype == rhs.label_dtype for l_key, r_key in zip(lhs, rhs): assert l_key == r_key assert_eq(lhs[l_key], rhs[r_key]) @@ -58,19 +65,26 @@ def test_to_pandas_simple(simple_data): # to ignore this `inferred_type` comparison, we pass exact=False. assert_eq( ca.to_pandas_index(), - pd.DataFrame(simple_data).columns, + pd.DataFrame( + {key: value.values_host for key, value in simple_data.items()} + ).columns, exact=False, ) def test_to_pandas_multiindex(mi_data): ca = ColumnAccessor(mi_data, multiindex=True) - assert_eq(ca.to_pandas_index(), pd.DataFrame(mi_data).columns) + assert_eq( + ca.to_pandas_index(), + pd.DataFrame( + {key: value.values_host for key, value in mi_data.items()} + ).columns, + ) def test_to_pandas_multiindex_names(): ca = ColumnAccessor( - {("a", "b"): [1, 2, 3], ("c", "d"): [3, 4, 5]}, + {("a", "b"): as_column([1, 2, 3]), ("c", "d"): as_column([3, 4, 5])}, multiindex=True, level_names=("foo", "bar"), ) @@ -108,16 +122,20 @@ def test_column_size_mismatch(): differing sizes throws an error. """ with pytest.raises(ValueError): - ColumnAccessor({"a": [1], "b": [1, 2]}) + ColumnAccessor({"a": as_column([1]), "b": as_column([1, 2])}) def test_select_by_label_simple(): """ Test getting a column by label """ - ca = ColumnAccessor({"a": [1, 2, 3], "b": [2, 3, 4]}) - check_ca_equal(ca.select_by_label("a"), ColumnAccessor({"a": [1, 2, 3]})) - check_ca_equal(ca.select_by_label("b"), ColumnAccessor({"b": [2, 3, 4]})) + ca = ColumnAccessor({"a": as_column([1, 2, 3]), "b": as_column([2, 3, 4])}) + check_ca_equal( + ca.select_by_label("a"), ColumnAccessor({"a": as_column([1, 2, 3])}) + ) + check_ca_equal( + ca.select_by_label("b"), ColumnAccessor({"b": as_column([2, 3, 4])}) + ) def test_select_by_label_multiindex(): @@ -126,40 +144,62 @@ def test_select_by_label_multiindex(): """ ca = ColumnAccessor( { - ("a", "b", "c"): [1, 2, 3], - ("a", "b", "e"): [2, 3, 4], - ("b", "x", ""): [4, 5, 6], - ("a", "d", "e"): [3, 4, 5], + ("a", "b", "c"): as_column([1, 2, 3]), + ("a", "b", "e"): as_column([2, 3, 4]), + ("b", "x", ""): as_column([4, 5, 6]), + ("a", "d", "e"): as_column([3, 4, 5]), }, multiindex=True, ) expect = ColumnAccessor( - {("b", "c"): [1, 2, 3], ("b", "e"): [2, 3, 4], ("d", "e"): [3, 4, 5]}, + { + ("b", "c"): as_column([1, 2, 3]), + ("b", "e"): as_column([2, 3, 4]), + ("d", "e"): as_column([3, 4, 5]), + }, multiindex=True, ) got = ca.select_by_label("a") check_ca_equal(expect, got) - expect = ColumnAccessor({"c": [1, 2, 3], "e": [2, 3, 4]}, multiindex=False) + expect = ColumnAccessor( + {"c": as_column([1, 2, 3]), "e": as_column([2, 3, 4])}, + multiindex=False, + ) got = ca.select_by_label(("a", "b")) check_ca_equal(expect, got) expect = ColumnAccessor( - {("b", "c"): [1, 2, 3], ("b", "e"): [2, 3, 4], ("d", "e"): [3, 4, 5]}, + { + ("b", "c"): as_column([1, 2, 3]), + ("b", "e"): as_column([2, 3, 4]), + ("d", "e"): as_column([3, 4, 5]), + }, multiindex=True, ) got = ca.select_by_label("a") check_ca_equal(expect, got) - expect = ColumnAccessor({"c": [1, 2, 3], "e": [2, 3, 4]}, multiindex=False) + expect = ColumnAccessor( + {"c": as_column([1, 2, 3]), "e": as_column([2, 3, 4])}, + multiindex=False, + ) got = ca.select_by_label(("a", "b")) check_ca_equal(expect, got) def test_select_by_label_simple_slice(): - ca = ColumnAccessor({"a": [1, 2, 3], "b": [2, 3, 4], "c": [3, 4, 5]}) - expect = ColumnAccessor({"b": [2, 3, 4], "c": [3, 4, 5]}) + ca = ColumnAccessor( + { + "a": as_column([1, 2, 3]), + "b": as_column([2, 3, 4]), + "c": as_column([3, 4, 5]), + } + ) + expect = ColumnAccessor( + {"b": as_column([2, 3, 4]), "c": as_column([3, 4, 5])} + ) got = ca.select_by_label(slice("b", "c")) check_ca_equal(expect, got) @@ -167,10 +207,10 @@ def test_select_by_label_simple_slice(): def test_select_by_label_multiindex_slice(): ca = ColumnAccessor( { - ("a", "b", "c"): [1, 2, 3], - ("a", "b", "e"): [2, 3, 4], - ("a", "d", "e"): [3, 4, 5], - ("b", "x", ""): [4, 5, 6], + ("a", "b", "c"): as_column([1, 2, 3]), + ("a", "b", "e"): as_column([2, 3, 4]), + ("a", "d", "e"): as_column([3, 4, 5]), + ("b", "x", ""): as_column([4, 5, 6]), }, multiindex=True, ) # pandas needs columns to be sorted to do slicing with multiindex @@ -180,9 +220,9 @@ def test_select_by_label_multiindex_slice(): expect = ColumnAccessor( { - ("a", "b", "e"): [2, 3, 4], - ("a", "d", "e"): [3, 4, 5], - ("b", "x", ""): [4, 5, 6], + ("a", "b", "e"): as_column([2, 3, 4]), + ("a", "d", "e"): as_column([3, 4, 5]), + ("b", "x", ""): as_column([4, 5, 6]), }, multiindex=True, ) @@ -191,8 +231,16 @@ def test_select_by_label_multiindex_slice(): def test_by_label_list(): - ca = ColumnAccessor({"a": [1, 2, 3], "b": [2, 3, 4], "c": [3, 4, 5]}) - expect = ColumnAccessor({"b": [2, 3, 4], "c": [3, 4, 5]}) + ca = ColumnAccessor( + { + "a": as_column([1, 2, 3]), + "b": as_column([2, 3, 4]), + "c": as_column([3, 4, 5]), + } + ) + expect = ColumnAccessor( + {"b": as_column([2, 3, 4]), "c": as_column([3, 4, 5])} + ) got = ca.select_by_label(["b", "c"]) check_ca_equal(expect, got) @@ -201,9 +249,13 @@ def test_select_by_index_simple(): """ Test getting a column by label """ - ca = ColumnAccessor({"a": [1, 2, 3], "b": [2, 3, 4]}) - check_ca_equal(ca.select_by_index(0), ColumnAccessor({"a": [1, 2, 3]})) - check_ca_equal(ca.select_by_index(1), ColumnAccessor({"b": [2, 3, 4]})) + ca = ColumnAccessor({"a": as_column([1, 2, 3]), "b": as_column([2, 3, 4])}) + check_ca_equal( + ca.select_by_index(0), ColumnAccessor({"a": as_column([1, 2, 3])}) + ) + check_ca_equal( + ca.select_by_index(1), ColumnAccessor({"b": as_column([2, 3, 4])}) + ) check_ca_equal(ca.select_by_index([0, 1]), ca) check_ca_equal(ca.select_by_index(slice(0, None)), ca) @@ -214,19 +266,19 @@ def test_select_by_index_multiindex(): """ ca = ColumnAccessor( { - ("a", "b", "c"): [1, 2, 3], - ("a", "b", "e"): [2, 3, 4], - ("b", "x", ""): [4, 5, 6], - ("a", "d", "e"): [3, 4, 5], + ("a", "b", "c"): as_column([1, 2, 3]), + ("a", "b", "e"): as_column([2, 3, 4]), + ("b", "x", ""): as_column([4, 5, 6]), + ("a", "d", "e"): as_column([3, 4, 5]), }, multiindex=True, ) expect = ColumnAccessor( { - ("a", "b", "c"): [1, 2, 3], - ("a", "b", "e"): [2, 3, 4], - ("b", "x", ""): [4, 5, 6], + ("a", "b", "c"): as_column([1, 2, 3]), + ("a", "b", "e"): as_column([2, 3, 4]), + ("b", "x", ""): as_column([4, 5, 6]), }, multiindex=True, ) @@ -235,9 +287,9 @@ def test_select_by_index_multiindex(): expect = ColumnAccessor( { - ("a", "b", "c"): [1, 2, 3], - ("a", "b", "e"): [2, 3, 4], - ("a", "d", "e"): [3, 4, 5], + ("a", "b", "c"): as_column([1, 2, 3]), + ("a", "b", "e"): as_column([2, 3, 4]), + ("a", "d", "e"): as_column([3, 4, 5]), }, multiindex=True, ) @@ -248,10 +300,10 @@ def test_select_by_index_multiindex(): def test_select_by_index_empty(): ca = ColumnAccessor( { - ("a", "b", "c"): [1, 2, 3], - ("a", "b", "e"): [2, 3, 4], - ("b", "x", ""): [4, 5, 6], - ("a", "d", "e"): [3, 4, 5], + ("a", "b", "c"): as_column([1, 2, 3]), + ("a", "b", "e"): as_column([2, 3, 4]), + ("b", "x", ""): as_column([4, 5, 6]), + ("a", "d", "e"): as_column([3, 4, 5]), }, multiindex=True, ) @@ -267,12 +319,20 @@ def test_select_by_index_empty(): def test_replace_level_values_RangeIndex(): ca = ColumnAccessor( - {("a"): [1, 2, 3], ("b"): [2, 3, 4], ("c"): [3, 4, 5]}, + { + ("a"): as_column([1, 2, 3]), + ("b"): as_column([2, 3, 4]), + ("c"): as_column([3, 4, 5]), + }, multiindex=False, ) expect = ColumnAccessor( - {("f"): [1, 2, 3], ("b"): [2, 3, 4], ("c"): [3, 4, 5]}, + { + ("f"): as_column([1, 2, 3]), + ("b"): as_column([2, 3, 4]), + ("c"): as_column([3, 4, 5]), + }, multiindex=False, ) @@ -282,12 +342,20 @@ def test_replace_level_values_RangeIndex(): def test_replace_level_values_MultiColumn(): ca = ColumnAccessor( - {("a", 1): [1, 2, 3], ("a", 2): [2, 3, 4], ("b", 1): [3, 4, 5]}, + { + ("a", 1): as_column([1, 2, 3]), + ("a", 2): as_column([2, 3, 4]), + ("b", 1): as_column([3, 4, 5]), + }, multiindex=True, ) expect = ColumnAccessor( - {("f", 1): [1, 2, 3], ("f", 2): [2, 3, 4], ("b", 1): [3, 4, 5]}, + { + ("f", 1): as_column([1, 2, 3]), + ("f", 2): as_column([2, 3, 4]), + ("b", 1): as_column([3, 4, 5]), + }, multiindex=True, ) @@ -303,7 +371,17 @@ def test_clear_nrows_empty_before(): def test_clear_nrows_empty_after(): - ca = ColumnAccessor({"new": [1]}) + ca = ColumnAccessor({"new": as_column([1])}) assert ca.nrows == 1 del ca["new"] assert ca.nrows == 0 + + +def test_not_rangeindex_and_multiindex(): + with pytest.raises(ValueError): + ColumnAccessor({}, multiindex=True, rangeindex=True) + + +def test_data_values_not_column_raises(): + with pytest.raises(ValueError): + ColumnAccessor({"a": [1]}) From c5b96003cef00b2635923d03edcd48a13821a61e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 19 Jul 2024 20:04:19 -0700 Subject: [PATCH 52/53] Migrate Parquet reader to pylibcudf (#16078) xref #15162 Migrates the parquet reader (and chunked parquet reader) to pylibcudf. (Does not migrate the writers or the metadata reader yet). Authors: - Thomas Li (https://github.com/lithomas1) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/16078 --- .../api_docs/pylibcudf/io/index.rst | 1 + .../api_docs/pylibcudf/io/parquet.rst | 6 + python/cudf/cudf/_lib/parquet.pyx | 312 ++++++------------ .../cudf/cudf/_lib/pylibcudf/expressions.pyx | 11 + .../cudf/_lib/pylibcudf/io/CMakeLists.txt | 4 +- .../cudf/cudf/_lib/pylibcudf/io/__init__.pxd | 2 +- .../cudf/cudf/_lib/pylibcudf/io/__init__.py | 2 +- .../cudf/cudf/_lib/pylibcudf/io/parquet.pxd | 35 ++ .../cudf/cudf/_lib/pylibcudf/io/parquet.pyx | 204 ++++++++++++ python/cudf/cudf/_lib/pylibcudf/io/types.pyx | 8 + .../_lib/pylibcudf/libcudf/io/parquet.pxd | 8 +- python/cudf/cudf/io/parquet.py | 4 +- .../cudf/cudf/pylibcudf_tests/common/utils.py | 80 ++++- python/cudf/cudf/pylibcudf_tests/conftest.py | 15 + .../cudf/pylibcudf_tests/io/test_parquet.py | 109 ++++++ python/cudf/cudf/tests/test_parquet.py | 5 +- 16 files changed, 581 insertions(+), 225 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/io/parquet.rst create mode 100644 python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx create mode 100644 python/cudf/cudf/pylibcudf_tests/io/test_parquet.py diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst index 697bce739de..e2d342ffe47 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/index.rst @@ -18,3 +18,4 @@ I/O Functions avro csv json + parquet diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/io/parquet.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/parquet.rst new file mode 100644 index 00000000000..9dfbadfa216 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/io/parquet.rst @@ -0,0 +1,6 @@ +======= +Parquet +======= + +.. automodule:: cudf._lib.pylibcudf.io.parquet + :members: diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index e7959d21e01..a2eed94bb3c 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -18,16 +18,14 @@ from cython.operator cimport dereference from cudf.api.types import is_list_like -from cudf._lib.utils cimport data_from_unique_ptr +from cudf._lib.utils cimport _data_from_columns, data_from_pylibcudf_io -from cudf._lib import pylibcudf from cudf._lib.utils import _index_level_name, generate_pandas_metadata from libc.stdint cimport uint8_t from libcpp cimport bool from libcpp.map cimport map from libcpp.memory cimport make_unique, unique_ptr -from libcpp.pair cimport pair from libcpp.string cimport string from libcpp.unordered_map cimport unordered_map from libcpp.utility cimport move @@ -35,25 +33,20 @@ from libcpp.vector cimport vector cimport cudf._lib.pylibcudf.libcudf.io.data_sink as cudf_io_data_sink cimport cudf._lib.pylibcudf.libcudf.io.types as cudf_io_types -cimport cudf._lib.pylibcudf.libcudf.types as cudf_types from cudf._lib.column cimport Column from cudf._lib.io.utils cimport ( + add_df_col_struct_names, make_sinks_info, make_source_info, - update_struct_field_names, ) from cudf._lib.pylibcudf.expressions cimport Expression from cudf._lib.pylibcudf.io.datasource cimport NativeFileDatasource -from cudf._lib.pylibcudf.libcudf.expressions cimport expression +from cudf._lib.pylibcudf.io.parquet cimport ChunkedParquetReader from cudf._lib.pylibcudf.libcudf.io.parquet cimport ( - chunked_parquet_reader as cpp_chunked_parquet_reader, chunked_parquet_writer_options, merge_row_group_metadata as parquet_merge_metadata, parquet_chunked_writer as cpp_parquet_chunked_writer, - parquet_reader_options, - parquet_reader_options_builder, parquet_writer_options, - read_parquet as parquet_reader, write_parquet as parquet_writer, ) from cudf._lib.pylibcudf.libcudf.io.parquet_metadata cimport ( @@ -63,19 +56,17 @@ from cudf._lib.pylibcudf.libcudf.io.parquet_metadata cimport ( from cudf._lib.pylibcudf.libcudf.io.types cimport ( column_in_metadata, table_input_metadata, - table_metadata, ) from cudf._lib.pylibcudf.libcudf.table.table_view cimport table_view -from cudf._lib.pylibcudf.libcudf.types cimport data_type, size_type +from cudf._lib.pylibcudf.libcudf.types cimport size_type from cudf._lib.utils cimport table_view_from_table from pyarrow.lib import NativeFile -from cudf._lib.concat import concat_columns +import cudf._lib.pylibcudf as plc +from cudf._lib.pylibcudf cimport Table from cudf.utils.ioutils import _ROW_GROUP_SIZE_BYTES_DEFAULT -from cudf._lib.utils cimport data_from_pylibcudf_table - cdef class BufferArrayFromVector: cdef Py_ssize_t length @@ -133,71 +124,37 @@ def _parse_metadata(meta): return file_is_range_index, file_index_cols, file_column_dtype -cdef pair[parquet_reader_options, bool] _setup_parquet_reader_options( - cudf_io_types.source_info source, - vector[vector[size_type]] row_groups, - bool use_pandas_metadata, - Expression filters, - object columns): - - cdef parquet_reader_options args - cdef parquet_reader_options_builder builder - cdef data_type cpp_timestamp_type = cudf_types.data_type( - cudf_types.type_id.EMPTY - ) - builder = ( - parquet_reader_options.builder(source) - .row_groups(row_groups) - .use_pandas_metadata(use_pandas_metadata) - .use_arrow_schema(True) - .timestamp_type(cpp_timestamp_type) - ) - if filters is not None: - builder = builder.filter(dereference(filters.c_obj.get())) - - args = move(builder.build()) - cdef vector[string] cpp_columns - allow_range_index = True - if columns is not None: - cpp_columns.reserve(len(columns)) - allow_range_index = len(columns) > 0 - for col in columns: - cpp_columns.push_back(str(col).encode()) - args.set_columns(cpp_columns) - allow_range_index &= filters is None - - return pair[parquet_reader_options, bool](args, allow_range_index) - cdef object _process_metadata(object df, - table_metadata table_meta, list names, + dict child_names, + list per_file_user_data, object row_groups, object filepaths_or_buffers, list pa_buffers, bool allow_range_index, bool use_pandas_metadata): - update_struct_field_names(df, table_meta.schema_info) + + add_df_col_struct_names(df, child_names) index_col = None is_range_index = True column_index_type = None index_col_names = None meta = None - cdef vector[unordered_map[string, string]] per_file_user_data = \ - table_meta.per_file_user_data for single_file in per_file_user_data: + if b'pandas' not in single_file: + continue json_str = single_file[b'pandas'].decode('utf-8') - if json_str != "": - meta = json.loads(json_str) - file_is_range_index, index_col, column_index_type = _parse_metadata(meta) - is_range_index &= file_is_range_index - - if not file_is_range_index and index_col is not None \ - and index_col_names is None: - index_col_names = {} - for idx_col in index_col: - for c in meta['columns']: - if c['field_name'] == idx_col: - index_col_names[idx_col] = c['name'] + meta = json.loads(json_str) + file_is_range_index, index_col, column_index_type = _parse_metadata(meta) + is_range_index &= file_is_range_index + + if not file_is_range_index and index_col is not None \ + and index_col_names is None: + index_col_names = {} + for idx_col in index_col: + for c in meta['columns']: + if c['field_name'] == idx_col: + index_col_names[idx_col] = c['name'] if meta is not None: # Book keep each column metadata as the order @@ -297,6 +254,76 @@ cdef object _process_metadata(object df, return df +def read_parquet_chunked( + filepaths_or_buffers, + columns=None, + row_groups=None, + use_pandas_metadata=True, + size_t chunk_read_limit=0, + size_t pass_read_limit=1024000000 +): + # Convert NativeFile buffers to NativeFileDatasource, + # but save original buffers in case we need to use + # pyarrow for metadata processing + # (See: https://github.com/rapidsai/cudf/issues/9599) + + pa_buffers = [] + + new_bufs = [] + for i, datasource in enumerate(filepaths_or_buffers): + if isinstance(datasource, NativeFile): + new_bufs.append(NativeFileDatasource(datasource)) + else: + new_bufs.append(datasource) + + # Note: If this function ever takes accepts filters + # allow_range_index needs to be False when a filter is passed + # (see read_parquet) + allow_range_index = columns is not None and len(columns) != 0 + + reader = ChunkedParquetReader( + plc.io.SourceInfo(new_bufs), + columns, + row_groups, + use_pandas_metadata, + chunk_read_limit=chunk_read_limit, + pass_read_limit=pass_read_limit + ) + + tbl_w_meta = reader.read_chunk() + column_names = tbl_w_meta.column_names(include_children=False) + child_names = tbl_w_meta.child_names + per_file_user_data = tbl_w_meta.per_file_user_data + concatenated_columns = tbl_w_meta.tbl.columns() + + # save memory + del tbl_w_meta + + cdef Table tbl + while reader.has_next(): + tbl = reader.read_chunk().tbl + + for i in range(tbl.num_columns()): + concatenated_columns[i] = plc.concatenate.concatenate( + [concatenated_columns[i], tbl._columns[i]] + ) + # Drop residual columns to save memory + tbl._columns[i] = None + + df = cudf.DataFrame._from_data( + *_data_from_columns( + columns=[Column.from_pylibcudf(plc) for plc in concatenated_columns], + column_names=column_names, + index_names=None + ) + ) + df = _process_metadata(df, column_names, child_names, + per_file_user_data, row_groups, + filepaths_or_buffers, pa_buffers, + allow_range_index, use_pandas_metadata) + return df + + cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, use_pandas_metadata=True, Expression filters=None): @@ -322,33 +349,28 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, pa_buffers.append(datasource) filepaths_or_buffers[i] = NativeFileDatasource(datasource) - cdef cudf_io_types.source_info source = make_source_info( - filepaths_or_buffers) - - cdef vector[vector[size_type]] cpp_row_groups - if row_groups is not None: - cpp_row_groups = row_groups - - # Setup parquet reader arguments - cdef parquet_reader_options args - cdef pair[parquet_reader_options, bool] c_res = _setup_parquet_reader_options( - source, cpp_row_groups, use_pandas_metadata, filters, columns) - args, allow_range_index = c_res.first, c_res.second + allow_range_index = True + if columns is not None and len(columns) == 0 or filters: + allow_range_index = False # Read Parquet - cdef cudf_io_types.table_with_metadata c_result - with nogil: - c_result = move(parquet_reader(args)) + tbl_w_meta = plc.io.parquet.read_parquet( + plc.io.SourceInfo(filepaths_or_buffers), + columns, + row_groups, + filters, + convert_strings_to_categories = False, + use_pandas_metadata = use_pandas_metadata, + ) - names = [info.name.decode() for info in c_result.metadata.schema_info] + df = cudf.DataFrame._from_data( + *data_from_pylibcudf_io(tbl_w_meta) + ) - df = cudf.DataFrame._from_data(*data_from_unique_ptr( - move(c_result.tbl), - column_names=names - )) - df = _process_metadata(df, c_result.metadata, names, row_groups, - filepaths_or_buffers, pa_buffers, + df = _process_metadata(df, tbl_w_meta.column_names(include_children=False), + tbl_w_meta.child_names, tbl_w_meta.per_file_user_data, + row_groups, filepaths_or_buffers, pa_buffers, allow_range_index, use_pandas_metadata) return df @@ -804,120 +826,6 @@ cdef class ParquetWriter: self.initialized = True -cdef class ParquetReader: - cdef bool initialized - cdef unique_ptr[cpp_chunked_parquet_reader] reader - cdef size_t chunk_read_limit - cdef size_t pass_read_limit - cdef size_t row_group_size_bytes - cdef table_metadata result_meta - cdef vector[unordered_map[string, string]] per_file_user_data - cdef object pandas_meta - cdef list pa_buffers - cdef bool allow_range_index - cdef object row_groups - cdef object filepaths_or_buffers - cdef object names - cdef object column_index_type - cdef object index_col_names - cdef bool is_range_index - cdef object index_col - cdef bool cpp_use_pandas_metadata - - def __cinit__(self, filepaths_or_buffers, columns=None, row_groups=None, - use_pandas_metadata=True, - size_t chunk_read_limit=0, - size_t pass_read_limit=1024000000): - - # Convert NativeFile buffers to NativeFileDatasource, - # but save original buffers in case we need to use - # pyarrow for metadata processing - # (See: https://github.com/rapidsai/cudf/issues/9599) - - pa_buffers = [] - for i, datasource in enumerate(filepaths_or_buffers): - if isinstance(datasource, NativeFile): - pa_buffers.append(datasource) - filepaths_or_buffers[i] = NativeFileDatasource(datasource) - self.pa_buffers = pa_buffers - cdef cudf_io_types.source_info source = make_source_info( - filepaths_or_buffers) - - self.cpp_use_pandas_metadata = use_pandas_metadata - - cdef vector[vector[size_type]] cpp_row_groups - if row_groups is not None: - cpp_row_groups = row_groups - cdef parquet_reader_options args - cdef pair[parquet_reader_options, bool] c_res = _setup_parquet_reader_options( - source, cpp_row_groups, use_pandas_metadata, None, columns) - args, self.allow_range_index = c_res.first, c_res.second - - with nogil: - self.reader.reset( - new cpp_chunked_parquet_reader( - chunk_read_limit, - pass_read_limit, - args - ) - ) - self.initialized = False - self.row_groups = row_groups - self.filepaths_or_buffers = filepaths_or_buffers - - def _has_next(self): - cdef bool res - with nogil: - res = self.reader.get()[0].has_next() - return res - - def _read_chunk(self): - # Read Parquet - cdef cudf_io_types.table_with_metadata c_result - - with nogil: - c_result = move(self.reader.get()[0].read_chunk()) - - if not self.initialized: - self.names = [info.name.decode() for info in c_result.metadata.schema_info] - self.result_meta = c_result.metadata - - df = cudf.DataFrame._from_data(*data_from_unique_ptr( - move(c_result.tbl), - column_names=self.names, - )) - - self.initialized = True - return df - - def read(self): - dfs = self._read_chunk() - column_names = dfs._column_names - concatenated_columns = list(dfs._columns) - del dfs - while self._has_next(): - new_chunk = list(self._read_chunk()._columns) - for i in range(len(column_names)): - concatenated_columns[i] = concat_columns( - [concatenated_columns[i], new_chunk[i]] - ) - # Must drop any residual GPU columns to save memory - new_chunk[i] = None - - dfs = cudf.DataFrame._from_data( - *data_from_pylibcudf_table( - pylibcudf.Table( - [col.to_pylibcudf(mode="read") for col in concatenated_columns] - ), - column_names=column_names, - index_names=None - ) - ) - - return _process_metadata(dfs, self.result_meta, self.names, self.row_groups, - self.filepaths_or_buffers, self.pa_buffers, - self.allow_range_index, self.cpp_use_pandas_metadata) - cpdef merge_filemetadata(object filemetadata_list): """ Cython function to call into libcudf API, see `merge_row_group_metadata`. diff --git a/python/cudf/cudf/_lib/pylibcudf/expressions.pyx b/python/cudf/cudf/_lib/pylibcudf/expressions.pyx index 38de11406ad..b983a617533 100644 --- a/python/cudf/cudf/_lib/pylibcudf/expressions.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/expressions.pyx @@ -38,6 +38,17 @@ from .types cimport DataType # Aliases for simplicity ctypedef unique_ptr[libcudf_exp.expression] expression_ptr +# Define this class just to have a docstring for it +cdef class Expression: + """ + The base class for all expression types. + This class cannot be instantiated directly, please + instantiate one of its child classes instead. + + For details, see :cpp:class:`cudf::ast::expression`. + """ + pass + cdef class Literal(Expression): """ A literal value used in an abstract syntax tree. diff --git a/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt index 8dd08d11dc8..55bea4fc262 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/io/CMakeLists.txt @@ -12,7 +12,7 @@ # the License. # ============================================================================= -set(cython_sources avro.pyx csv.pyx datasource.pyx json.pyx types.pyx) +set(cython_sources avro.pyx csv.pyx datasource.pyx json.pyx parquet.pyx types.pyx) set(linked_libraries cudf::cudf) rapids_cython_create_modules( @@ -22,6 +22,6 @@ rapids_cython_create_modules( ) set(targets_using_arrow_headers pylibcudf_io_avro pylibcudf_io_csv pylibcudf_io_datasource - pylibcudf_io_json pylibcudf_io_types + pylibcudf_io_json pylibcudf_io_parquet pylibcudf_io_types ) link_to_pyarrow_headers("${targets_using_arrow_headers}") diff --git a/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd index 5b3272d60e0..62820048584 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/io/__init__.pxd @@ -1,5 +1,5 @@ # Copyright (c) 2024, NVIDIA CORPORATION. # CSV is removed since it is def not cpdef (to force kw-only arguments) -from . cimport avro, datasource, json, types +from . cimport avro, datasource, json, parquet, types from .types cimport SourceInfo, TableWithMetadata diff --git a/python/cudf/cudf/_lib/pylibcudf/io/__init__.py b/python/cudf/cudf/_lib/pylibcudf/io/__init__.py index e17deaa4663..27640f7d955 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/io/__init__.py @@ -1,4 +1,4 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from . import avro, csv, datasource, json, types +from . import avro, csv, datasource, json, parquet, types from .types import SinkInfo, SourceInfo, TableWithMetadata diff --git a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd b/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd new file mode 100644 index 00000000000..027f215fb91 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/io/parquet.pxd @@ -0,0 +1,35 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libc.stdint cimport int64_t +from libcpp cimport bool +from libcpp.memory cimport unique_ptr + +from cudf._lib.pylibcudf.expressions cimport Expression +from cudf._lib.pylibcudf.io.types cimport SourceInfo, TableWithMetadata +from cudf._lib.pylibcudf.libcudf.io.parquet cimport ( + chunked_parquet_reader as cpp_chunked_parquet_reader, +) +from cudf._lib.pylibcudf.libcudf.types cimport size_type +from cudf._lib.pylibcudf.types cimport DataType + + +cdef class ChunkedParquetReader: + cdef unique_ptr[cpp_chunked_parquet_reader] reader + + cpdef bool has_next(self) + cpdef TableWithMetadata read_chunk(self) + + +cpdef read_parquet( + SourceInfo source_info, + list columns = *, + list row_groups = *, + Expression filters = *, + bool convert_strings_to_categories = *, + bool use_pandas_metadata = *, + int64_t skip_rows = *, + size_type num_rows = *, + # disabled see comment in parquet.pyx for more + # ReaderColumnSchema reader_column_schema = *, + # DataType timestamp_type = * +) diff --git a/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx b/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx new file mode 100644 index 00000000000..96119e1b714 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/io/parquet.pyx @@ -0,0 +1,204 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +from cython.operator cimport dereference +from libc.stdint cimport int64_t +from libcpp cimport bool +from libcpp.string cimport string +from libcpp.utility cimport move +from libcpp.vector cimport vector + +from cudf._lib.pylibcudf.expressions cimport Expression +from cudf._lib.pylibcudf.io.types cimport SourceInfo, TableWithMetadata +from cudf._lib.pylibcudf.libcudf.expressions cimport expression +from cudf._lib.pylibcudf.libcudf.io.parquet cimport ( + chunked_parquet_reader as cpp_chunked_parquet_reader, + parquet_reader_options, + read_parquet as cpp_read_parquet, +) +from cudf._lib.pylibcudf.libcudf.io.types cimport table_with_metadata +from cudf._lib.pylibcudf.libcudf.types cimport size_type + + +cdef parquet_reader_options _setup_parquet_reader_options( + SourceInfo source_info, + list columns = None, + list row_groups = None, + Expression filters = None, + bool convert_strings_to_categories = False, + bool use_pandas_metadata = True, + int64_t skip_rows = 0, + size_type num_rows = -1, + # ReaderColumnSchema reader_column_schema = None, + # DataType timestamp_type = DataType(type_id.EMPTY) +): + cdef vector[string] col_vec + cdef parquet_reader_options opts = ( + parquet_reader_options.builder(source_info.c_obj) + .convert_strings_to_categories(convert_strings_to_categories) + .use_pandas_metadata(use_pandas_metadata) + .use_arrow_schema(True) + .build() + ) + if row_groups is not None: + opts.set_row_groups(row_groups) + if num_rows != -1: + opts.set_num_rows(num_rows) + if skip_rows != 0: + opts.set_skip_rows(skip_rows) + if columns is not None: + col_vec.reserve(len(columns)) + for col in columns: + col_vec.push_back(str(col).encode()) + opts.set_columns(col_vec) + if filters is not None: + opts.set_filter(dereference(filters.c_obj.get())) + return opts + + +cdef class ChunkedParquetReader: + """ + Reads chunks of a Parquet file into a :py:class:`~.types.TableWithMetadata`. + + Parameters + ---------- + source_info : SourceInfo + The SourceInfo object to read the Parquet file from. + columns : list, default None + The names of the columns to be read + row_groups : list[list[size_type]], default None + List of row groups to be read. + use_pandas_metadata : bool, default True + If True, return metadata about the index column in + the per-file user metadata of the ``TableWithMetadata`` + convert_strings_to_categories : bool, default False + Whether to convert string columns to the category type + skip_rows : int64_t, default 0 + The number of rows to skip from the start of the file. + num_rows : size_type, default -1 + The number of rows to read. By default, read the entire file. + chunk_read_limit : size_t, default 0 + Limit on total number of bytes to be returned per read, + or 0 if there is no limit. + pass_read_limit : size_t, default 1024000000 + Limit on the amount of memory used for reading and decompressing data + or 0 if there is no limit. + """ + def __init__( + self, + SourceInfo source_info, + list columns=None, + list row_groups=None, + bool use_pandas_metadata=True, + bool convert_strings_to_categories=False, + int64_t skip_rows = 0, + size_type num_rows = -1, + size_t chunk_read_limit=0, + size_t pass_read_limit=1024000000 + ): + + cdef parquet_reader_options opts = _setup_parquet_reader_options( + source_info, + columns, + row_groups, + filters=None, + convert_strings_to_categories=convert_strings_to_categories, + use_pandas_metadata=use_pandas_metadata, + skip_rows=skip_rows, + num_rows=num_rows, + ) + + with nogil: + self.reader.reset( + new cpp_chunked_parquet_reader( + chunk_read_limit, + pass_read_limit, + opts + ) + ) + + cpdef bool has_next(self): + """ + Returns True if there is another chunk in the Parquet file + to be read. + + Returns + ------- + True if we have not finished reading the file. + """ + with nogil: + return self.reader.get()[0].has_next() + + cpdef TableWithMetadata read_chunk(self): + """ + Read the next chunk into a :py:class:`~.types.TableWithMetadata` + + Returns + ------- + TableWithMetadata + The Table and its corresponding metadata (column names) that were read in. + """ + # Read Parquet + cdef table_with_metadata c_result + + with nogil: + c_result = move(self.reader.get()[0].read_chunk()) + + return TableWithMetadata.from_libcudf(c_result) + +cpdef read_parquet( + SourceInfo source_info, + list columns = None, + list row_groups = None, + Expression filters = None, + bool convert_strings_to_categories = False, + bool use_pandas_metadata = True, + int64_t skip_rows = 0, + size_type num_rows = -1, + # Disabled, these aren't used by cudf-python + # we should only add them back in if there's user demand + # ReaderColumnSchema reader_column_schema = None, + # DataType timestamp_type = DataType(type_id.EMPTY) +): + """Reads an Parquet file into a :py:class:`~.types.TableWithMetadata`. + + Parameters + ---------- + source_info : SourceInfo + The SourceInfo object to read the Parquet file from. + columns : list, default None + The string names of the columns to be read. + row_groups : list[list[size_type]], default None + List of row groups to be read. + filters : Expression, default None + An AST :py:class:`cudf._lib.pylibcudf.expressions.Expression` + to use for predicate pushdown. + convert_strings_to_categories : bool, default False + Whether to convert string columns to the category type + use_pandas_metadata : bool, default True + If True, return metadata about the index column in + the per-file user metadata of the ``TableWithMetadata`` + skip_rows : int64_t, default 0 + The number of rows to skip from the start of the file. + num_rows : size_type, default -1 + The number of rows to read. By default, read the entire file. + + Returns + ------- + TableWithMetadata + The Table and its corresponding metadata (column names) that were read in. + """ + cdef table_with_metadata c_result + cdef parquet_reader_options opts = _setup_parquet_reader_options( + source_info, + columns, + row_groups, + filters, + convert_strings_to_categories, + use_pandas_metadata, + skip_rows, + num_rows, + ) + + with nogil: + c_result = move(cpp_read_parquet(opts)) + + return TableWithMetadata.from_libcudf(c_result) diff --git a/python/cudf/cudf/_lib/pylibcudf/io/types.pyx b/python/cudf/cudf/_lib/pylibcudf/io/types.pyx index 68498ff88f4..95fa7d4c2ee 100644 --- a/python/cudf/cudf/_lib/pylibcudf/io/types.pyx +++ b/python/cudf/cudf/_lib/pylibcudf/io/types.pyx @@ -122,6 +122,14 @@ cdef class TableWithMetadata: out.metadata = tbl_with_meta.metadata return out + @property + def per_file_user_data(self): + """ + Returns a list containing a dict + containing file-format specific metadata, + for each file being read in. + """ + return self.metadata.per_file_user_data cdef class SourceInfo: """A class containing details on a source to read from. diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet.pxd index c38f39f7749..d86915c7da9 100644 --- a/python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/io/parquet.pxd @@ -1,6 +1,6 @@ # Copyright (c) 2020-2024, NVIDIA CORPORATION. -from libc.stdint cimport uint8_t +from libc.stdint cimport int64_t, uint8_t from libcpp cimport bool from libcpp.functional cimport reference_wrapper from libcpp.map cimport map @@ -27,8 +27,11 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: # setter + void set_filter(expression &filter) except + void set_columns(vector[string] col_names) except + + void set_num_rows(size_type val) except + void set_row_groups(vector[vector[size_type]] row_grp) except + + void set_skip_rows(int64_t val) except + void enable_use_arrow_schema(bool val) except + void enable_use_pandas_metadata(bool val) except + void set_timestamp_type(data_type type) except + @@ -49,6 +52,9 @@ cdef extern from "cudf/io/parquet.hpp" namespace "cudf::io" nogil: parquet_reader_options_builder& row_groups( vector[vector[size_type]] row_grp ) except + + parquet_reader_options_builder& convert_strings_to_categories( + bool val + ) except + parquet_reader_options_builder& use_pandas_metadata( bool val ) except + diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 0f0a240b5d0..7dab2f20100 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -929,12 +929,12 @@ def _read_parquet( f"following positional arguments: {list(args)}" ) if cudf.get_option("io.parquet.low_memory"): - return libparquet.ParquetReader( + return libparquet.read_parquet_chunked( filepaths_or_buffers, columns=columns, row_groups=row_groups, use_pandas_metadata=use_pandas_metadata, - ).read() + ) else: return libparquet.read_parquet( filepaths_or_buffers, diff --git a/python/cudf/cudf/pylibcudf_tests/common/utils.py b/python/cudf/cudf/pylibcudf_tests/common/utils.py index ed2c5ca06c9..e19ff58927f 100644 --- a/python/cudf/cudf/pylibcudf_tests/common/utils.py +++ b/python/cudf/cudf/pylibcudf_tests/common/utils.py @@ -7,6 +7,7 @@ import numpy as np import pyarrow as pa import pytest +from pyarrow.parquet import write_table as pq_write_table from cudf._lib import pylibcudf as plc from cudf._lib.pylibcudf.io.types import CompressionType @@ -103,25 +104,68 @@ def _make_fields_nullable(typ): return pa.list_(new_fields[0]) return typ + def _contains_type(parent_typ, typ_checker): + """ + Check whether the parent or one of the children + satisfies the typ_checker. + """ + if typ_checker(parent_typ): + return True + if pa.types.is_nested(parent_typ): + for i in range(parent_typ.num_fields): + if _contains_type(parent_typ.field(i).type, typ_checker): + return True + return False + if not check_field_nullability: rhs_type = _make_fields_nullable(rhs.type) rhs = rhs.cast(rhs_type) lhs_type = _make_fields_nullable(lhs.type) - lhs = rhs.cast(lhs_type) - - if pa.types.is_floating(lhs.type) and pa.types.is_floating(rhs.type): - lhs_nans = pa.compute.is_nan(lhs) - rhs_nans = pa.compute.is_nan(rhs) - assert lhs_nans.equals(rhs_nans) - - if pa.compute.any(lhs_nans) or pa.compute.any(rhs_nans): - # masks must be equal at this point - mask = pa.compute.fill_null(pa.compute.invert(lhs_nans), True) - lhs = lhs.filter(mask) - rhs = rhs.filter(mask) + lhs = lhs.cast(lhs_type) - np.testing.assert_array_almost_equal(lhs, rhs) + assert lhs.type == rhs.type, f"{lhs.type} != {rhs.type}" + if _contains_type(lhs.type, pa.types.is_floating) and _contains_type( + rhs.type, pa.types.is_floating + ): + # Flatten nested arrays to liststo do comparisons if nested + # This is so we can do approximate comparisons + # for floats in numpy + def _flatten_arrays(arr): + if pa.types.is_nested(arr.type): + flattened = arr.flatten() + flat_arrs = [] + if isinstance(flattened, list): + for flat_arr in flattened: + flat_arrs += _flatten_arrays(flat_arr) + else: + flat_arrs = [flattened] + else: + flat_arrs = [arr] + return flat_arrs + + if isinstance(lhs, (pa.ListArray, pa.StructArray)): + lhs = _flatten_arrays(lhs) + rhs = _flatten_arrays(rhs) + else: + # Just a regular doublearray + lhs = [lhs] + rhs = [rhs] + + for lh_arr, rh_arr in zip(lhs, rhs): + # Check NaNs positions match + # and then filter out nans + lhs_nans = pa.compute.is_nan(lh_arr) + rhs_nans = pa.compute.is_nan(rh_arr) + assert lhs_nans.equals(rhs_nans) + + if pa.compute.any(lhs_nans) or pa.compute.any(rhs_nans): + # masks must be equal at this point + mask = pa.compute.fill_null(pa.compute.invert(lhs_nans), True) + lh_arr = lh_arr.filter(mask) + rh_arr = rh_arr.filter(mask) + + np.testing.assert_array_almost_equal(lh_arr, rh_arr) else: assert lhs.equals(rhs) @@ -276,6 +320,16 @@ def make_source(path_or_buf, pa_table, format, **kwargs): df.to_json(path_or_buf, mode=mode, **kwargs) elif format == "csv": df.to_csv(path_or_buf, mode=mode, **kwargs) + elif format == "parquet": + # The conversion to pandas is lossy (doesn't preserve + # nested types) so we + # will just use pyarrow directly to write this + pq_write_table( + pa_table, + pa.PythonFile(path_or_buf) + if isinstance(path_or_buf, io.IOBase) + else path_or_buf, + ) if isinstance(path_or_buf, io.IOBase): path_or_buf.seek(0) return path_or_buf diff --git a/python/cudf/cudf/pylibcudf_tests/conftest.py b/python/cudf/cudf/pylibcudf_tests/conftest.py index 4a7194a6d8d..945e1689229 100644 --- a/python/cudf/cudf/pylibcudf_tests/conftest.py +++ b/python/cudf/cudf/pylibcudf_tests/conftest.py @@ -170,6 +170,21 @@ def source_or_sink(request, tmp_path): return fp_or_buf() +@pytest.fixture( + params=["a.txt", pathlib.Path("a.txt"), io.BytesIO], +) +def binary_source_or_sink(request, tmp_path): + fp_or_buf = request.param + if isinstance(fp_or_buf, str): + return f"{tmp_path}/{fp_or_buf}" + elif isinstance(fp_or_buf, os.PathLike): + return tmp_path.joinpath(fp_or_buf) + elif issubclass(fp_or_buf, io.IOBase): + # Must construct io.StringIO/io.BytesIO inside + # fixture, or we'll end up re-using it + return fp_or_buf() + + unsupported_types = { # Not supported by pandas # TODO: find a way to test these diff --git a/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py b/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py new file mode 100644 index 00000000000..07d2ab3d69a --- /dev/null +++ b/python/cudf/cudf/pylibcudf_tests/io/test_parquet.py @@ -0,0 +1,109 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +import pyarrow as pa +import pyarrow.compute as pc +import pytest +from pyarrow.parquet import read_table +from utils import assert_table_and_meta_eq, make_source + +import cudf._lib.pylibcudf as plc +from cudf._lib.pylibcudf.expressions import ( + ASTOperator, + ColumnNameReference, + ColumnReference, + Literal, + Operation, +) + +# Shared kwargs to pass to make_source +_COMMON_PARQUET_SOURCE_KWARGS = {"format": "parquet"} + + +@pytest.mark.parametrize("columns", [None, ["col_int64", "col_bool"]]) +def test_read_parquet_basic( + table_data, binary_source_or_sink, nrows_skiprows, columns +): + _, pa_table = table_data + nrows, skiprows = nrows_skiprows + + source = make_source( + binary_source_or_sink, pa_table, **_COMMON_PARQUET_SOURCE_KWARGS + ) + + res = plc.io.parquet.read_parquet( + plc.io.SourceInfo([source]), + num_rows=nrows, + skip_rows=skiprows, + columns=columns, + ) + + if columns is not None: + pa_table = pa_table.select(columns) + + # Adapt to nrows/skiprows + pa_table = pa_table.slice( + offset=skiprows, length=nrows if nrows != -1 else None + ) + + assert_table_and_meta_eq(pa_table, res, check_field_nullability=False) + + +@pytest.mark.parametrize( + "pa_filters,plc_filters", + [ + ( + pc.field("col_int64") >= 10, + Operation( + ASTOperator.GREATER_EQUAL, + ColumnNameReference("col_int64"), + Literal(plc.interop.from_arrow(pa.scalar(10))), + ), + ), + ( + (pc.field("col_int64") >= 10) & (pc.field("col_double") < 0), + Operation( + ASTOperator.LOGICAL_AND, + Operation( + ASTOperator.GREATER_EQUAL, + ColumnNameReference("col_int64"), + Literal(plc.interop.from_arrow(pa.scalar(10))), + ), + Operation( + ASTOperator.LESS, + ColumnNameReference("col_double"), + Literal(plc.interop.from_arrow(pa.scalar(0.0))), + ), + ), + ), + ( + (pc.field(0) == 10), + Operation( + ASTOperator.EQUAL, + ColumnReference(0), + Literal(plc.interop.from_arrow(pa.scalar(10))), + ), + ), + ], +) +def test_read_parquet_filters( + table_data, binary_source_or_sink, pa_filters, plc_filters +): + _, pa_table = table_data + + source = make_source( + binary_source_or_sink, pa_table, **_COMMON_PARQUET_SOURCE_KWARGS + ) + + plc_table_w_meta = plc.io.parquet.read_parquet( + plc.io.SourceInfo([source]), filters=plc_filters + ) + exp = read_table(source, filters=pa_filters) + assert_table_and_meta_eq( + exp, plc_table_w_meta, check_field_nullability=False + ) + + +# TODO: Test these options +# list row_groups = None, +# ^^^ This one is not tested since it's not in pyarrow/pandas, deprecate? +# bool convert_strings_to_categories = False, +# bool use_pandas_metadata = True diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index f2820d9c112..3806b901b10 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -22,7 +22,7 @@ from pyarrow import fs as pa_fs, parquet as pq import cudf -from cudf._lib.parquet import ParquetReader +from cudf._lib.parquet import read_parquet_chunked from cudf.io.parquet import ( ParquetDatasetWriter, ParquetWriter, @@ -3755,7 +3755,7 @@ def test_parquet_chunked_reader( ) buffer = BytesIO() df.to_parquet(buffer) - reader = ParquetReader( + actual = read_parquet_chunked( [buffer], chunk_read_limit=chunk_read_limit, pass_read_limit=pass_read_limit, @@ -3765,7 +3765,6 @@ def test_parquet_chunked_reader( expected = cudf.read_parquet( buffer, use_pandas_metadata=use_pandas_metadata, row_groups=row_groups ) - actual = reader.read() assert_eq(expected, actual) From e6537de7474c91b4153542e6611c8a4e33a58caa Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 19 Jul 2024 20:10:40 -0700 Subject: [PATCH 53/53] Experimental support for configurable prefetching (#16020) This PR adds experimental support for prefetching managed memory at a select few points in libcudf. A new configuration object is introduced for handling whether prefetching is enabled or disabled, and whether to print debug information about pointers being prefetched. Prefetching control is managed on a per API basis to enable profiling of the effects of prefetching different classes of data in different contexts. Prefetching in this PR always occurs on the default stream, so it will trigger synchronization with any blocking streams that the user has created. Turning on prefetching and then passing non-blocking to any libcudf APIs will trigger undefined behavior. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - David Wendt (https://github.com/davidwendt) - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Thomas Li (https://github.com/lithomas1) - Muhammad Haseeb (https://github.com/mhaseeb123) URL: https://github.com/rapidsai/cudf/pull/16020 --- cpp/CMakeLists.txt | 1 + cpp/include/cudf/column/column_view.hpp | 54 ++++-- cpp/include/cudf/detail/join.hpp | 3 - cpp/include/cudf/strings/detail/gather.cuh | 7 +- .../cudf/strings/detail/strings_children.cuh | 2 + cpp/include/cudf/utilities/prefetch.hpp | 155 ++++++++++++++++++ cpp/src/column/column_view.cpp | 42 +++++ cpp/src/join/hash_join.cu | 2 + cpp/src/utilities/prefetch.cpp | 89 ++++++++++ .../cudf/cudf/_lib/pylibcudf/CMakeLists.txt | 1 + python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 3 + python/cudf/cudf/_lib/pylibcudf/__init__.py | 3 + .../cudf/cudf/_lib/pylibcudf/experimental.pxd | 10 ++ .../cudf/cudf/_lib/pylibcudf/experimental.pyx | 43 +++++ .../_lib/pylibcudf/libcudf/experimental.pxd | 16 ++ 15 files changed, 416 insertions(+), 15 deletions(-) create mode 100644 cpp/include/cudf/utilities/prefetch.hpp create mode 100644 cpp/src/utilities/prefetch.cpp create mode 100644 python/cudf/cudf/_lib/pylibcudf/experimental.pxd create mode 100644 python/cudf/cudf/_lib/pylibcudf/experimental.pyx create mode 100644 python/cudf/cudf/_lib/pylibcudf/libcudf/experimental.pxd diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 65347bd6689..5e79204a558 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -674,6 +674,7 @@ add_library( src/utilities/linked_column.cpp src/utilities/logger.cpp src/utilities/pinned_memory.cpp + src/utilities/prefetch.cpp src/utilities/stacktrace.cpp src/utilities/stream_pool.cpp src/utilities/traits.cpp diff --git a/cpp/include/cudf/column/column_view.hpp b/cpp/include/cudf/column/column_view.hpp index 134e835911f..03352fdce13 100644 --- a/cpp/include/cudf/column/column_view.hpp +++ b/cpp/include/cudf/column/column_view.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,9 @@ #pragma once #include +#include #include +#include #include #include #include @@ -72,7 +74,7 @@ class column_view_base { CUDF_ENABLE_IF(std::is_same_v or is_rep_layout_compatible())> T const* head() const noexcept { - return static_cast(_data); + return static_cast(get_data()); } /** @@ -225,6 +227,17 @@ class column_view_base { [[nodiscard]] size_type offset() const noexcept { return _offset; } protected: + /** + * @brief Returns pointer to the base device memory allocation. + * + * The primary purpose of this function is to allow derived classes to + * override the fundamental properties of memory accesses without needing to + * change all of the different accessors for the underlying pointer. + * + * @return Typed pointer to underlying data + */ + virtual void const* get_data() const noexcept { return _data; } + data_type _type{type_id::EMPTY}; ///< Element type size_type _size{}; ///< Number of elements void const* _data{}; ///< Pointer to device memory containing elements @@ -236,7 +249,7 @@ class column_view_base { ///< Enables zero-copy slicing column_view_base() = default; - ~column_view_base() = default; + virtual ~column_view_base() = default; column_view_base(column_view_base const&) = default; ///< Copy constructor column_view_base(column_view_base&&) = default; ///< Move constructor /** @@ -283,11 +296,6 @@ class column_view_base { size_type null_count, size_type offset = 0); }; - -class mutable_column_view_base : public column_view_base { - public: - protected: -}; } // namespace detail /** @@ -323,7 +331,7 @@ class column_view : public detail::column_view_base { #ifdef __CUDACC__ #pragma nv_exec_check_disable #endif - ~column_view() = default; + ~column_view() override = default; #ifdef __CUDACC__ #pragma nv_exec_check_disable #endif @@ -447,6 +455,18 @@ class column_view : public detail::column_view_base { return device_span(data(), size()); } + protected: + /** + * @brief Returns pointer to the base device memory allocation. + * + * The primary purpose of this function is to allow derived classes to + * override the fundamental properties of memory accesses without needing to + * change all of the different accessors for the underlying pointer. + * + * @return Typed pointer to underlying data + */ + void const* get_data() const noexcept override; + private: friend column_view bit_cast(column_view const& input, data_type type); @@ -478,7 +498,7 @@ class mutable_column_view : public detail::column_view_base { public: mutable_column_view() = default; - ~mutable_column_view(){ + ~mutable_column_view() override{ // Needed so that the first instance of the implicit destructor for any TU isn't 'constructed' // from a host+device function marking the implicit version also as host+device }; @@ -572,7 +592,7 @@ class mutable_column_view : public detail::column_view_base { } /** - * @brief Return first element (accounting for offset) when underlying data is + * @brief Return first element (accounting for offset) after underlying data is * casted to the specified type. * * This function does not participate in overload resolution if `is_rep_layout_compatible` is @@ -665,6 +685,18 @@ class mutable_column_view : public detail::column_view_base { */ operator column_view() const; + protected: + /** + * @brief Returns pointer to the base device memory allocation. + * + * The primary purpose of this function is to allow derived classes to + * override the fundamental properties of memory accesses without needing to + * change all of the different accessors for the underlying pointer. + * + * @return Typed pointer to underlying data + */ + void const* get_data() const noexcept override; + private: friend mutable_column_view bit_cast(mutable_column_view const& input, data_type type); diff --git a/cpp/include/cudf/detail/join.hpp b/cpp/include/cudf/detail/join.hpp index aabfff746ea..b4ec5f2cc69 100644 --- a/cpp/include/cudf/detail/join.hpp +++ b/cpp/include/cudf/detail/join.hpp @@ -40,9 +40,6 @@ class preprocessed_table; namespace cudf { namespace detail { -// Forward declaration -class cuco_allocator; - constexpr int DEFAULT_JOIN_CG_SIZE = 2; enum class join_kind { INNER_JOIN, LEFT_JOIN, FULL_JOIN, LEFT_SEMI_JOIN, LEFT_ANTI_JOIN }; diff --git a/cpp/include/cudf/strings/detail/gather.cuh b/cpp/include/cudf/strings/detail/gather.cuh index fcd74bebfe8..4369de317b3 100644 --- a/cpp/include/cudf/strings/detail/gather.cuh +++ b/cpp/include/cudf/strings/detail/gather.cuh @@ -18,11 +18,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -230,7 +232,8 @@ rmm::device_uvector gather_chars(StringIterator strings_begin, if (output_count == 0) return rmm::device_uvector(0, stream, mr); auto chars_data = rmm::device_uvector(chars_bytes, stream, mr); - auto d_chars = chars_data.data(); + cudf::experimental::prefetch::detail::prefetch("gather", chars_data, stream); + auto d_chars = chars_data.data(); constexpr int warps_per_threadblock = 4; // String parallel strategy will be used if average string length is above this threshold. @@ -312,6 +315,8 @@ std::unique_ptr gather(strings_column_view const& strings, // build chars column auto const offsets_view = cudf::detail::offsetalator_factory::make_input_iterator(out_offsets_column->view()); + cudf::experimental::prefetch::detail::prefetch( + "gather", strings.chars_begin(stream), strings.chars_size(stream), stream); auto out_chars_data = gather_chars( d_strings->begin(), begin, end, offsets_view, total_bytes, stream, mr); diff --git a/cpp/include/cudf/strings/detail/strings_children.cuh b/cpp/include/cudf/strings/detail/strings_children.cuh index f5f3982a5d6..55b59dd4ff2 100644 --- a/cpp/include/cudf/strings/detail/strings_children.cuh +++ b/cpp/include/cudf/strings/detail/strings_children.cuh @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -186,6 +187,7 @@ auto make_strings_children(SizeAndExecuteFunction size_and_exec_fn, // Now build the chars column rmm::device_uvector chars(bytes, stream, mr); + cudf::experimental::prefetch::detail::prefetch("gather", chars, stream); size_and_exec_fn.d_chars = chars.data(); // Execute the function fn again to fill in the chars data. diff --git a/cpp/include/cudf/utilities/prefetch.hpp b/cpp/include/cudf/utilities/prefetch.hpp new file mode 100644 index 00000000000..5ca6fd6f4b0 --- /dev/null +++ b/cpp/include/cudf/utilities/prefetch.hpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include + +#include +#include +#include + +namespace cudf::experimental::prefetch { + +namespace detail { + +/** + * @brief A singleton class that manages the prefetching configuration. + */ +class PrefetchConfig { + public: + PrefetchConfig& operator=(const PrefetchConfig&) = delete; + PrefetchConfig(const PrefetchConfig&) = delete; + + /** + * @brief Get the singleton instance of the prefetching configuration. + * + * @return The singleton instance of the prefetching configuration. + */ + static PrefetchConfig& instance(); + + /** + * @brief Get the value of a configuration key. + * + * @param key The configuration key. + * @return The value of the configuration key. + */ + bool get(std::string_view key); + /** + * @brief Set the value of a configuration key. + * + * @param key The configuration key. + * @param value The value to set. + */ + void set(std::string_view key, bool value); + /** + * @brief Enable or disable debug mode. + * + * In debug mode, the pointers being prefetched are printed to stderr. + */ + bool debug{false}; + + private: + PrefetchConfig() = default; //< Private constructor to enforce singleton pattern + std::map config_values; //< Map of configuration keys to values +}; + +/** + * @brief Enable prefetching for a particular structure or algorithm. + * + * @param key The key to enable prefetching for. + * @param ptr The pointer to prefetch. + * @param size The size of the memory region to prefetch. + * @param stream The stream to prefetch on. + * @param device_id The device to prefetch on. + */ +void prefetch(std::string_view key, + void const* ptr, + std::size_t size, + rmm::cuda_stream_view stream, + rmm::cuda_device_id device_id = rmm::get_current_cuda_device()); + +/** + * @brief Enable prefetching for a particular structure or algorithm. + * + * @note This function will not throw exceptions, so it is safe to call in + * noexcept contexts. If an error occurs, the error code is returned. This + * function primarily exists for [mutable_]column_view::get_data and should be + * removed once an method for stream-ordered data pointer access is added to + * those data structures. + * + * @param key The key to enable prefetching for. + * @param ptr The pointer to prefetch. + * @param size The size of the memory region to prefetch. + * @param stream The stream to prefetch on. + * @param device_id The device to prefetch on. + */ +cudaError_t prefetch_noexcept( + std::string_view key, + void const* ptr, + std::size_t size, + rmm::cuda_stream_view stream, + rmm::cuda_device_id device_id = rmm::get_current_cuda_device()) noexcept; + +/** + * @brief Prefetch the data in a device_uvector. + * + * @note At present this function does not support stream-ordered execution. Prefetching always + * occurs on the default stream. + * + * @param key The key to enable prefetching for. + * @param v The device_uvector to prefetch. + * @param stream The stream to prefetch on. + * @param device_id The device to prefetch on. + */ +template +void prefetch(std::string_view key, + rmm::device_uvector const& v, + rmm::cuda_stream_view stream, + rmm::cuda_device_id device_id = rmm::get_current_cuda_device()) +{ + if (v.is_empty()) { return; } + prefetch(key, v.data(), v.size(), stream, device_id); +} + +} // namespace detail + +/** + * @brief Enable prefetching for a particular structure or algorithm. + * + * @param key The key to enable prefetching for. + */ +void enable_prefetching(std::string_view key); + +/** + * @brief Disable prefetching for a particular structure or algorithm. + * + * @param key The key to disable prefetching for. + */ +void disable_prefetching(std::string_view key); + +/** + * @brief Enable or disable debug mode. + * + * In debug mode, the pointers being prefetched are printed to stderr. + * + * @param enable Whether to enable or disable debug mode. + */ +void prefetch_debugging(bool enable); + +} // namespace cudf::experimental::prefetch diff --git a/cpp/src/column/column_view.cpp b/cpp/src/column/column_view.cpp index 4d16298c605..a9605efb362 100644 --- a/cpp/src/column/column_view.cpp +++ b/cpp/src/column/column_view.cpp @@ -15,8 +15,10 @@ */ #include +#include #include #include +#include #include #include #include @@ -27,10 +29,37 @@ #include #include #include +#include #include namespace cudf { namespace detail { +namespace { + +template +void prefetch_col_data(ColumnView& col, void const* data_ptr, std::string_view key) noexcept +{ + if (cudf::experimental::prefetch::detail::PrefetchConfig::instance().get(key)) { + if (cudf::is_fixed_width(col.type())) { + cudf::experimental::prefetch::detail::prefetch_noexcept( + key, data_ptr, col.size() * size_of(col.type()), cudf::get_default_stream()); + } else if (col.type().id() == type_id::STRING) { + strings_column_view scv{col}; + + cudf::experimental::prefetch::detail::prefetch_noexcept( + key, + data_ptr, + scv.chars_size(cudf::get_default_stream()) * sizeof(char), + cudf::get_default_stream()); + } else { + std::cout << key << ": Unsupported type: " << static_cast(col.type().id()) + << std::endl; + } + } +} + +} // namespace + column_view_base::column_view_base(data_type type, size_type size, void const* data, @@ -126,6 +155,7 @@ bool is_shallow_equivalent(column_view const& lhs, column_view const& rhs) { return shallow_equivalent_impl(lhs, rhs); } + } // namespace detail // Immutable view constructor @@ -175,6 +205,18 @@ mutable_column_view::operator column_view() const return column_view{_type, _size, _data, _null_mask, _null_count, _offset, std::move(child_views)}; } +void const* column_view::get_data() const noexcept +{ + detail::prefetch_col_data(*this, _data, "column_view::get_data"); + return _data; +} + +void const* mutable_column_view::get_data() const noexcept +{ + detail::prefetch_col_data(*this, _data, "mutable_column_view::get_data"); + return _data; +} + size_type count_descendants(column_view parent) { auto descendants = [](auto const& child) { return count_descendants(child); }; diff --git a/cpp/src/join/hash_join.cu b/cpp/src/join/hash_join.cu index b0184ff6a86..eb9b687630b 100644 --- a/cpp/src/join/hash_join.cu +++ b/cpp/src/join/hash_join.cu @@ -185,6 +185,8 @@ probe_join_hash_table( auto left_indices = std::make_unique>(join_size, stream, mr); auto right_indices = std::make_unique>(join_size, stream, mr); + cudf::experimental::prefetch::detail::prefetch("hash_join", *left_indices, stream); + cudf::experimental::prefetch::detail::prefetch("hash_join", *right_indices, stream); auto const probe_nulls = cudf::nullate::DYNAMIC{has_nulls}; diff --git a/cpp/src/utilities/prefetch.cpp b/cpp/src/utilities/prefetch.cpp new file mode 100644 index 00000000000..21f2e40c82a --- /dev/null +++ b/cpp/src/utilities/prefetch.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2020-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include + +namespace cudf::experimental::prefetch { + +namespace detail { + +PrefetchConfig& PrefetchConfig::instance() +{ + static PrefetchConfig instance; + return instance; +} + +bool PrefetchConfig::get(std::string_view key) +{ + // Default to not prefetching + if (config_values.find(key.data()) == config_values.end()) { + return (config_values[key.data()] = false); + } + return config_values[key.data()]; +} +void PrefetchConfig::set(std::string_view key, bool value) { config_values[key.data()] = value; } + +cudaError_t prefetch_noexcept(std::string_view key, + void const* ptr, + std::size_t size, + rmm::cuda_stream_view stream, + rmm::cuda_device_id device_id) noexcept +{ + if (PrefetchConfig::instance().get(key)) { + if (PrefetchConfig::instance().debug) { + std::cerr << "Prefetching " << size << " bytes for key " << key << " at location " << ptr + << std::endl; + } + auto result = cudaMemPrefetchAsync(ptr, size, device_id.value(), stream.value()); + // Need to flush the CUDA error so that the context is not corrupted. + if (result == cudaErrorInvalidValue) { cudaGetLastError(); } + return result; + } + return cudaSuccess; +} + +void prefetch(std::string_view key, + void const* ptr, + std::size_t size, + rmm::cuda_stream_view stream, + rmm::cuda_device_id device_id) +{ + auto result = prefetch_noexcept(key, ptr, size, stream, device_id); + // Ignore cudaErrorInvalidValue because that will be raised if prefetching is + // attempted on unmanaged memory. + if ((result != cudaErrorInvalidValue) && (result != cudaSuccess)) { + std::cerr << "Prefetch failed" << std::endl; + CUDF_CUDA_TRY(result); + } +} + +} // namespace detail + +void enable_prefetching(std::string_view key) { detail::PrefetchConfig::instance().set(key, true); } + +void disable_prefetching(std::string_view key) +{ + detail::PrefetchConfig::instance().set(key, false); +} + +void prefetch_debugging(bool enable) { detail::PrefetchConfig::instance().debug = enable; } +} // namespace cudf::experimental::prefetch diff --git a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt index 0800fa18e94..df4591baa71 100644 --- a/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt +++ b/python/cudf/cudf/_lib/pylibcudf/CMakeLists.txt @@ -20,6 +20,7 @@ set(cython_sources concatenate.pyx copying.pyx datetime.pyx + experimental.pyx expressions.pyx filling.pyx gpumemoryview.pyx diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 26e89b818d3..71f523fc3cd 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -8,6 +8,7 @@ from . cimport ( concatenate, copying, datetime, + experimental, expressions, filling, groupby, @@ -48,6 +49,8 @@ __all__ = [ "concatenate", "copying", "datetime", + "experimental", + "expressions", "filling", "gpumemoryview", "groupby", diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.py b/python/cudf/cudf/_lib/pylibcudf/__init__.py index e89a5ed9f96..9705eba84b1 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.py +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.py @@ -7,6 +7,7 @@ concatenate, copying, datetime, + experimental, expressions, filling, groupby, @@ -48,6 +49,8 @@ "concatenate", "copying", "datetime", + "experimental", + "expressions", "filling", "gpumemoryview", "groupby", diff --git a/python/cudf/cudf/_lib/pylibcudf/experimental.pxd b/python/cudf/cudf/_lib/pylibcudf/experimental.pxd new file mode 100644 index 00000000000..107c91c8365 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/experimental.pxd @@ -0,0 +1,10 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp cimport bool + + +cpdef enable_prefetching(str key) + +cpdef disable_prefetching(str key) + +cpdef prefetch_debugging(bool enable) diff --git a/python/cudf/cudf/_lib/pylibcudf/experimental.pyx b/python/cudf/cudf/_lib/pylibcudf/experimental.pyx new file mode 100644 index 00000000000..1e2a682d879 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/experimental.pyx @@ -0,0 +1,43 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp cimport bool +from libcpp.string cimport string + +from cudf._lib.pylibcudf.libcudf cimport experimental as cpp_experimental + + +cpdef enable_prefetching(str key): + """Turn on prefetch instructions for the given key. + + Parameters + ---------- + key : str + The key to enable prefetching for. + """ + cdef string c_key = key.encode("utf-8") + cpp_experimental.enable_prefetching(c_key) + + +cpdef disable_prefetching(str key): + """Turn off prefetch instructions for the given key. + + Parameters + ---------- + key : str + The key to disable prefetching for. + """ + cdef string c_key = key.encode("utf-8") + cpp_experimental.disable_prefetching(c_key) + + +cpdef prefetch_debugging(bool enable): + """Enable or disable prefetch debugging. + + When enabled, any prefetch instructions will be logged to the console. + + Parameters + ---------- + enable : bool + Whether to enable or disable prefetch debugging. + """ + cpp_experimental.prefetch_debugging(enable) diff --git a/python/cudf/cudf/_lib/pylibcudf/libcudf/experimental.pxd b/python/cudf/cudf/_lib/pylibcudf/libcudf/experimental.pxd new file mode 100644 index 00000000000..f280a382a04 --- /dev/null +++ b/python/cudf/cudf/_lib/pylibcudf/libcudf/experimental.pxd @@ -0,0 +1,16 @@ +# Copyright (c) 2022-2024, NVIDIA CORPORATION. + +from libcpp cimport bool +from libcpp.string cimport string + + +cdef extern from "cudf/utilities/prefetch.hpp" \ + namespace "cudf::experimental::prefetch" nogil: + # Not technically the right signature, but it's good enough to let Cython + # generate valid C++ code. It just means we'll be copying a host string + # extra, but that's OK. If we care we could generate string_view bindings, + # but there's no real rush so if we go that route we might as well + # contribute them upstream to Cython itself. + void enable_prefetching(string key) + void disable_prefetching(string key) + void prefetch_debugging(bool enable)