From 3bee6788a795ebc22125d8726fbc79dbfb50b0b5 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 14 Oct 2024 18:13:29 +0100 Subject: [PATCH 01/12] Made cudftestutil header-only and removed GTest dependency (#16839) This merge request follows up on https://github.com/rapidsai/cudf/issues/16658. It removes the dependency on GTest by cudftestutil. It satisfies the requirement that we only need API compatibility with the GTest API and we don't expose the GTest symbols to our consumers nor ship any binary artifact. The source files defining the symbols are late-binded to the resulting executable (via library INTERFACE sources). The user has to link to manually link the GTest and GMock libraries to the final executable as illustrated below. Closes #16658 ### Usage CMakeLists.txt: ```cmake add_executable(test1 test1.cpp) target_link_libraries(test1 PRIVATE GTest::gtest GTest::gmock GTest::gtest_main cudf::cudftestutil cudf::cudftestutil_impl) ``` Authors: - Basit Ayantunde (https://github.com/lamarrr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Robert Maynard (https://github.com/robertmaynard) - David Wendt (https://github.com/davidwendt) - Mike Sarahan (https://github.com/msarahan) URL: https://github.com/rapidsai/cudf/pull/16839 --- cpp/CMakeLists.txt | 65 +++++++++++------- cpp/benchmarks/CMakeLists.txt | 25 +++---- cpp/include/cudf_test/testing_main.hpp | 67 +++++++++++++------ cpp/tests/CMakeLists.txt | 11 ++- cpp/tests/io/metadata_utilities.cpp | 5 +- .../large_strings/large_strings_fixture.cpp | 9 +-- cpp/tests/utilities/table_utilities.cu | 5 +- 7 files changed, 115 insertions(+), 72 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a0ea9579475..c8f8ae2dcfe 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -863,15 +863,7 @@ if(CUDF_BUILD_TESTUTIL) add_library(cudf::cudftest_default_stream ALIAS cudftest_default_stream) - add_library( - cudftestutil SHARED - tests/io/metadata_utilities.cpp - tests/utilities/column_utilities.cu - tests/utilities/debug_utilities.cu - tests/utilities/random_seed.cpp - tests/utilities/table_utilities.cu - tests/utilities/tdigest_utilities.cu - ) + add_library(cudftestutil INTERFACE) set_target_properties( cudftestutil @@ -880,32 +872,56 @@ if(CUDF_BUILD_TESTUTIL) # set target compile options CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON - CXX_VISIBILITY_PRESET hidden CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON - CUDA_VISIBILITY_PRESET hidden - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON ) target_compile_options( - cudftestutil PUBLIC "$:${CUDF_CXX_FLAGS}>>" - "$:${CUDF_CUDA_FLAGS}>>" + cudftestutil INTERFACE "$:${CUDF_CXX_FLAGS}>>" + "$:${CUDF_CUDA_FLAGS}>>" ) target_link_libraries( - cudftestutil - PUBLIC Threads::Threads cudf cudftest_default_stream - PRIVATE GTest::gmock GTest::gtest $ + cudftestutil INTERFACE Threads::Threads cudf cudftest_default_stream + $ ) target_include_directories( - cudftestutil PUBLIC "$" - "$" + cudftestutil INTERFACE "$" + "$" ) rapids_cuda_set_runtime(cudftestutil USE_STATIC ${CUDA_STATIC_RUNTIME}) add_library(cudf::cudftestutil ALIAS cudftestutil) + add_library(cudftestutil_impl INTERFACE) + add_library(cudf::cudftestutil_impl ALIAS cudftestutil_impl) + target_sources( + cudftestutil_impl + INTERFACE $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + ) + target_link_libraries(cudftestutil_impl INTERFACE cudf::cudftestutil) + + install(FILES tests/io/metadata_utilities.cpp DESTINATION src/cudftestutil/io) + install( + FILES tests/utilities/column_utilities.cu + tests/utilities/debug_utilities.cu + tests/utilities/random_seed.cpp + tests/utilities/table_utilities.cu + tests/utilities/tdigest_utilities.cu + DESTINATION src/cudftestutil/utilities + ) + endif() # * build cudf_identify_stream_usage -------------------------------------------------------------- @@ -1006,7 +1022,7 @@ install( set(_components_export_string) if(TARGET cudftestutil) install( - TARGETS cudftest_default_stream cudftestutil + TARGETS cudftest_default_stream cudftestutil cudftestutil_impl DESTINATION ${lib_dir} EXPORT cudf-testing-exports ) @@ -1046,14 +1062,15 @@ targets: This module offers an optional testing component which defines the following IMPORTED GLOBAL targets: - cudf::cudftestutil - The main cudf testing library + cudf::cudftestutil - The main cudf testing library + cudf::cudftestutil_impl - C++ and CUDA sources to compile for definitions in cudf::cudftestutil ]=] ) rapids_export( INSTALL cudf EXPORT_SET cudf-exports ${_components_export_string} - GLOBAL_TARGETS cudf cudftestutil + GLOBAL_TARGETS cudf cudftestutil cudftestutil_impl NAMESPACE cudf:: DOCUMENTATION doc_string ) @@ -1074,7 +1091,7 @@ endif() rapids_export( BUILD cudf EXPORT_SET cudf-exports ${_components_export_string} - GLOBAL_TARGETS cudf cudftestutil + GLOBAL_TARGETS cudf cudftestutil cudftestutil_impl NAMESPACE cudf:: DOCUMENTATION doc_string FINAL_CODE_BLOCK build_code_string diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index b0f75b25975..d6fc5dc6039 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -25,7 +25,7 @@ target_compile_options( target_link_libraries( cudf_datagen PUBLIC GTest::gmock GTest::gtest benchmark::benchmark nvbench::nvbench Threads::Threads cudf - cudftestutil nvtx3::nvtx3-cpp + cudf::cudftestutil nvtx3::nvtx3-cpp PRIVATE $ ) @@ -49,7 +49,7 @@ target_compile_options( target_link_libraries( ndsh_data_generator - PUBLIC cudf cudftestutil nvtx3::nvtx3-cpp + PUBLIC cudf GTest::gmock GTest::gtest cudf::cudftestutil nvtx3::nvtx3-cpp PRIVATE $ ) @@ -65,14 +65,14 @@ target_include_directories( # Use an OBJECT library so we only compile these helper source files only once add_library( cudf_benchmark_common OBJECT - "${CUDF_SOURCE_DIR}/tests/utilities/random_seed.cpp" - synchronization/synchronization.cpp - io/cuio_common.cpp - common/table_utilities.cpp - common/benchmark_utilities.cpp - common/nvbench_utilities.cpp + synchronization/synchronization.cpp io/cuio_common.cpp common/table_utilities.cpp + common/benchmark_utilities.cpp common/nvbench_utilities.cpp ) -target_link_libraries(cudf_benchmark_common PRIVATE cudf_datagen $) +target_link_libraries( + cudf_benchmark_common PRIVATE cudf_datagen $ GTest::gmock + GTest::gtest +) + add_custom_command( OUTPUT CUDF_BENCHMARKS COMMAND echo Running benchmarks @@ -99,7 +99,7 @@ function(ConfigureBench CMAKE_BENCH_NAME) ) target_link_libraries( ${CMAKE_BENCH_NAME} PRIVATE cudf_benchmark_common cudf_datagen benchmark::benchmark_main - $ + cudf::cudftestutil_impl $ ) add_custom_command( OUTPUT CUDF_BENCHMARKS @@ -127,8 +127,9 @@ function(ConfigureNVBench CMAKE_BENCH_NAME) INSTALL_RPATH "\$ORIGIN/../../../lib" ) target_link_libraries( - ${CMAKE_BENCH_NAME} PRIVATE cudf_benchmark_common ndsh_data_generator cudf_datagen - nvbench::nvbench $ + ${CMAKE_BENCH_NAME} + PRIVATE cudf_benchmark_common ndsh_data_generator cudf_datagen nvbench::nvbench + $ cudf::cudftestutil_impl ) install( TARGETS ${CMAKE_BENCH_NAME} diff --git a/cpp/include/cudf_test/testing_main.hpp b/cpp/include/cudf_test/testing_main.hpp index 272c91133f8..2bd08f410e0 100644 --- a/cpp/include/cudf_test/testing_main.hpp +++ b/cpp/include/cudf_test/testing_main.hpp @@ -16,6 +16,7 @@ #pragma once +#include #include #include @@ -36,6 +37,12 @@ namespace CUDF_EXPORT cudf { namespace test { +struct config { + std::string rmm_mode; + std::string stream_mode; + std::string stream_error_mode; +}; + /// MR factory functions inline auto make_cuda() { return std::make_shared(); } @@ -157,10 +164,9 @@ inline auto parse_cudf_test_opts(int argc, char** argv) * @param cmd_opts Command line options returned by parse_cudf_test_opts * @return Memory resource adaptor */ -inline auto make_memory_resource_adaptor(cxxopts::ParseResult const& cmd_opts) +inline auto make_memory_resource_adaptor(cudf::test::config const& config) { - auto const rmm_mode = cmd_opts["rmm_mode"].as(); - auto resource = cudf::test::create_memory_resource(rmm_mode); + auto resource = cudf::test::create_memory_resource(config.rmm_mode); cudf::set_current_device_resource(resource.get()); return resource; } @@ -176,37 +182,54 @@ inline auto make_memory_resource_adaptor(cxxopts::ParseResult const& cmd_opts) * @param cmd_opts Command line options returned by parse_cudf_test_opts * @return Memory resource adaptor */ -inline auto make_stream_mode_adaptor(cxxopts::ParseResult const& cmd_opts) +inline auto make_stream_mode_adaptor(cudf::test::config const& config) { auto resource = cudf::get_current_device_resource_ref(); - auto const stream_mode = cmd_opts["stream_mode"].as(); - auto const stream_error_mode = cmd_opts["stream_error_mode"].as(); - auto const error_on_invalid_stream = (stream_error_mode == "error"); - auto const check_default_stream = (stream_mode == "new_cudf_default"); + auto const error_on_invalid_stream = (config.stream_error_mode == "error"); + auto const check_default_stream = (config.stream_mode == "new_cudf_default"); auto adaptor = cudf::test::stream_checking_resource_adaptor( resource, error_on_invalid_stream, check_default_stream); - if ((stream_mode == "new_cudf_default") || (stream_mode == "new_testing_default")) { + if ((config.stream_mode == "new_cudf_default") || (config.stream_mode == "new_testing_default")) { cudf::set_current_device_resource(&adaptor); } return adaptor; } +/** + * @brief Should be called in every test program that uses rmm allocators since it maintains the + * lifespan of the rmm default memory resource. this function parses the command line to customize + * test behavior, like the allocation mode used for creating the default memory resource. + * + */ +inline void init_cudf_test(int argc, char** argv, cudf::test::config const& config_override = {}) +{ + // static lifetime to keep rmm resource alive till tests end + auto const cmd_opts = parse_cudf_test_opts(argc, argv); + cudf::test::config config = config_override; + if (config.rmm_mode.empty()) { config.rmm_mode = cmd_opts["rmm_mode"].as(); } + + if (config.stream_mode.empty()) { + config.stream_mode = cmd_opts["stream_mode"].as(); + } + + if (config.stream_error_mode.empty()) { + config.stream_error_mode = cmd_opts["stream_error_mode"].as(); + } + + [[maybe_unused]] static auto mr = make_memory_resource_adaptor(config); + [[maybe_unused]] static auto adaptor = make_stream_mode_adaptor(config); +} + /** * @brief Macro that defines main function for gtest programs that use rmm * - * Should be included in every test program that uses rmm allocators since - * it maintains the lifespan of the rmm default memory resource. * This `main` function is a wrapper around the google test generated `main`, - * maintaining the original functionality. In addition, this custom `main` - * function parses the command line to customize test behavior, like the - * allocation mode used for creating the default memory resource. + * maintaining the original functionality. */ -#define CUDF_TEST_PROGRAM_MAIN() \ - int main(int argc, char** argv) \ - { \ - ::testing::InitGoogleTest(&argc, argv); \ - auto const cmd_opts = parse_cudf_test_opts(argc, argv); \ - [[maybe_unused]] auto mr = make_memory_resource_adaptor(cmd_opts); \ - [[maybe_unused]] auto adaptor = make_stream_mode_adaptor(cmd_opts); \ - return RUN_ALL_TESTS(); \ +#define CUDF_TEST_PROGRAM_MAIN() \ + int main(int argc, char** argv) \ + { \ + ::testing::InitGoogleTest(&argc, argv); \ + init_cudf_test(argc, argv); \ + return RUN_ALL_TESTS(); \ } diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 4596ec65ce7..62189f76cae 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -56,8 +56,15 @@ function(ConfigureTest CMAKE_TEST_NAME) target_link_libraries( ${CMAKE_TEST_NAME} - PRIVATE cudftestutil GTest::gmock GTest::gmock_main GTest::gtest GTest::gtest_main - nvtx3::nvtx3-cpp $ "${_CUDF_TEST_EXTRA_LIBS}" + PRIVATE cudf::cudftestutil + cudf::cudftestutil_impl + GTest::gmock + GTest::gmock_main + GTest::gtest + GTest::gtest_main + nvtx3::nvtx3-cpp + $ + "${_CUDF_TEST_EXTRA_LIBS}" ) rapids_cuda_set_runtime(${CMAKE_TEST_NAME} USE_STATIC ${CUDA_STATIC_RUNTIME}) rapids_test_add( diff --git a/cpp/tests/io/metadata_utilities.cpp b/cpp/tests/io/metadata_utilities.cpp index 84f04f67038..380d66c53f9 100644 --- a/cpp/tests/io/metadata_utilities.cpp +++ b/cpp/tests/io/metadata_utilities.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,9 @@ * limitations under the License. */ +#include #include -#include - namespace cudf::test { void expect_metadata_equal(cudf::io::table_input_metadata in_meta, diff --git a/cpp/tests/large_strings/large_strings_fixture.cpp b/cpp/tests/large_strings/large_strings_fixture.cpp index 249319da7f7..7b61be113f9 100644 --- a/cpp/tests/large_strings/large_strings_fixture.cpp +++ b/cpp/tests/large_strings/large_strings_fixture.cpp @@ -123,12 +123,9 @@ LargeStringsData* StringsLargeTest::g_ls_data = nullptr; int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - auto const cmd_opts = parse_cudf_test_opts(argc, argv); - // hardcoding the CUDA memory resource to keep from exceeding the pool - auto mr = cudf::test::make_cuda(); - cudf::set_current_device_resource(mr.get()); - auto adaptor = make_stream_mode_adaptor(cmd_opts); - + cudf::test::config config; + config.rmm_mode = "cuda"; + init_cudf_test(argc, argv, config); // create object to automatically be destroyed at the end of main() auto lsd = cudf::test::StringsLargeTest::get_ls_data(); diff --git a/cpp/tests/utilities/table_utilities.cu b/cpp/tests/utilities/table_utilities.cu index 354c0b1b57e..8e4906408de 100644 --- a/cpp/tests/utilities/table_utilities.cu +++ b/cpp/tests/utilities/table_utilities.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,9 @@ */ #include +#include #include -#include - namespace cudf::test::detail { void expect_table_properties_equal(cudf::table_view lhs, cudf::table_view rhs) { From e41dea933f044183ccbfe26875a2b6c3ff383814 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 14 Oct 2024 10:35:37 -0700 Subject: [PATCH 02/12] Add profilers to CUDA 12 conda devcontainers (#17066) This will make sure that profilers are available by default for everyone using our devcontainers. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/17066 --- .../cuda12.5-conda/devcontainer.json | 22 +++++++++++++++++++ ci/release/update-version.sh | 1 + 2 files changed, 23 insertions(+) diff --git a/.devcontainer/cuda12.5-conda/devcontainer.json b/.devcontainer/cuda12.5-conda/devcontainer.json index 2a195c6c81d..a0e193ff0bf 100644 --- a/.devcontainer/cuda12.5-conda/devcontainer.json +++ b/.devcontainer/cuda12.5-conda/devcontainer.json @@ -15,9 +15,31 @@ ], "hostRequirements": {"gpu": "optional"}, "features": { + "ghcr.io/rapidsai/devcontainers/features/cuda:24.12": { + "version": "12.5", + "installCompilers": false, + "installProfilers": true, + "installDevPackages": false, + "installcuDNN": false, + "installcuTensor": false, + "installNCCL": false, + "installCUDARuntime": false, + "installNVRTC": false, + "installOpenCL": false, + "installcuBLAS": false, + "installcuSPARSE": false, + "installcuFFT": false, + "installcuFile": false, + "installcuRAND": false, + "installcuSOLVER": false, + "installNPP": false, + "installnvJPEG": false, + "pruneStaticLibs": true + }, "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.12": {} }, "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/cuda", "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.5-envs}"], diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 870901d223b..95f36653c2c 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -93,6 +93,7 @@ sed_runner "s/cudf-.*-SNAPSHOT/cudf-${NEXT_FULL_JAVA_TAG}/g" java/ci/README.md # .devcontainer files find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r -d '' filename; do sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}" + sed_runner "s@rapidsai/devcontainers/features/cuda:[0-9.]*@rapidsai/devcontainers/features/cuda:${NEXT_SHORT_TAG_PEP440}@" "${filename}" sed_runner "s@rapidsai/devcontainers/features/rapids-build-utils:[0-9.]*@rapidsai/devcontainers/features/rapids-build-utils:${NEXT_SHORT_TAG_PEP440}@" "${filename}" sed_runner "s@rapids-\${localWorkspaceFolderBasename}-[0-9.]*@rapids-\${localWorkspaceFolderBasename}-${NEXT_SHORT_TAG}@g" "${filename}" done From 768fbaa28033446dc899872b3c94213c75bc1a98 Mon Sep 17 00:00:00 2001 From: Nghia Truong <7416935+ttnghia@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:53:28 -0700 Subject: [PATCH 03/12] Fix ORC reader when using `device_read_async` while the destination device buffers are not ready (#17074) This fixes a bug in ORC reader when `device_read_async` is called while the destination device buffers are not ready to write in. In detail, this bug is because `device_read_async` does not use the user-provided stream but its own generated stream for data copying. As such, the copying ops could happen before the destination device buffers are being allocated, causing data corruption. This bug only shows up in certain conditions, and also hard to reproduce. It occurs when copying buffers with small sizes (below `gds_threshold`) and most likely to show up with setting `rmm_mode=async`. Authors: - Nghia Truong (https://github.com/ttnghia) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/17074 --- cpp/src/io/orc/reader_impl_chunking.cu | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/src/io/orc/reader_impl_chunking.cu b/cpp/src/io/orc/reader_impl_chunking.cu index 01ee5ad177d..ecf319e75ab 100644 --- a/cpp/src/io/orc/reader_impl_chunking.cu +++ b/cpp/src/io/orc/reader_impl_chunking.cu @@ -500,6 +500,8 @@ void reader_impl::load_next_stripe_data(read_mode mode) auto const [read_begin, read_end] = merge_selected_ranges(_file_itm_data.stripe_data_read_ranges, load_stripe_range); + bool stream_synchronized{false}; + for (auto read_idx = read_begin; read_idx < read_end; ++read_idx) { auto const& read_info = _file_itm_data.data_read_info[read_idx]; auto const source_ptr = _metadata.per_file_metadata[read_info.source_idx].source; @@ -507,6 +509,13 @@ void reader_impl::load_next_stripe_data(read_mode mode) lvl_stripe_data[read_info.level][read_info.stripe_idx - stripe_start].data()); if (source_ptr->is_device_read_preferred(read_info.length)) { + // `device_read_async` may not use _stream at all. + // Instead, it may use some other stream(s) to sync the H->D memcpy. + // As such, we need to make sure the device buffers in `lvl_stripe_data` are ready first. + if (!stream_synchronized) { + _stream.synchronize(); + stream_synchronized = true; + } device_read_tasks.push_back( std::pair(source_ptr->device_read_async( read_info.offset, read_info.length, dst_base + read_info.dst_pos, _stream), From 44afc5109b342b653797a20db1e2654fa450417f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 14 Oct 2024 16:02:14 -0700 Subject: [PATCH 04/12] Add clang-tidy to CI (#16958) This PR adds clang-tidy checks to our CI. clang-tidy will be run in nightly CI via CMake. For now, only the parts of the code base that were already made compliant in the PRs leading up to this have been enabled, namely cudf source and test cpp files. Over time we can add more files like benchmarks and examples, add or subtract more rules, or enable linting of cu files (see https://gitlab.kitware.com/cmake/cmake/-/issues/25399). This PR is intended to be the starting point enabling systematic linting, at which point everything else should be significantly easier. Resolves #584 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/16958 --- .github/workflows/test.yaml | 14 +- ci/clang_tidy.sh | 29 +++ cpp/.clang-tidy | 2 +- cpp/CMakeLists.txt | 54 ++++++ cpp/scripts/run-clang-tidy.py | 253 -------------------------- cpp/tests/CMakeLists.txt | 1 + cpp/tests/interop/nanoarrow_utils.hpp | 3 +- dependencies.yaml | 30 ++- 8 files changed, 129 insertions(+), 257 deletions(-) create mode 100755 ci/clang_tidy.sh delete mode 100644 cpp/scripts/run-clang-tidy.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a22d3c5b9cc..1275aad757c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,11 +47,23 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 with: - build_type: pull-request + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} # Use the wheel container so we can skip conda solves and since our # primary static consumers (Spark) are not in conda anyway. container_image: "rapidsai/ci-wheel:latest" run_script: "ci/configure_cpp_static.sh" + clang-tidy: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + run_script: "ci/clang_tidy.sh" conda-python-cudf-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 diff --git a/ci/clang_tidy.sh b/ci/clang_tidy.sh new file mode 100755 index 00000000000..4d5d3fc3136 --- /dev/null +++ b/ci/clang_tidy.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-logger "Create clang-tidy conda environment" +. /opt/conda/etc/profile.d/conda.sh + +ENV_YAML_DIR="$(mktemp -d)" + +rapids-dependency-file-generator \ + --output conda \ + --file-key clang_tidy \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n clang_tidy + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate clang_tidy +set -u + +RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" + +source rapids-configure-sccache + +# Run the build via CMake, which will run clang-tidy when CUDF_CLANG_TIDY is enabled. +cmake -S cpp -B cpp/build -DCMAKE_BUILD_TYPE=Release -DCUDF_CLANG_TIDY=ON -GNinja +cmake --build cpp/build diff --git a/cpp/.clang-tidy b/cpp/.clang-tidy index 2d4f8c0d80e..12120a5c6d1 100644 --- a/cpp/.clang-tidy +++ b/cpp/.clang-tidy @@ -39,7 +39,7 @@ Checks: -clang-analyzer-optin.core.EnumCastOutOfRange, -clang-analyzer-optin.cplusplus.UninitializedObject' -WarningsAsErrors: '' +WarningsAsErrors: '*' HeaderFilterRegex: '.*cudf/cpp/(src|include|tests).*' ExcludeHeaderFilterRegex: '.*(Message_generated.h|Schema_generated.h|brotli_dict.hpp|unbz2.hpp|cxxopts.hpp).*' FormatStyle: none diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c8f8ae2dcfe..32a753c9f40 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -88,6 +88,7 @@ option( ${DEFAULT_CUDF_BUILD_STREAMS_TEST_UTIL} ) mark_as_advanced(CUDF_BUILD_STREAMS_TEST_UTIL) +option(CUDF_CLANG_TIDY "Enable clang-tidy checking" OFF) message(VERBOSE "CUDF: Build with NVTX support: ${USE_NVTX}") message(VERBOSE "CUDF: Configure CMake to build tests: ${BUILD_TESTS}") @@ -144,6 +145,58 @@ if(NOT CUDF_GENERATED_INCLUDE_DIR) set(CUDF_GENERATED_INCLUDE_DIR ${CUDF_BINARY_DIR}) endif() +# ################################################################################################## +# * clang-tidy configuration ---------------------------------------------------------------------- +if(CUDF_CLANG_TIDY) + find_program( + CLANG_TIDY_EXE + NAMES "clang-tidy" + DOC "Path to clang-tidy executable" REQUIRED + ) + + execute_process( + COMMAND ${CLANG_TIDY_EXE} --version + OUTPUT_VARIABLE CLANG_TIDY_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + string(REGEX MATCH "LLVM version ([0-9]+\\.[0-9]+)\\.[0-9]+" LLVM_VERSION_MATCH + "${CLANG_TIDY_OUTPUT}" + ) + # Discard the patch version and allow it to float. Empirically, results between patch versions are + # mostly stable, and looking at available packages on some package managers sometimes patch + # versions are skipped so we don't want to constrain to a patch version that the user can't + # install. + set(LLVM_VERSION "${CMAKE_MATCH_1}") + set(expected_clang_tidy_version 19.1) + if(NOT expected_clang_tidy_version VERSION_EQUAL LLVM_VERSION) + message( + FATAL_ERROR + "clang-tidy version ${expected_clang_tidy_version} is required, but found ${LLVM_VERSION}" + ) + endif() +endif() + +# Turn on the clang-tidy property for a target excluding the files specified in SKIPPED_FILES. +function(enable_clang_tidy target) + set(_tidy_options) + set(_tidy_one_value) + set(_tidy_multi_value SKIPPED_FILES) + cmake_parse_arguments( + _TIDY "${_tidy_options}" "${_tidy_one_value}" "${_tidy_multi_value}" ${ARGN} + ) + + if(CUDF_CLANG_TIDY) + # clang will complain about unused link libraries on the compile line unless we specify + # -Qunused-arguments. + set_target_properties( + ${target} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};--extra-arg=-Qunused-arguments" + ) + foreach(file IN LISTS _TIDY_SKIPPED_FILES) + set_source_files_properties(${file} PROPERTIES SKIP_LINTING ON) + endforeach() + endif() +endfunction() + # ################################################################################################## # * conda environment ----------------------------------------------------------------------------- rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) @@ -714,6 +767,7 @@ target_compile_options( cudf PRIVATE "$<$:${CUDF_CXX_FLAGS}>" "$<$:${CUDF_CUDA_FLAGS}>" ) +enable_clang_tidy(cudf SKIPPED_FILES src/io/comp/cpu_unbz2.cpp src/io/comp/brotli_dict.cpp) if(CUDF_BUILD_STACKTRACE_DEBUG) # Remove any optimization level to avoid nvcc warning "incompatible redefinition for option diff --git a/cpp/scripts/run-clang-tidy.py b/cpp/scripts/run-clang-tidy.py deleted file mode 100644 index e5e57dbf562..00000000000 --- a/cpp/scripts/run-clang-tidy.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright (c) 2021-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import re -import os -import subprocess -import argparse -import json -import multiprocessing as mp -import shutil - - -EXPECTED_VERSION = "16.0.6" -VERSION_REGEX = re.compile(r" LLVM version ([0-9.]+)") -GPU_ARCH_REGEX = re.compile(r"sm_(\d+)") -SPACES = re.compile(r"\s+") -SEPARATOR = "-" * 16 - - -def parse_args(): - argparser = argparse.ArgumentParser("Runs clang-tidy on a project") - argparser.add_argument("-cdb", type=str, - # TODO This is a hack, needs to be fixed - default="cpp/build/cuda-11.5.0/clang-tidy/release/compile_commands.clangd.json", - help="Path to cmake-generated compilation database" - " file. It is always found inside the root of the " - "cmake build folder. So make sure that `cmake` has " - "been run once before running this script!") - argparser.add_argument("-exe", type=str, default="clang-tidy", - help="Path to clang-tidy exe") - argparser.add_argument("-ignore", type=str, default="[.]cu$|examples/kmeans/", - help="Regex used to ignore files from checking") - argparser.add_argument("-select", type=str, default=None, - help="Regex used to select files for checking") - argparser.add_argument("-j", type=int, default=-1, - help="Number of parallel jobs to launch.") - args = argparser.parse_args() - if args.j <= 0: - args.j = mp.cpu_count() - args.ignore_compiled = re.compile(args.ignore) if args.ignore else None - args.select_compiled = re.compile(args.select) if args.select else None - ret = subprocess.check_output("%s --version" % args.exe, shell=True) - ret = ret.decode("utf-8") - version = VERSION_REGEX.search(ret) - if version is None: - raise Exception("Failed to figure out clang-tidy version!") - version = version.group(1) - if version != EXPECTED_VERSION: - raise Exception("clang-tidy exe must be v%s found '%s'" % \ - (EXPECTED_VERSION, version)) - if not os.path.exists(args.cdb): - raise Exception("Compilation database '%s' missing" % args.cdb) - return args - - -def get_all_commands(cdb): - with open(cdb) as fp: - return json.load(fp) - - -def get_gpu_archs(command): - archs = [] - for loc in range(len(command)): - if command[loc] != "-gencode": - continue - arch_flag = command[loc + 1] - match = GPU_ARCH_REGEX.search(arch_flag) - if match is not None: - archs.append("--cuda-gpu-arch=sm_%s" % match.group(1)) - return archs - - -def get_index(arr, item): - try: - return arr.index(item) - except: - return -1 - - -def remove_item(arr, item): - loc = get_index(arr, item) - if loc >= 0: - del arr[loc] - return loc - - -def remove_item_plus_one(arr, item): - loc = get_index(arr, item) - if loc >= 0: - del arr[loc + 1] - del arr[loc] - return loc - - -def get_clang_includes(exe): - dir = os.getenv("CONDA_PREFIX") - if dir is None: - ret = subprocess.check_output("which %s 2>&1" % exe, shell=True) - ret = ret.decode("utf-8") - dir = os.path.dirname(os.path.dirname(ret)) - header = os.path.join(dir, "include", "ClangHeaders") - return ["-I", header] - - -def get_tidy_args(cmd, exe): - command, file = cmd["command"], cmd["file"] - is_cuda = file.endswith(".cu") - command = re.split(SPACES, command) - # compiler is always clang++! - command[0] = "clang++" - # remove compilation and output targets from the original command - remove_item_plus_one(command, "-c") - remove_item_plus_one(command, "-o") - if is_cuda: - # replace nvcc's "-gencode ..." with clang's "--cuda-gpu-arch ..." - archs = get_gpu_archs(command) - command.extend(archs) - while True: - loc = remove_item_plus_one(command, "-gencode") - if loc < 0: - break - # "-x cuda" is the right usage in clang - loc = get_index(command, "-x") - if loc >= 0: - command[loc + 1] = "cuda" - remove_item_plus_one(command, "-ccbin") - remove_item(command, "--expt-extended-lambda") - remove_item(command, "--diag_suppress=unrecognized_gcc_pragma") - command.extend(get_clang_includes(exe)) - return command, is_cuda - - -def run_clang_tidy_command(tidy_cmd): - cmd = " ".join(tidy_cmd) - result = subprocess.run(cmd, check=False, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - status = result.returncode == 0 - if status: - out = "" - else: - out = "CMD: " + cmd - out += result.stdout.decode("utf-8").rstrip() - return status, out - - -def run_clang_tidy(cmd, args): - command, is_cuda = get_tidy_args(cmd, args.exe) - tidy_cmd = [args.exe, - "-header-filter='.*cudf/cpp/(src|include|bench|comms).*'", - cmd["file"], "--", ] - tidy_cmd.extend(command) - status = True - out = "" - if is_cuda: - tidy_cmd.append("--cuda-device-only") - tidy_cmd.append(cmd["file"]) - ret, out1 = run_clang_tidy_command(tidy_cmd) - out += out1 - out += "%s" % SEPARATOR - if not ret: - status = ret - tidy_cmd[-2] = "--cuda-host-only" - ret, out1 = run_clang_tidy_command(tidy_cmd) - if not ret: - status = ret - out += out1 - else: - tidy_cmd.append(cmd["file"]) - ret, out1 = run_clang_tidy_command(tidy_cmd) - if not ret: - status = ret - out += out1 - return status, out, cmd["file"] - - -# yikes! global var :( -results = [] -def collect_result(result): - global results - results.append(result) - - -def print_result(passed, stdout, file): - status_str = "PASSED" if passed else "FAILED" - print(f"{SEPARATOR} File:{file} {status_str} {SEPARATOR}") - if stdout: - print(stdout) - print(f"{SEPARATOR} File:{file} ENDS {SEPARATOR}") - - -def print_results(): - global results - status = True - for passed, stdout, file in results: - print_result(passed, stdout, file) - if not passed: - status = False - return status - - -def run_tidy_for_all_files(args, all_files): - pool = None if args.j == 1 else mp.Pool(args.j) - # actual tidy checker - for cmd in all_files: - # skip files that we don't want to look at - if args.ignore_compiled is not None and \ - re.search(args.ignore_compiled, cmd["file"]) is not None: - continue - if args.select_compiled is not None and \ - re.search(args.select_compiled, cmd["file"]) is None: - continue - if pool is not None: - pool.apply_async(run_clang_tidy, args=(cmd, args), - callback=collect_result) - else: - passed, stdout, file = run_clang_tidy(cmd, args) - collect_result((passed, stdout, file)) - if pool is not None: - pool.close() - pool.join() - return print_results() - - -def main(): - args = parse_args() - # Attempt to making sure that we run this script from root of repo always - if not os.path.exists(".git"): - raise Exception("This needs to always be run from the root of repo") - # Check whether clang-tidy exists - # print(args) - if "exe" not in args and shutil.which("clang-tidy") is not None: - print("clang-tidy not found. Exiting...") - return - all_files = get_all_commands(args.cdb) - status = run_tidy_for_all_files(args, all_files) - if not status: - raise Exception("clang-tidy failed! Refer to the errors above.") - - -if __name__ == "__main__": - main() diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 62189f76cae..799a84cbc37 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -83,6 +83,7 @@ function(ConfigureTest CMAKE_TEST_NAME) "GTEST_CUDF_STREAM_MODE=new_${_CUDF_TEST_STREAM_MODE}_default;LD_PRELOAD=$" ) endif() + enable_clang_tidy(${CMAKE_TEST_NAME}) endfunction() # ################################################################################################## diff --git a/cpp/tests/interop/nanoarrow_utils.hpp b/cpp/tests/interop/nanoarrow_utils.hpp index a961f73d955..8be7e087b6d 100644 --- a/cpp/tests/interop/nanoarrow_utils.hpp +++ b/cpp/tests/interop/nanoarrow_utils.hpp @@ -256,7 +256,8 @@ std::enable_if_t, nanoarrow::UniqueArray> get_nanoarrow_ ArrowBitmap out; ArrowBitmapInit(&out); NANOARROW_THROW_NOT_OK(ArrowBitmapResize(&out, b.size(), 1)); - std::memset(out.buffer.data, 0, out.buffer.size_bytes); + // TODO: Investigate clang-tidy issue further after nanoarrow is made compliant + std::memset(out.buffer.data, 0, out.buffer.size_bytes); // NOLINT for (size_t i = 0; i < b.size(); ++i) { ArrowBitSetTo(out.buffer.data, i, static_cast(b[i])); diff --git a/dependencies.yaml b/dependencies.yaml index ca17917c905..ff97b67f0ce 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,10 +6,18 @@ files: cuda: ["11.8", "12.5"] arch: [x86_64] includes: + # Note that clang-tidy is not included here because cudf's preferred + # version conflicts with the rest of RAPIDS as well as its own + # clang-format version. Until we update our clang-format version we will + # not be able to install both into the same environment. Moreover, using + # this version will break compatibility with other RAPIDS libraries that + # are still using 16.0.6, and as such will and that would break any + # unified environment like that used in unified devcontainers. - build_base - build_all - build_cpp - build_python_common + - clang_format - cuda - cuda_version - depends_on_cupy @@ -86,6 +94,16 @@ files: includes: - develop - py_version + clang_tidy: + output: none + includes: + - build_all + - build_base + - clang_tidy + - cuda + - cuda_version + - develop + - py_version docs: output: none includes: @@ -553,11 +571,21 @@ dependencies: # pre-commit requires identify minimum version 1.0, but clang-format requires textproto support and that was # added in 2.5.20, so we need to call out the minimum version needed for our plugins - identify>=2.5.20 + - output_types: conda + packages: + - &doxygen doxygen=1.9.1 # pre-commit hook needs a specific version. + clang_format: + common: - output_types: conda packages: - clang==16.0.6 - clang-tools=16.0.6 - - &doxygen doxygen=1.9.1 # pre-commit hook needs a specific version. + clang_tidy: + common: + - output_types: conda + packages: + - clang==19.1.0 + - clang-tools==19.1.0 docs: common: - output_types: [conda] From 86db9804746fb20554c1900b311a228dc1d6e349 Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Mon, 14 Oct 2024 16:30:22 -0700 Subject: [PATCH 05/12] Clean up hash-groupby `var_hash_functor` (#17034) This work is part of splitting the original bulk shared memory groupby PR #16619. This PR renames the file originally titled `multi_pass_kernels.cuh`, which contains the `var_hash_functor`, to `var_hash_functor.cuh`. It also includes cleanups such as utilizing `cuda::std::` utilities in device code and removing redundant template parameters. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/17034 --- cpp/src/groupby/hash/groupby.cu | 5 +- cpp/src/groupby/hash/groupby_kernels.cuh | 2 - ..._pass_kernels.cuh => var_hash_functor.cuh} | 51 ++++++++----------- 3 files changed, 25 insertions(+), 33 deletions(-) rename cpp/src/groupby/hash/{multi_pass_kernels.cuh => var_hash_functor.cuh} (69%) diff --git a/cpp/src/groupby/hash/groupby.cu b/cpp/src/groupby/hash/groupby.cu index 75767786272..0432b9d120a 100644 --- a/cpp/src/groupby/hash/groupby.cu +++ b/cpp/src/groupby/hash/groupby.cu @@ -16,7 +16,8 @@ #include "flatten_single_pass_aggs.hpp" #include "groupby/common/utils.hpp" -#include "groupby/hash/groupby_kernels.cuh" +#include "groupby_kernels.cuh" +#include "var_hash_functor.cuh" #include #include @@ -261,7 +262,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final rmm::exec_policy(stream), thrust::make_counting_iterator(0), col.size(), - ::cudf::detail::var_hash_functor{ + var_hash_functor{ set, row_bitmask, *var_result_view, *values_view, *sum_view, *count_view, agg._ddof}); sparse_results->add_result(col, agg, std::move(var_result)); dense_results->add_result(col, agg, to_dense_agg_result(agg)); diff --git a/cpp/src/groupby/hash/groupby_kernels.cuh b/cpp/src/groupby/hash/groupby_kernels.cuh index 188d0cff3f1..86f4d76487f 100644 --- a/cpp/src/groupby/hash/groupby_kernels.cuh +++ b/cpp/src/groupby/hash/groupby_kernels.cuh @@ -16,8 +16,6 @@ #pragma once -#include "multi_pass_kernels.cuh" - #include #include #include diff --git a/cpp/src/groupby/hash/multi_pass_kernels.cuh b/cpp/src/groupby/hash/var_hash_functor.cuh similarity index 69% rename from cpp/src/groupby/hash/multi_pass_kernels.cuh rename to cpp/src/groupby/hash/var_hash_functor.cuh index 7043eafdc10..bb55cc9188c 100644 --- a/cpp/src/groupby/hash/multi_pass_kernels.cuh +++ b/cpp/src/groupby/hash/var_hash_functor.cuh @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #pragma once #include @@ -21,17 +20,14 @@ #include #include #include -#include #include +#include #include +#include -#include - -namespace cudf { -namespace detail { - -template +namespace cudf::groupby::detail::hash { +template struct var_hash_functor { SetType set; bitmask_type const* __restrict__ row_bitmask; @@ -47,13 +43,13 @@ struct var_hash_functor { column_device_view sum, column_device_view count, size_type ddof) - : set(set), - row_bitmask(row_bitmask), - target(target), - source(source), - sum(sum), - count(count), - ddof(ddof) + : set{set}, + row_bitmask{row_bitmask}, + target{target}, + source{source}, + sum{sum}, + count{count}, + ddof{ddof} { } @@ -64,23 +60,21 @@ struct var_hash_functor { } template - __device__ std::enable_if_t()> operator()(column_device_view const& source, - size_type source_index, - size_type target_index) noexcept + __device__ cuda::std::enable_if_t()> operator()( + column_device_view const& source, size_type source_index, size_type target_index) noexcept { CUDF_UNREACHABLE("Invalid source type for std, var aggregation combination."); } template - __device__ std::enable_if_t()> operator()(column_device_view const& source, - size_type source_index, - size_type target_index) noexcept + __device__ cuda::std::enable_if_t()> operator()( + column_device_view const& source, size_type source_index, size_type target_index) noexcept { - using Target = target_type_t; - using SumType = target_type_t; - using CountType = target_type_t; + using Target = cudf::detail::target_type_t; + using SumType = cudf::detail::target_type_t; + using CountType = cudf::detail::target_type_t; - if (source_has_nulls and source.is_null(source_index)) return; + if (source.is_null(source_index)) return; CountType group_size = count.element(target_index); if (group_size == 0 or group_size - ddof <= 0) return; @@ -91,8 +85,9 @@ struct var_hash_functor { ref.fetch_add(result, cuda::std::memory_order_relaxed); // STD sqrt is applied in finalize() - if (target_has_nulls and target.is_null(target_index)) { target.set_valid(target_index); } + if (target.is_null(target_index)) { target.set_valid(target_index); } } + __device__ inline void operator()(size_type source_index) { if (row_bitmask == nullptr or cudf::bit_is_set(row_bitmask, source_index)) { @@ -110,6 +105,4 @@ struct var_hash_functor { } } }; - -} // namespace detail -} // namespace cudf +} // namespace cudf::groupby::detail::hash From 319ec3b8031e4deb7dfc3f4c4a07a10ef88c131f Mon Sep 17 00:00:00 2001 From: Shruti Shivakumar Date: Mon, 14 Oct 2024 19:58:30 -0400 Subject: [PATCH 06/12] Adding assertion to check for regular JSON inputs of size greater than `INT_MAX` bytes (#17057) Addresses #17017 Libcudf does not support parsing regular JSON inputs of size greater than `INT_MAX` bytes. Note that the batched reader can only be used for JSON lines inputs. Authors: - Shruti Shivakumar (https://github.com/shrshi) Approvers: - Muhammad Haseeb (https://github.com/mhaseeb123) - Vukasin Milovanovic (https://github.com/vuule) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/17057 --- cpp/src/io/json/nested_json_gpu.cu | 3 +-- cpp/src/io/json/read_json.cu | 14 ++++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cpp/src/io/json/nested_json_gpu.cu b/cpp/src/io/json/nested_json_gpu.cu index 76816071d8c..69a51fab5dc 100644 --- a/cpp/src/io/json/nested_json_gpu.cu +++ b/cpp/src/io/json/nested_json_gpu.cu @@ -83,8 +83,7 @@ struct tree_node { void check_input_size(std::size_t input_size) { // Transduce() writes symbol offsets that may be as large input_size-1 - CUDF_EXPECTS(input_size == 0 || - (input_size - 1) <= std::numeric_limits::max(), + CUDF_EXPECTS(input_size == 0 || (input_size - 1) <= std::numeric_limits::max(), "Given JSON input is too large"); } } // namespace diff --git a/cpp/src/io/json/read_json.cu b/cpp/src/io/json/read_json.cu index 99a5b17bce8..c424d2b3b62 100644 --- a/cpp/src/io/json/read_json.cu +++ b/cpp/src/io/json/read_json.cu @@ -351,10 +351,16 @@ table_with_metadata read_json(host_span> sources, * JSON inputs. */ std::size_t const total_source_size = sources_size(sources, 0, 0); - std::size_t chunk_offset = reader_opts.get_byte_range_offset(); - std::size_t chunk_size = reader_opts.get_byte_range_size(); - chunk_size = !chunk_size ? total_source_size - chunk_offset - : std::min(chunk_size, total_source_size - chunk_offset); + + // Batching is enabled only for JSONL inputs, not regular JSON files + CUDF_EXPECTS( + reader_opts.is_enabled_lines() || total_source_size < std::numeric_limits::max(), + "Parsing Regular JSON inputs of size greater than INT_MAX bytes is not supported"); + + std::size_t chunk_offset = reader_opts.get_byte_range_offset(); + std::size_t chunk_size = reader_opts.get_byte_range_size(); + chunk_size = !chunk_size ? total_source_size - chunk_offset + : std::min(chunk_size, total_source_size - chunk_offset); std::size_t const size_per_subchunk = estimate_size_per_subchunk(chunk_size); std::size_t const batch_size_upper_bound = get_batch_size_upper_bound(); From c141ca5ae2867909911839ad680bbf52580f8305 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 15 Oct 2024 05:42:17 -1000 Subject: [PATCH 07/12] Add string.convert.convert_integers APIs to pylibcudf (#16991) Contributes to https://github.com/rapidsai/cudf/issues/15162 Authors: - Matthew Roeschke (https://github.com/mroeschke) - https://github.com/brandon-b-miller Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - https://github.com/brandon-b-miller - Matthew Murray (https://github.com/Matt711) URL: https://github.com/rapidsai/cudf/pull/16991 --- .../strings/convert/convert_integers.rst | 6 + .../pylibcudf/strings/convert/index.rst | 1 + python/cudf/cudf/_lib/string_casting.pyx | 128 ++++------- .../strings/convert/convert_integers.pxd | 24 +- .../pylibcudf/strings/convert/CMakeLists.txt | 2 +- .../pylibcudf/strings/convert/__init__.pxd | 1 + .../pylibcudf/strings/convert/__init__.py | 1 + .../strings/convert/convert_integers.pxd | 17 ++ .../strings/convert/convert_integers.pyx | 206 ++++++++++++++++++ .../tests/test_string_convert_integers.py | 69 ++++++ 10 files changed, 354 insertions(+), 101 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/strings/convert/convert_integers.rst create mode 100644 python/pylibcudf/pylibcudf/strings/convert/convert_integers.pxd create mode 100644 python/pylibcudf/pylibcudf/strings/convert/convert_integers.pyx create mode 100644 python/pylibcudf/pylibcudf/tests/test_string_convert_integers.py diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/convert/convert_integers.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/convert/convert_integers.rst new file mode 100644 index 00000000000..71d146c0379 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/convert/convert_integers.rst @@ -0,0 +1,6 @@ +================ +convert_integers +================ + +.. automodule:: pylibcudf.strings.convert.convert_integers + :members: diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/convert/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/convert/index.rst index fa05cb7d786..3d07c1271b4 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/convert/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/strings/convert/index.rst @@ -9,6 +9,7 @@ convert convert_durations convert_fixed_point convert_floats + convert_integers convert_ipv4 convert_lists convert_urls diff --git a/python/cudf/cudf/_lib/string_casting.pyx b/python/cudf/cudf/_lib/string_casting.pyx index 93b67bd4c9d..06ee07d8e2b 100644 --- a/python/cudf/cudf/_lib/string_casting.pyx +++ b/python/cudf/cudf/_lib/string_casting.pyx @@ -2,28 +2,10 @@ from cudf._lib.column cimport Column -from cudf._lib.scalar import as_device_scalar -from cudf._lib.types import SUPPORTED_NUMPY_TO_LIBCUDF_TYPES - -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move - -from pylibcudf.libcudf.column.column cimport column -from pylibcudf.libcudf.column.column_view cimport column_view -from pylibcudf.libcudf.strings.convert.convert_integers cimport ( - from_integers as cpp_from_integers, - hex_to_integers as cpp_hex_to_integers, - integers_to_hex as cpp_integers_to_hex, - is_hex as cpp_is_hex, - to_integers as cpp_to_integers, -) -from pylibcudf.libcudf.types cimport data_type, type_id - -from cudf._lib.types cimport underlying_type_t_type_id - import pylibcudf as plc +from pylibcudf.types cimport DataType -import cudf +from cudf._lib.scalar import as_device_scalar from cudf._lib.types cimport dtype_to_pylibcudf_type @@ -35,10 +17,10 @@ def floating_to_string(Column input_col): return Column.from_pylibcudf(plc_column) -def string_to_floating(Column input_col, object out_type): +def string_to_floating(Column input_col, DataType out_type): plc_column = plc.strings.convert.convert_floats.to_floats( input_col.to_pylibcudf(mode="read"), - dtype_to_pylibcudf_type(out_type) + out_type ) return Column.from_pylibcudf(plc_column) @@ -72,7 +54,7 @@ def stod(Column input_col): A Column with strings cast to double """ - return string_to_floating(input_col, cudf.dtype("float64")) + return string_to_floating(input_col, plc.DataType(plc.TypeId.FLOAT64)) def ftos(Column input_col): @@ -104,36 +86,22 @@ def stof(Column input_col): A Column with strings cast to float """ - return string_to_floating(input_col, cudf.dtype("float32")) + return string_to_floating(input_col, plc.DataType(plc.TypeId.FLOAT32)) def integer_to_string(Column input_col): - cdef column_view input_column_view = input_col.view() - cdef unique_ptr[column] c_result - with nogil: - c_result = move( - cpp_from_integers( - input_column_view)) - - return Column.from_unique_ptr(move(c_result)) - - -def string_to_integer(Column input_col, object out_type): - cdef column_view input_column_view = input_col.view() - cdef unique_ptr[column] c_result - cdef type_id tid = ( - ( - SUPPORTED_NUMPY_TO_LIBCUDF_TYPES[out_type] - ) + plc_column = plc.strings.convert.convert_integers.from_integers( + input_col.to_pylibcudf(mode="read"), ) - cdef data_type c_out_type = data_type(tid) - with nogil: - c_result = move( - cpp_to_integers( - input_column_view, - c_out_type)) + return Column.from_pylibcudf(plc_column) - return Column.from_unique_ptr(move(c_result)) + +def string_to_integer(Column input_col, DataType out_type): + plc_column = plc.strings.convert.convert_integers.to_integers( + input_col.to_pylibcudf(mode="read"), + out_type + ) + return Column.from_pylibcudf(plc_column) def i8tos(Column input_col): @@ -165,7 +133,7 @@ def stoi8(Column input_col): A Column with strings cast to int8 """ - return string_to_integer(input_col, cudf.dtype("int8")) + return string_to_integer(input_col, plc.DataType(plc.TypeId.INT8)) def i16tos(Column input_col): @@ -197,7 +165,7 @@ def stoi16(Column input_col): A Column with strings cast to int16 """ - return string_to_integer(input_col, cudf.dtype("int16")) + return string_to_integer(input_col, plc.DataType(plc.TypeId.INT16)) def itos(Column input_col): @@ -229,7 +197,7 @@ def stoi(Column input_col): A Column with strings cast to int32 """ - return string_to_integer(input_col, cudf.dtype("int32")) + return string_to_integer(input_col, plc.DataType(plc.TypeId.INT32)) def ltos(Column input_col): @@ -261,7 +229,7 @@ def stol(Column input_col): A Column with strings cast to int64 """ - return string_to_integer(input_col, cudf.dtype("int64")) + return string_to_integer(input_col, plc.DataType(plc.TypeId.INT64)) def ui8tos(Column input_col): @@ -293,7 +261,7 @@ def stoui8(Column input_col): A Column with strings cast to uint8 """ - return string_to_integer(input_col, cudf.dtype("uint8")) + return string_to_integer(input_col, plc.DataType(plc.TypeId.UINT8)) def ui16tos(Column input_col): @@ -325,7 +293,7 @@ def stoui16(Column input_col): A Column with strings cast to uint16 """ - return string_to_integer(input_col, cudf.dtype("uint16")) + return string_to_integer(input_col, plc.DataType(plc.TypeId.UINT16)) def uitos(Column input_col): @@ -357,7 +325,7 @@ def stoui(Column input_col): A Column with strings cast to uint32 """ - return string_to_integer(input_col, cudf.dtype("uint32")) + return string_to_integer(input_col, plc.DataType(plc.TypeId.UINT32)) def ultos(Column input_col): @@ -389,7 +357,7 @@ def stoul(Column input_col): A Column with strings cast to uint64 """ - return string_to_integer(input_col, cudf.dtype("uint64")) + return string_to_integer(input_col, plc.DataType(plc.TypeId.UINT64)) def to_booleans(Column input_col): @@ -477,8 +445,6 @@ def istimestamp(Column input_col, str format): A Column of boolean values identifying strings that matched the format. """ - if input_col.size == 0: - return cudf.core.column.column_empty(0, dtype=cudf.dtype("bool")) plc_column = plc.strings.convert.convert_datetime.is_timestamp( input_col.to_pylibcudf(mode="read"), format @@ -582,7 +548,7 @@ def is_ipv4(Column source_strings): return Column.from_pylibcudf(plc_column) -def htoi(Column input_col, **kwargs): +def htoi(Column input_col): """ Converting input column of type string having hex values to integer of out_type @@ -595,22 +561,11 @@ def htoi(Column input_col, **kwargs): ------- A Column of integers parsed from hexadecimal string values. """ - - cdef column_view input_column_view = input_col.view() - cdef type_id tid = ( - ( - SUPPORTED_NUMPY_TO_LIBCUDF_TYPES[cudf.dtype("int64")] - ) + plc_column = plc.strings.convert.convert_integers.hex_to_integers( + input_col.to_pylibcudf(mode="read"), + plc.DataType(plc.TypeId.INT64) ) - cdef data_type c_out_type = data_type(tid) - - cdef unique_ptr[column] c_result - with nogil: - c_result = move( - cpp_hex_to_integers(input_column_view, - c_out_type)) - - return Column.from_unique_ptr(move(c_result)) + return Column.from_pylibcudf(plc_column) def is_hex(Column source_strings): @@ -618,15 +573,10 @@ def is_hex(Column source_strings): Returns a Column of boolean values with True for `source_strings` that have hex characters. """ - cdef unique_ptr[column] c_result - cdef column_view source_view = source_strings.view() - - with nogil: - c_result = move(cpp_is_hex( - source_view - )) - - return Column.from_unique_ptr(move(c_result)) + plc_column = plc.strings.convert.convert_integers.is_hex( + source_strings.to_pylibcudf(mode="read"), + ) + return Column.from_pylibcudf(plc_column) def itoh(Column input_col): @@ -642,11 +592,7 @@ def itoh(Column input_col): ------- A Column of strings with hexadecimal characters. """ - - cdef column_view input_column_view = input_col.view() - cdef unique_ptr[column] c_result - with nogil: - c_result = move( - cpp_integers_to_hex(input_column_view)) - - return Column.from_unique_ptr(move(c_result)) + plc_column = plc.strings.convert.convert_integers.integers_to_hex( + input_col.to_pylibcudf(mode="read"), + ) + return Column.from_pylibcudf(plc_column) diff --git a/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_integers.pxd b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_integers.pxd index f12aab0a2e4..69d566b8c49 100644 --- a/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_integers.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/strings/convert/convert_integers.pxd @@ -1,6 +1,7 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr +from pylibcudf.exception_handler cimport libcudf_exception_handler from pylibcudf.libcudf.column.column cimport column from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.types cimport data_type @@ -9,23 +10,28 @@ from pylibcudf.libcudf.types cimport data_type cdef extern from "cudf/strings/convert/convert_integers.hpp" namespace \ "cudf::strings" nogil: cdef unique_ptr[column] to_integers( - column_view input_col, - data_type output_type) except + + column_view input, + data_type output_type) except +libcudf_exception_handler cdef unique_ptr[column] from_integers( - column_view input_col) except + + column_view integers) except +libcudf_exception_handler + + cdef unique_ptr[column] is_integer( + column_view input + ) except +libcudf_exception_handler cdef unique_ptr[column] is_integer( - column_view source_strings - ) except + + column_view input, + data_type int_type + ) except +libcudf_exception_handler cdef unique_ptr[column] hex_to_integers( - column_view input_col, + column_view input, data_type output_type) except + cdef unique_ptr[column] is_hex( - column_view source_strings - ) except + + column_view input + ) except +libcudf_exception_handler cdef unique_ptr[column] integers_to_hex( - column_view input_col) except + + column_view input) except +libcudf_exception_handler diff --git a/python/pylibcudf/pylibcudf/strings/convert/CMakeLists.txt b/python/pylibcudf/pylibcudf/strings/convert/CMakeLists.txt index 846070870b1..8ba84ba7d50 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/strings/convert/CMakeLists.txt @@ -14,7 +14,7 @@ set(cython_sources convert_booleans.pyx convert_datetime.pyx convert_durations.pyx convert_fixed_point.pyx - convert_floats.pyx convert_ipv4.pyx convert_lists.pyx convert_urls.pyx + convert_floats.pyx convert_integers.pyx convert_ipv4.pyx convert_lists.pyx convert_urls.pyx ) set(linked_libraries cudf::cudf) diff --git a/python/pylibcudf/pylibcudf/strings/convert/__init__.pxd b/python/pylibcudf/pylibcudf/strings/convert/__init__.pxd index 799532d72c6..85300936e4d 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/__init__.pxd +++ b/python/pylibcudf/pylibcudf/strings/convert/__init__.pxd @@ -5,6 +5,7 @@ from . cimport ( convert_durations, convert_fixed_point, convert_floats, + convert_integers, convert_ipv4, convert_lists, convert_urls, diff --git a/python/pylibcudf/pylibcudf/strings/convert/__init__.py b/python/pylibcudf/pylibcudf/strings/convert/__init__.py index deb2d8ab74b..aa27a7c8929 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/__init__.py +++ b/python/pylibcudf/pylibcudf/strings/convert/__init__.py @@ -5,6 +5,7 @@ convert_durations, convert_fixed_point, convert_floats, + convert_integers, convert_ipv4, convert_lists, convert_urls, diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pxd b/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pxd new file mode 100644 index 00000000000..eff2e080c27 --- /dev/null +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pxd @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from pylibcudf.column cimport Column +from pylibcudf.types cimport DataType + + +cpdef Column to_integers(Column input, DataType output_type) + +cpdef Column from_integers(Column integers) + +cpdef Column is_integer(Column input, DataType int_type=*) + +cpdef Column hex_to_integers(Column input, DataType output_type) + +cpdef Column is_hex(Column input) + +cpdef Column integers_to_hex(Column input) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pyx new file mode 100644 index 00000000000..5558683a502 --- /dev/null +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pyx @@ -0,0 +1,206 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.strings.convert cimport ( + convert_integers as cpp_convert_integers, +) +from pylibcudf.types cimport DataType + + +cpdef Column to_integers(Column input, DataType output_type): + """ + Returns a new integer numeric column parsing integer values from the + provided strings column. + + For details, cpp:func:`cudf::strings::to_integers`. + + Parameters + ---------- + input : Column + Strings instance for this operation. + + output_type : DataType + Type of integer numeric column to return. + + Returns + ------- + Column + New column with integers converted from strings. + """ + cdef unique_ptr[column] c_result + + with nogil: + c_result = move( + cpp_convert_integers.to_integers( + input.view(), + output_type.c_obj + ) + ) + + return Column.from_libcudf(move(c_result)) + + +cpdef Column from_integers(Column integers): + """ + Returns a new strings column converting the integer values from the + provided column into strings. + + For details, cpp:func:`cudf::strings::from_integers`. + + Parameters + ---------- + integers : Column + Strings instance for this operation. + + Returns + ------- + Column + New strings column with integers as strings. + """ + cdef unique_ptr[column] c_result + + with nogil: + c_result = move( + cpp_convert_integers.from_integers( + integers.view(), + ) + ) + + return Column.from_libcudf(move(c_result)) + + +cpdef Column is_integer(Column input, DataType int_type=None): + """ + Returns a boolean column identifying strings in which all + characters are valid for conversion to integers. + + For details, cpp:func:`cudf::strings::is_integer`. + + Parameters + ---------- + input : Column + Strings instance for this operation. + + int_type : DataType + Integer type used for checking underflow and overflow. + By default, does not check an integer type for underflow + or overflow. + + Returns + ------- + Column + New column of boolean results for each string. + """ + cdef unique_ptr[column] c_result + + if int_type is None: + with nogil: + c_result = move( + cpp_convert_integers.is_integer( + input.view(), + ) + ) + else: + with nogil: + c_result = move( + cpp_convert_integers.is_integer( + input.view(), + int_type.c_obj + ) + ) + + return Column.from_libcudf(move(c_result)) + + +cpdef Column hex_to_integers(Column input, DataType output_type): + """ + Returns a new integer numeric column parsing hexadecimal values + from the provided strings column. + + For details, cpp:func:`cudf::strings::hex_to_integers`. + + Parameters + ---------- + input : Column + Strings instance for this operation. + + output_type : DataType + Type of integer numeric column to return. + + Returns + ------- + Column + New column with integers converted from strings. + """ + cdef unique_ptr[column] c_result + + with nogil: + c_result = move( + cpp_convert_integers.hex_to_integers( + input.view(), + output_type.c_obj + ) + ) + + return Column.from_libcudf(move(c_result)) + + +cpdef Column is_hex(Column input): + """ + Returns a boolean column identifying strings in which all + characters are valid for conversion to integers from hex. + + For details, cpp:func:`cudf::strings::is_hex`. + + Parameters + ---------- + input : Column + Strings instance for this operation. + + Returns + ------- + Column + New column of boolean results for each string. + """ + cdef unique_ptr[column] c_result + + with nogil: + c_result = move( + cpp_convert_integers.is_hex( + input.view(), + ) + ) + + return Column.from_libcudf(move(c_result)) + + +cpdef Column integers_to_hex(Column input): + """ + Returns a new strings column converting integer columns to hexadecimal + characters. + + For details, cpp:func:`cudf::strings::integers_to_hex`. + + Parameters + ---------- + input : Column + Integer column to convert to hex. + + Returns + ------- + Column + New strings column with hexadecimal characters. + """ + cdef unique_ptr[column] c_result + + with nogil: + c_result = move( + cpp_convert_integers.integers_to_hex( + input.view(), + ) + ) + + return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/tests/test_string_convert_integers.py b/python/pylibcudf/pylibcudf/tests/test_string_convert_integers.py new file mode 100644 index 00000000000..6d1d565af30 --- /dev/null +++ b/python/pylibcudf/pylibcudf/tests/test_string_convert_integers.py @@ -0,0 +1,69 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +import pyarrow as pa +import pylibcudf as plc +from utils import assert_column_eq + + +def test_to_integers(): + typ = pa.int8() + arr = pa.array(["1", "-1", None]) + result = plc.strings.convert.convert_integers.to_integers( + plc.interop.from_arrow(arr), plc.interop.from_arrow(typ) + ) + expected = arr.cast(typ) + assert_column_eq(result, expected) + + +def test_from_integers(): + arr = pa.array([1, -1, None]) + result = plc.strings.convert.convert_integers.from_integers( + plc.interop.from_arrow(arr) + ) + expected = pa.array(["1", "-1", None]) + assert_column_eq(result, expected) + + +def test_is_integer(): + arr = pa.array(["1", "-1", "1.2", "A", None]) + plc_column = plc.interop.from_arrow(arr) + result = plc.strings.convert.convert_integers.is_integer(plc_column) + expected = pa.array([True, True, False, False, None]) + assert_column_eq(result, expected) + + result = plc.strings.convert.convert_integers.is_integer( + plc_column, plc.interop.from_arrow(pa.uint8()) + ) + expected = pa.array([True, False, False, False, None]) + assert_column_eq(result, expected) + + +def test_hex_to_integers(): + typ = pa.int32() + data = ["0xff", "0x2a", None] + result = plc.strings.convert.convert_integers.hex_to_integers( + plc.interop.from_arrow(pa.array(data)), plc.interop.from_arrow(typ) + ) + expected = pa.array( + [int(val, 16) if isinstance(val, str) else val for val in data], + type=typ, + ) + assert_column_eq(result, expected) + + +def test_is_hex(): + arr = pa.array(["0xff", "123", "!", None]) + result = plc.strings.convert.convert_integers.is_hex( + plc.interop.from_arrow(arr) + ) + expected = pa.array([True, True, False, None]) + assert_column_eq(result, expected) + + +def test_integers_to_hex(): + data = [255, -42, None] + arr = pa.array(data) + result = plc.strings.convert.convert_integers.integers_to_hex( + plc.interop.from_arrow(arr) + ) + expected = pa.array(["FF", "FFFFFFFFFFFFFFD6", None]) + assert_column_eq(result, expected) From 7bcfc87935b7a202002d54e17e140789b02f16e9 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:17:53 -0400 Subject: [PATCH 08/12] Fix regex handling of fixed quantifier with 0 range (#17067) Fixes regex logic handling of a pattern with a fixed quantifier that includes a zero-range. Added new gtests for this specific case. Bug was introduced in #16798 Closes #17065 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) - Vyas Ramasubramani (https://github.com/vyasr) - MithunR (https://github.com/mythrocks) - Basit Ayantunde (https://github.com/lamarrr) URL: https://github.com/rapidsai/cudf/pull/17067 --- cpp/src/strings/regex/regcomp.cpp | 6 ++-- cpp/tests/strings/contains_tests.cpp | 34 +++++++++++++++++++++++ cpp/tests/strings/replace_regex_tests.cpp | 28 +++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/cpp/src/strings/regex/regcomp.cpp b/cpp/src/strings/regex/regcomp.cpp index 775a2580f60..b923a301f84 100644 --- a/cpp/src/strings/regex/regcomp.cpp +++ b/cpp/src/strings/regex/regcomp.cpp @@ -710,9 +710,7 @@ class regex_parser { std::stack lbra_stack; auto repeat_start_index = -1; - for (std::size_t index = 0; index < in.size(); index++) { - auto const item = in[index]; - + for (auto const item : in) { if (item.type != COUNTED && item.type != COUNTED_LAZY) { out.push_back(item); if (item.type == LBRA || item.type == LBRA_NC) { @@ -739,7 +737,7 @@ class regex_parser { auto const m = item.d.count.m; // maximum count assert(n >= 0 && "invalid repeat count value n"); // zero-repeat edge-case: need to erase the previous items - if (n == 0 && m == 0) { out.erase(begin, end); } + if (n == 0) { out.erase(begin, end); } std::vector repeat_copy(begin, end); // special handling for quantified capture groups diff --git a/cpp/tests/strings/contains_tests.cpp b/cpp/tests/strings/contains_tests.cpp index 216ddfce5f1..cceec1d3537 100644 --- a/cpp/tests/strings/contains_tests.cpp +++ b/cpp/tests/strings/contains_tests.cpp @@ -474,6 +474,40 @@ TEST_F(StringsContainsTests, FixedQuantifier) } } +TEST_F(StringsContainsTests, ZeroRangeQuantifier) +{ + auto input = cudf::test::strings_column_wrapper({"a", "", "abc", "XYAZ", "ABC", "ZYXA"}); + auto sv = cudf::strings_column_view(input); + + auto pattern = std::string("A{0,}"); // should match everyting + auto prog = cudf::strings::regex_program::create(pattern); + + { + auto expected = cudf::test::fixed_width_column_wrapper({1, 1, 1, 1, 1, 1}); + auto results = cudf::strings::contains_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + { + auto expected = cudf::test::fixed_width_column_wrapper({2, 1, 4, 5, 4, 5}); + auto results = cudf::strings::count_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + + pattern = std::string("(?:ab){0,3}"); + prog = cudf::strings::regex_program::create(pattern); + + { + auto expected = cudf::test::fixed_width_column_wrapper({1, 1, 1, 1, 1, 1}); + auto results = cudf::strings::contains_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + { + auto expected = cudf::test::fixed_width_column_wrapper({2, 1, 3, 5, 4, 5}); + auto results = cudf::strings::count_re(sv, *prog); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } +} + TEST_F(StringsContainsTests, NestedQuantifier) { auto input = cudf::test::strings_column_wrapper({"TEST12 1111 2222 3333 4444 5555", diff --git a/cpp/tests/strings/replace_regex_tests.cpp b/cpp/tests/strings/replace_regex_tests.cpp index 9847d8d6bb5..abc12b00a81 100644 --- a/cpp/tests/strings/replace_regex_tests.cpp +++ b/cpp/tests/strings/replace_regex_tests.cpp @@ -200,6 +200,34 @@ TEST_F(StringsReplaceRegexTest, ZeroLengthMatch) CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*results, expected); } +TEST_F(StringsReplaceRegexTest, ZeroRangeQuantifier) +{ + auto input = cudf::test::strings_column_wrapper({"a", "", "123", "XYAZ", "abc", "zéyab"}); + auto sv = cudf::strings_column_view(input); + + auto pattern = std::string("A{0,5}"); + auto prog = cudf::strings::regex_program::create(pattern); + auto repl = cudf::string_scalar("_"); + auto expected = cudf::test::strings_column_wrapper( + {"_a_", "_", "_1_2_3_", "_X_Y__Z_", "_a_b_c_", "_z_é_y_a_b_"}); + auto results = cudf::strings::replace_re(sv, *prog, repl); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + + pattern = std::string("[a0-9]{0,2}"); + prog = cudf::strings::regex_program::create(pattern); + expected = + cudf::test::strings_column_wrapper({"__", "_", "___", "_X_Y_A_Z_", "__b_c_", "_z_é_y__b_"}); + results = cudf::strings::replace_re(sv, *prog, repl); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + + pattern = std::string("(?:ab){0,3}"); + prog = cudf::strings::regex_program::create(pattern); + expected = + cudf::test::strings_column_wrapper({"_a_", "_", "_1_2_3_", "_X_Y_A_Z_", "__c_", "_z_é_y__"}); + results = cudf::strings::replace_re(sv, *prog, repl); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); +} + TEST_F(StringsReplaceRegexTest, Multiline) { auto const multiline = cudf::strings::regex_flags::MULTILINE; From 3420c71cb72f63db8d63164446cca042f354a08e Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:45:17 -0400 Subject: [PATCH 09/12] Migrate remaining nvtext NGrams APIs to pylibcudf (#17070) Apart of #15162 Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - https://github.com/brandon-b-miller URL: https://github.com/rapidsai/cudf/pull/17070 --- .../api_docs/pylibcudf/nvtext/index.rst | 1 + .../pylibcudf/nvtext/ngrams_tokenize.rst | 6 +++ .../cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx | 46 ++++------------ .../pylibcudf/pylibcudf/nvtext/CMakeLists.txt | 4 +- .../pylibcudf/pylibcudf/nvtext/__init__.pxd | 11 +++- python/pylibcudf/pylibcudf/nvtext/__init__.py | 3 +- .../pylibcudf/nvtext/ngrams_tokenize.pxd | 13 +++++ .../pylibcudf/nvtext/ngrams_tokenize.pyx | 54 +++++++++++++++++++ .../tests/test_nvtext_ngrams_tokenize.py | 37 +++++++++++++ 9 files changed, 135 insertions(+), 40 deletions(-) create mode 100644 docs/cudf/source/user_guide/api_docs/pylibcudf/nvtext/ngrams_tokenize.rst create mode 100644 python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pxd create mode 100644 python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pyx create mode 100644 python/pylibcudf/pylibcudf/tests/test_nvtext_ngrams_tokenize.py diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/nvtext/index.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/nvtext/index.rst index f6caabe324d..58303356336 100644 --- a/docs/cudf/source/user_guide/api_docs/pylibcudf/nvtext/index.rst +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/nvtext/index.rst @@ -8,3 +8,4 @@ nvtext generate_ngrams jaccard minhash + ngrams_tokenize diff --git a/docs/cudf/source/user_guide/api_docs/pylibcudf/nvtext/ngrams_tokenize.rst b/docs/cudf/source/user_guide/api_docs/pylibcudf/nvtext/ngrams_tokenize.rst new file mode 100644 index 00000000000..ce6db76f889 --- /dev/null +++ b/docs/cudf/source/user_guide/api_docs/pylibcudf/nvtext/ngrams_tokenize.rst @@ -0,0 +1,6 @@ +=============== +ngrams_tokenize +=============== + +.. automodule:: pylibcudf.nvtext.ngrams_tokenize + :members: diff --git a/python/cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx b/python/cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx index dec4f037d98..6521116eafe 100644 --- a/python/cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx +++ b/python/cudf/cudf/_lib/nvtext/ngrams_tokenize.pyx @@ -2,48 +2,22 @@ from cudf.core.buffer import acquire_spill_lock -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move - -from pylibcudf.libcudf.column.column cimport column -from pylibcudf.libcudf.column.column_view cimport column_view -from pylibcudf.libcudf.nvtext.ngrams_tokenize cimport ( - ngrams_tokenize as cpp_ngrams_tokenize, -) -from pylibcudf.libcudf.scalar.scalar cimport string_scalar -from pylibcudf.libcudf.types cimport size_type - from cudf._lib.column cimport Column -from cudf._lib.scalar cimport DeviceScalar + +from pylibcudf import nvtext @acquire_spill_lock() def ngrams_tokenize( - Column strings, + Column input, int ngrams, object py_delimiter, object py_separator ): - - cdef DeviceScalar delimiter = py_delimiter.device_value - cdef DeviceScalar separator = py_separator.device_value - - cdef column_view c_strings = strings.view() - cdef size_type c_ngrams = ngrams - cdef const string_scalar* c_separator = separator\ - .get_raw_ptr() - cdef const string_scalar* c_delimiter = delimiter\ - .get_raw_ptr() - cdef unique_ptr[column] c_result - - with nogil: - c_result = move( - cpp_ngrams_tokenize( - c_strings, - c_ngrams, - c_delimiter[0], - c_separator[0] - ) - ) - - return Column.from_unique_ptr(move(c_result)) + result = nvtext.ngrams_tokenize.ngrams_tokenize( + input.to_pylibcudf(mode="read"), + ngrams, + py_delimiter.device_value.c_value, + py_separator.device_value.c_value + ) + return Column.from_pylibcudf(result) diff --git a/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt b/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt index 7fd65beeeb0..94df9bbbebb 100644 --- a/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/nvtext/CMakeLists.txt @@ -12,7 +12,9 @@ # the License. # ============================================================================= -set(cython_sources edit_distance.pyx generate_ngrams.pyx jaccard.pyx minhash.pyx) +set(cython_sources edit_distance.pyx generate_ngrams.pyx jaccard.pyx minhash.pyx + ngrams_tokenize.pyx +) set(linked_libraries cudf::cudf) rapids_cython_create_modules( diff --git a/python/pylibcudf/pylibcudf/nvtext/__init__.pxd b/python/pylibcudf/pylibcudf/nvtext/__init__.pxd index 9eed1da1ab5..b6659827688 100644 --- a/python/pylibcudf/pylibcudf/nvtext/__init__.pxd +++ b/python/pylibcudf/pylibcudf/nvtext/__init__.pxd @@ -1,10 +1,17 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from . cimport edit_distance, generate_ngrams, jaccard, minhash +from . cimport ( + edit_distance, + generate_ngrams, + jaccard, + minhash, + ngrams_tokenize, +) __all__ = [ "edit_distance", "generate_ngrams", "jaccard", - "minhash" + "minhash", + "ngrams_tokenize" ] diff --git a/python/pylibcudf/pylibcudf/nvtext/__init__.py b/python/pylibcudf/pylibcudf/nvtext/__init__.py index a3a2363f7ef..f74633a3521 100644 --- a/python/pylibcudf/pylibcudf/nvtext/__init__.py +++ b/python/pylibcudf/pylibcudf/nvtext/__init__.py @@ -1,10 +1,11 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from . import edit_distance, generate_ngrams, jaccard, minhash +from . import edit_distance, generate_ngrams, jaccard, minhash, ngrams_tokenize __all__ = [ "edit_distance", "generate_ngrams", "jaccard", "minhash", + "ngrams_tokenize", ] diff --git a/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pxd b/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pxd new file mode 100644 index 00000000000..4f791ba1ee9 --- /dev/null +++ b/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pxd @@ -0,0 +1,13 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from pylibcudf.column cimport Column +from pylibcudf.libcudf.types cimport size_type +from pylibcudf.scalar cimport Scalar + + +cpdef Column ngrams_tokenize( + Column input, + size_type ngrams, + Scalar delimiter, + Scalar separator +) diff --git a/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pyx b/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pyx new file mode 100644 index 00000000000..8a1854c5f0d --- /dev/null +++ b/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pyx @@ -0,0 +1,54 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +from cython.operator cimport dereference +from libcpp.memory cimport unique_ptr +from libcpp.utility cimport move +from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.nvtext.ngrams_tokenize cimport ( + ngrams_tokenize as cpp_ngrams_tokenize, +) +from pylibcudf.libcudf.scalar.scalar cimport string_scalar +from pylibcudf.libcudf.types cimport size_type +from pylibcudf.scalar cimport Scalar + + +cpdef Column ngrams_tokenize( + Column input, + size_type ngrams, + Scalar delimiter, + Scalar separator +): + """ + Returns a single column of strings by tokenizing the input strings column + and then producing ngrams of each string. + + For details, see :cpp:func:`ngrams_tokenize` + + Parameters + ---------- + input : Column + Input strings + ngrams : size_type + The ngram number to generate + delimiter : Scalar + UTF-8 characters used to separate each string into tokens. + An empty string will separate tokens using whitespace. + separator : Scalar + The string to use for separating ngram tokens + + Returns + ------- + Column + New strings columns of tokens + """ + cdef unique_ptr[column] c_result + + with nogil: + c_result = cpp_ngrams_tokenize( + input.view(), + ngrams, + dereference(delimiter.get()), + dereference(separator.get()), + ) + return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/tests/test_nvtext_ngrams_tokenize.py b/python/pylibcudf/pylibcudf/tests/test_nvtext_ngrams_tokenize.py new file mode 100644 index 00000000000..283a009288d --- /dev/null +++ b/python/pylibcudf/pylibcudf/tests/test_nvtext_ngrams_tokenize.py @@ -0,0 +1,37 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import pyarrow as pa +import pylibcudf as plc +import pytest +from utils import assert_column_eq + + +@pytest.fixture(scope="module") +def input_col(): + arr = ["a*b*c*d", "a b c d", "a-b-c-d", "a*b c-d"] + return pa.array(arr) + + +@pytest.mark.parametrize("ngrams", [2, 3]) +@pytest.mark.parametrize("delim", ["*", " ", "-"]) +@pytest.mark.parametrize("sep", ["_", "&", ","]) +def test_ngrams_tokenize(input_col, ngrams, delim, sep): + def ngrams_tokenize(strings, ngrams, delim, sep): + tokens = [] + for s in strings: + ss = s.split(delim) + for i in range(len(ss) - ngrams + 1): + token = sep.join(ss[i : i + ngrams]) + tokens.append(token) + return tokens + + result = plc.nvtext.ngrams_tokenize.ngrams_tokenize( + plc.interop.from_arrow(input_col), + ngrams, + plc.interop.from_arrow(pa.scalar(delim)), + plc.interop.from_arrow(pa.scalar(sep)), + ) + expected = pa.array( + ngrams_tokenize(input_col.to_pylist(), ngrams, delim, sep) + ) + assert_column_eq(result, expected) From 95df62a1d76026e876fff4e51811de5e95a6b06e Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:15:50 -0400 Subject: [PATCH 10/12] Remove unnecessary `std::move`'s in pylibcudf (#16983) This PR removes a lot of unnecessary `std::move`'s from pylibcudf. These were necessary with older versions of Cython, but newer versions appear to generate the correct C++ without needing the extra hints. Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16983 --- python/pylibcudf/pylibcudf/binaryop.pyx | 36 +++--- python/pylibcudf/pylibcudf/column.pyx | 8 +- .../pylibcudf/pylibcudf/column_factories.pxd | 4 +- .../pylibcudf/pylibcudf/column_factories.pyx | 68 ++++------ python/pylibcudf/pylibcudf/concatenate.pyx | 4 +- python/pylibcudf/pylibcudf/copying.pyx | 103 +++++++-------- python/pylibcudf/pylibcudf/datetime.pyx | 4 +- python/pylibcudf/pylibcudf/filling.pyx | 38 +++--- python/pylibcudf/pylibcudf/groupby.pyx | 17 +-- python/pylibcudf/pylibcudf/interop.pyx | 4 +- python/pylibcudf/pylibcudf/io/avro.pyx | 2 +- python/pylibcudf/pylibcudf/io/csv.pyx | 2 +- python/pylibcudf/pylibcudf/io/json.pyx | 2 +- python/pylibcudf/pylibcudf/io/orc.pyx | 2 +- python/pylibcudf/pylibcudf/io/timezone.pyx | 8 +- python/pylibcudf/pylibcudf/join.pyx | 2 +- python/pylibcudf/pylibcudf/json.pyx | 10 +- python/pylibcudf/pylibcudf/labeling.pyx | 14 +-- python/pylibcudf/pylibcudf/lists.pyx | 74 +++++------ python/pylibcudf/pylibcudf/merge.pyx | 12 +- python/pylibcudf/pylibcudf/null_mask.pyx | 8 +- .../pylibcudf/nvtext/edit_distance.pyx | 4 +- .../pylibcudf/nvtext/generate_ngrams.pyx | 26 ++-- python/pylibcudf/pylibcudf/nvtext/jaccard.pyx | 10 +- python/pylibcudf/pylibcudf/nvtext/minhash.pyx | 40 +++--- python/pylibcudf/pylibcudf/partitioning.pyx | 20 +-- python/pylibcudf/pylibcudf/quantiles.pyx | 30 ++--- python/pylibcudf/pylibcudf/reduce.pyx | 22 ++-- python/pylibcudf/pylibcudf/replace.pyx | 57 ++++----- python/pylibcudf/pylibcudf/reshape.pyx | 4 +- python/pylibcudf/pylibcudf/rolling.pyx | 29 ++--- python/pylibcudf/pylibcudf/round.pyx | 10 +- python/pylibcudf/pylibcudf/search.pyx | 32 ++--- python/pylibcudf/pylibcudf/sorting.pyx | 118 ++++++++---------- .../pylibcudf/pylibcudf/stream_compaction.pyx | 42 +++---- .../pylibcudf/strings/attributes.pyx | 6 +- .../pylibcudf/strings/char_types.pyx | 22 ++-- .../pylibcudf/pylibcudf/strings/contains.pyx | 20 +-- .../strings/convert/convert_booleans.pyx | 18 ++- .../strings/convert/convert_durations.pyx | 18 ++- .../strings/convert/convert_fixed_point.pyx | 22 ++-- .../strings/convert/convert_floats.pyx | 20 +-- .../strings/convert/convert_ipv4.pyx | 18 +-- .../strings/convert/convert_lists.pyx | 10 +- .../strings/convert/convert_urls.pyx | 12 +- .../pylibcudf/pylibcudf/strings/extract.pyx | 16 +-- python/pylibcudf/pylibcudf/strings/find.pyx | 82 +++++------- .../pylibcudf/strings/find_multiple.pyx | 8 +- .../pylibcudf/pylibcudf/strings/findall.pyx | 16 +-- .../pylibcudf/pylibcudf/strings/padding.pyx | 20 ++- python/pylibcudf/pylibcudf/strings/repeat.pyx | 16 +-- .../pylibcudf/pylibcudf/strings/replace.pyx | 12 +- .../pylibcudf/strings/split/partition.pyx | 16 +-- .../pylibcudf/strings/split/split.pyx | 80 +++++------- .../pylibcudf/pylibcudf/strings/translate.pyx | 20 ++- python/pylibcudf/pylibcudf/strings/wrap.pyx | 8 +- python/pylibcudf/pylibcudf/table.pyx | 4 +- python/pylibcudf/pylibcudf/transform.pyx | 16 ++- python/pylibcudf/pylibcudf/transpose.pyx | 2 +- python/pylibcudf/pylibcudf/unary.pyx | 12 +- 60 files changed, 544 insertions(+), 816 deletions(-) diff --git a/python/pylibcudf/pylibcudf/binaryop.pyx b/python/pylibcudf/pylibcudf/binaryop.pyx index 5f9d145139a..51b2b4cfaa3 100644 --- a/python/pylibcudf/pylibcudf/binaryop.pyx +++ b/python/pylibcudf/pylibcudf/binaryop.pyx @@ -52,33 +52,27 @@ cpdef Column binary_operation( if LeftBinaryOperand is Column and RightBinaryOperand is Column: with nogil: - result = move( - cpp_binaryop.binary_operation( - lhs.view(), - rhs.view(), - op, - output_type.c_obj - ) + result = cpp_binaryop.binary_operation( + lhs.view(), + rhs.view(), + op, + output_type.c_obj ) elif LeftBinaryOperand is Column and RightBinaryOperand is Scalar: with nogil: - result = move( - cpp_binaryop.binary_operation( - lhs.view(), - dereference(rhs.c_obj), - op, - output_type.c_obj - ) + result = cpp_binaryop.binary_operation( + lhs.view(), + dereference(rhs.c_obj), + op, + output_type.c_obj ) elif LeftBinaryOperand is Scalar and RightBinaryOperand is Column: with nogil: - result = move( - cpp_binaryop.binary_operation( - dereference(lhs.c_obj), - rhs.view(), - op, - output_type.c_obj - ) + result = cpp_binaryop.binary_operation( + dereference(lhs.c_obj), + rhs.view(), + op, + output_type.c_obj ) else: raise ValueError(f"Invalid arguments {lhs} and {rhs}") diff --git a/python/pylibcudf/pylibcudf/column.pyx b/python/pylibcudf/pylibcudf/column.pyx index 03808f0b664..4e5698566d0 100644 --- a/python/pylibcudf/pylibcudf/column.pyx +++ b/python/pylibcudf/pylibcudf/column.pyx @@ -138,7 +138,7 @@ cdef class Column: cdef size_type null_count = libcudf_col.get().null_count() - cdef column_contents contents = move(libcudf_col.get().release()) + cdef column_contents contents = libcudf_col.get().release() # Note that when converting to cudf Column objects we'll need to pull # out the base object. @@ -247,7 +247,7 @@ cdef class Column: cdef const scalar* c_scalar = slr.get() cdef unique_ptr[column] c_result with nogil: - c_result = move(make_column_from_scalar(dereference(c_scalar), size)) + c_result = make_column_from_scalar(dereference(c_scalar), size) return Column.from_libcudf(move(c_result)) @staticmethod @@ -269,7 +269,7 @@ cdef class Column: cdef Scalar slr = Scalar.empty_like(like) cdef unique_ptr[column] c_result with nogil: - c_result = move(make_column_from_scalar(dereference(slr.get()), size)) + c_result = make_column_from_scalar(dereference(slr.get()), size) return Column.from_libcudf(move(c_result)) @staticmethod @@ -373,7 +373,7 @@ cdef class Column: """Create a copy of the column.""" cdef unique_ptr[column] c_result with nogil: - c_result = move(make_unique[column](self.view())) + c_result = make_unique[column](self.view()) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/column_factories.pxd b/python/pylibcudf/pylibcudf/column_factories.pxd index fef02359240..d556085ab64 100644 --- a/python/pylibcudf/pylibcudf/column_factories.pxd +++ b/python/pylibcudf/pylibcudf/column_factories.pxd @@ -1,7 +1,5 @@ # Copyright (c) 2024, NVIDIA CORPORATION. -from libcpp.memory cimport unique_ptr -from libcpp.utility cimport move -from pylibcudf.libcudf.types cimport mask_state, size_type +from pylibcudf.libcudf.types cimport mask_state from .column cimport Column from .types cimport DataType, size_type, type_id diff --git a/python/pylibcudf/pylibcudf/column_factories.pyx b/python/pylibcudf/pylibcudf/column_factories.pyx index e9085e3ea02..ac942a620b5 100644 --- a/python/pylibcudf/pylibcudf/column_factories.pyx +++ b/python/pylibcudf/pylibcudf/column_factories.pyx @@ -39,29 +39,17 @@ cpdef Column make_empty_column(MakeEmptyColumnOperand type_or_id): if isinstance(type_or_id, TypeId): id = type_or_id with nogil: - result = move( - cpp_make_empty_column( - id - ) - ) + result = cpp_make_empty_column(id) else: raise TypeError( "Must pass a TypeId or DataType" ) elif MakeEmptyColumnOperand is DataType: with nogil: - result = move( - cpp_make_empty_column( - type_or_id.c_obj - ) - ) + result = cpp_make_empty_column(type_or_id.c_obj) elif MakeEmptyColumnOperand is type_id: with nogil: - result = move( - cpp_make_empty_column( - type_or_id - ) - ) + result = cpp_make_empty_column(type_or_id) else: raise TypeError( "Must pass a TypeId or DataType" @@ -92,12 +80,10 @@ cpdef Column make_numeric_column( else: raise TypeError("Invalid mask argument") with nogil: - result = move( - cpp_make_numeric_column( - type_.c_obj, - size, - state - ) + result = cpp_make_numeric_column( + type_.c_obj, + size, + state ) return Column.from_libcudf(move(result)) @@ -121,12 +107,10 @@ cpdef Column make_fixed_point_column( else: raise TypeError("Invalid mask argument") with nogil: - result = move( - cpp_make_fixed_point_column( - type_.c_obj, - size, - state - ) + result = cpp_make_fixed_point_column( + type_.c_obj, + size, + state ) return Column.from_libcudf(move(result)) @@ -151,12 +135,10 @@ cpdef Column make_timestamp_column( else: raise TypeError("Invalid mask argument") with nogil: - result = move( - cpp_make_timestamp_column( - type_.c_obj, - size, - state - ) + result = cpp_make_timestamp_column( + type_.c_obj, + size, + state ) return Column.from_libcudf(move(result)) @@ -181,12 +163,10 @@ cpdef Column make_duration_column( else: raise TypeError("Invalid mask argument") with nogil: - result = move( - cpp_make_duration_column( - type_.c_obj, - size, - state - ) + result = cpp_make_duration_column( + type_.c_obj, + size, + state ) return Column.from_libcudf(move(result)) @@ -211,12 +191,10 @@ cpdef Column make_fixed_width_column( else: raise TypeError("Invalid mask argument") with nogil: - result = move( - cpp_make_fixed_width_column( - type_.c_obj, - size, - state - ) + result = cpp_make_fixed_width_column( + type_.c_obj, + size, + state ) return Column.from_libcudf(move(result)) diff --git a/python/pylibcudf/pylibcudf/concatenate.pyx b/python/pylibcudf/pylibcudf/concatenate.pyx index 8bdcc086e0f..10c860d97bb 100644 --- a/python/pylibcudf/pylibcudf/concatenate.pyx +++ b/python/pylibcudf/pylibcudf/concatenate.pyx @@ -40,14 +40,14 @@ cpdef concatenate(list objects): c_tables.push_back((tbl).view()) with nogil: - c_tbl_result = move(cpp_concatenate.concatenate(c_tables)) + c_tbl_result = cpp_concatenate.concatenate(c_tables) return Table.from_libcudf(move(c_tbl_result)) elif isinstance(objects[0], Column): for column in objects: c_columns.push_back((column).view()) with nogil: - c_col_result = move(cpp_concatenate.concatenate(c_columns)) + c_col_result = cpp_concatenate.concatenate(c_columns) return Column.from_libcudf(move(c_col_result)) else: raise ValueError("input must be a list of Columns or Tables") diff --git a/python/pylibcudf/pylibcudf/copying.pyx b/python/pylibcudf/pylibcudf/copying.pyx index 9743119d92a..4938f1a3dda 100644 --- a/python/pylibcudf/pylibcudf/copying.pyx +++ b/python/pylibcudf/pylibcudf/copying.pyx @@ -67,13 +67,12 @@ cpdef Table gather( """ cdef unique_ptr[table] c_result with nogil: - c_result = move( - cpp_copying.gather( - source_table.view(), - gather_map.view(), - bounds_policy - ) + c_result = cpp_copying.gather( + source_table.view(), + gather_map.view(), + bounds_policy ) + return Table.from_libcudf(move(c_result)) @@ -121,22 +120,18 @@ cpdef Table scatter( cdef vector[reference_wrapper[const scalar]] source_scalars if TableOrListOfScalars is Table: with nogil: - c_result = move( - cpp_copying.scatter( - source.view(), - scatter_map.view(), - target_table.view(), - ) + c_result = cpp_copying.scatter( + source.view(), + scatter_map.view(), + target_table.view(), ) else: source_scalars = _as_vector(source) with nogil: - c_result = move( - cpp_copying.scatter( - source_scalars, - scatter_map.view(), - target_table.view(), - ) + c_result = cpp_copying.scatter( + source_scalars, + scatter_map.view(), + target_table.view(), ) return Table.from_libcudf(move(c_result)) @@ -160,11 +155,11 @@ cpdef ColumnOrTable empty_like(ColumnOrTable input): cdef unique_ptr[column] c_col_result if ColumnOrTable is Column: with nogil: - c_col_result = move(cpp_copying.empty_like(input.view())) + c_col_result = cpp_copying.empty_like(input.view()) return Column.from_libcudf(move(c_col_result)) else: with nogil: - c_tbl_result = move(cpp_copying.empty_like(input.view())) + c_tbl_result = cpp_copying.empty_like(input.view()) return Table.from_libcudf(move(c_tbl_result)) @@ -195,13 +190,11 @@ cpdef Column allocate_like( cdef size_type c_size = size if size is not None else input_column.size() with nogil: - c_result = move( - cpp_copying.allocate_like( + c_result = cpp_copying.allocate_like( input_column.view(), c_size, policy, ) - ) return Column.from_libcudf(move(c_result)) @@ -298,12 +291,12 @@ cpdef Column copy_range( cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_copying.copy_range( + c_result = cpp_copying.copy_range( input_column.view(), target_column.view(), input_begin, input_end, - target_begin) + target_begin ) return Column.from_libcudf(move(c_result)) @@ -337,13 +330,11 @@ cpdef Column shift(Column input, size_type offset, Scalar fill_value): """ cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_copying.shift( + c_result = cpp_copying.shift( input.view(), offset, dereference(fill_value.c_obj) ) - ) return Column.from_libcudf(move(c_result)) @@ -378,7 +369,7 @@ cpdef list slice(ColumnOrTable input, list indices): cdef int i if ColumnOrTable is Column: with nogil: - c_col_result = move(cpp_copying.slice(input.view(), c_indices)) + c_col_result = cpp_copying.slice(input.view(), c_indices) return [ Column.from_column_view(c_col_result[i], input) @@ -386,7 +377,7 @@ cpdef list slice(ColumnOrTable input, list indices): ] else: with nogil: - c_tbl_result = move(cpp_copying.slice(input.view(), c_indices)) + c_tbl_result = cpp_copying.slice(input.view(), c_indices) return [ Table.from_table_view(c_tbl_result[i], input) @@ -418,7 +409,7 @@ cpdef list split(ColumnOrTable input, list splits): if ColumnOrTable is Column: with nogil: - c_col_result = move(cpp_copying.split(input.view(), c_splits)) + c_col_result = cpp_copying.split(input.view(), c_splits) return [ Column.from_column_view(c_col_result[i], input) @@ -426,7 +417,7 @@ cpdef list split(ColumnOrTable input, list splits): ] else: with nogil: - c_tbl_result = move(cpp_copying.split(input.view(), c_splits)) + c_tbl_result = cpp_copying.split(input.view(), c_splits) return [ Table.from_table_view(c_tbl_result[i], input) @@ -472,29 +463,25 @@ cpdef Column copy_if_else( if LeftCopyIfElseOperand is Column and RightCopyIfElseOperand is Column: with nogil: - result = move( - cpp_copying.copy_if_else(lhs.view(), rhs.view(), boolean_mask.view()) + result = cpp_copying.copy_if_else( + lhs.view(), + rhs.view(), + boolean_mask.view() ) elif LeftCopyIfElseOperand is Column and RightCopyIfElseOperand is Scalar: with nogil: - result = move( - cpp_copying.copy_if_else( - lhs.view(), dereference(rhs.c_obj), boolean_mask.view() - ) + result = cpp_copying.copy_if_else( + lhs.view(), dereference(rhs.c_obj), boolean_mask.view() ) elif LeftCopyIfElseOperand is Scalar and RightCopyIfElseOperand is Column: with nogil: - result = move( - cpp_copying.copy_if_else( - dereference(lhs.c_obj), rhs.view(), boolean_mask.view() - ) + result = cpp_copying.copy_if_else( + dereference(lhs.c_obj), rhs.view(), boolean_mask.view() ) else: with nogil: - result = move( - cpp_copying.copy_if_else( - dereference(lhs.c_obj), dereference(rhs.c_obj), boolean_mask.view() - ) + result = cpp_copying.copy_if_else( + dereference(lhs.c_obj), dereference(rhs.c_obj), boolean_mask.view() ) return Column.from_libcudf(move(result)) @@ -541,22 +528,18 @@ cpdef Table boolean_mask_scatter( if TableOrListOfScalars is Table: with nogil: - result = move( - cpp_copying.boolean_mask_scatter( - input.view(), - target.view(), - boolean_mask.view() - ) + result = cpp_copying.boolean_mask_scatter( + input.view(), + target.view(), + boolean_mask.view() ) else: source_scalars = _as_vector(input) with nogil: - result = move( - cpp_copying.boolean_mask_scatter( - source_scalars, - target.view(), - boolean_mask.view(), - ) + result = cpp_copying.boolean_mask_scatter( + source_scalars, + target.view(), + boolean_mask.view(), ) return Table.from_libcudf(move(result)) @@ -586,8 +569,6 @@ cpdef Scalar get_element(Column input_column, size_type index): """ cdef unique_ptr[scalar] c_output with nogil: - c_output = move( - cpp_copying.get_element(input_column.view(), index) - ) + c_output = cpp_copying.get_element(input_column.view(), index) return Scalar.from_libcudf(move(c_output)) diff --git a/python/pylibcudf/pylibcudf/datetime.pyx b/python/pylibcudf/pylibcudf/datetime.pyx index 784d29128bf..ac4335cca56 100644 --- a/python/pylibcudf/pylibcudf/datetime.pyx +++ b/python/pylibcudf/pylibcudf/datetime.pyx @@ -33,7 +33,7 @@ cpdef Column extract_year( cdef unique_ptr[column] result with nogil: - result = move(cpp_extract_year(values.view())) + result = cpp_extract_year(values.view()) return Column.from_libcudf(move(result)) cpdef Column extract_datetime_component( @@ -60,5 +60,5 @@ cpdef Column extract_datetime_component( cdef unique_ptr[column] result with nogil: - result = move(cpp_extract_datetime_component(values.view(), component)) + result = cpp_extract_datetime_component(values.view(), component) return Column.from_libcudf(move(result)) diff --git a/python/pylibcudf/pylibcudf/filling.pyx b/python/pylibcudf/pylibcudf/filling.pyx index 61b430e64aa..0372e1132cc 100644 --- a/python/pylibcudf/pylibcudf/filling.pyx +++ b/python/pylibcudf/pylibcudf/filling.pyx @@ -48,13 +48,11 @@ cpdef Column fill( cdef unique_ptr[column] result with nogil: - result = move( - cpp_fill( - destination.view(), - begin, - end, - dereference(( value).c_obj) - ) + result = cpp_fill( + destination.view(), + begin, + end, + dereference(( value).c_obj) ) return Column.from_libcudf(move(result)) @@ -112,12 +110,10 @@ cpdef Column sequence(size_type size, Scalar init, Scalar step): cdef unique_ptr[column] result cdef size_type c_size = size with nogil: - result = move( - cpp_sequence( - c_size, - dereference(init.c_obj), - dereference(step.c_obj), - ) + result = cpp_sequence( + c_size, + dereference(init.c_obj), + dereference(step.c_obj), ) return Column.from_libcudf(move(result)) @@ -152,18 +148,14 @@ cpdef Table repeat( if ColumnOrSize is Column: with nogil: - result = move( - cpp_repeat( - input_table.view(), - count.view() - ) + result = cpp_repeat( + input_table.view(), + count.view() ) if ColumnOrSize is size_type: with nogil: - result = move( - cpp_repeat( - input_table.view(), - count - ) + result = cpp_repeat( + input_table.view(), + count ) return Table.from_libcudf(move(result)) diff --git a/python/pylibcudf/pylibcudf/groupby.pyx b/python/pylibcudf/pylibcudf/groupby.pyx index afb95dba5b3..71f9ecb0453 100644 --- a/python/pylibcudf/pylibcudf/groupby.pyx +++ b/python/pylibcudf/pylibcudf/groupby.pyx @@ -176,7 +176,7 @@ cdef class GroupBy: # We rely on libcudf to tell us this rather than checking the types beforehand # ourselves. with nogil: - c_res = move(dereference(self.c_obj).aggregate(c_requests)) + c_res = dereference(self.c_obj).aggregate(c_requests) return GroupBy._parse_outputs(move(c_res)) cpdef tuple scan(self, list requests): @@ -205,7 +205,7 @@ cdef class GroupBy: cdef pair[unique_ptr[table], vector[aggregation_result]] c_res with nogil: - c_res = move(dereference(self.c_obj).scan(c_requests)) + c_res = dereference(self.c_obj).scan(c_requests) return GroupBy._parse_outputs(move(c_res)) cpdef tuple shift(self, Table values, list offset, list fill_values): @@ -234,10 +234,11 @@ cdef class GroupBy: cdef vector[size_type] c_offset = offset cdef pair[unique_ptr[table], unique_ptr[table]] c_res with nogil: - c_res = move( - dereference(self.c_obj).shift(values.view(), c_offset, c_fill_values) + c_res = dereference(self.c_obj).shift( + values.view(), + c_offset, + c_fill_values ) - return ( Table.from_libcudf(move(c_res.first)), Table.from_libcudf(move(c_res.second)), @@ -264,10 +265,10 @@ cdef class GroupBy: cdef pair[unique_ptr[table], unique_ptr[table]] c_res cdef vector[replace_policy] c_replace_policies = replace_policies with nogil: - c_res = move( - dereference(self.c_obj).replace_nulls(value.view(), c_replace_policies) + c_res = dereference(self.c_obj).replace_nulls( + value.view(), + c_replace_policies ) - return ( Table.from_libcudf(move(c_res.first)), Table.from_libcudf(move(c_res.second)), diff --git a/python/pylibcudf/pylibcudf/interop.pyx b/python/pylibcudf/pylibcudf/interop.pyx index 1a03fa5b45b..642516a1b90 100644 --- a/python/pylibcudf/pylibcudf/interop.pyx +++ b/python/pylibcudf/pylibcudf/interop.pyx @@ -131,7 +131,7 @@ def _from_arrow_table(pyarrow_object, *, DataType data_type=None): cdef unique_ptr[table] c_result with nogil: # The libcudf function here will release the stream. - c_result = move(cpp_from_arrow_stream(c_stream)) + c_result = cpp_from_arrow_stream(c_stream) return Table.from_libcudf(move(c_result)) @@ -166,7 +166,7 @@ def _from_arrow_column(pyarrow_object, *, DataType data_type=None): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_from_arrow_column(c_schema, c_array)) + c_result = cpp_from_arrow_column(c_schema, c_array) # The capsule destructors should release automatically for us, but we # choose to do it explicitly here for clarity. diff --git a/python/pylibcudf/pylibcudf/io/avro.pyx b/python/pylibcudf/pylibcudf/io/avro.pyx index 438b0ff1634..fe765b34f82 100644 --- a/python/pylibcudf/pylibcudf/io/avro.pyx +++ b/python/pylibcudf/pylibcudf/io/avro.pyx @@ -45,7 +45,7 @@ cpdef TableWithMetadata read_avro( for col in columns: c_columns.push_back(str(col).encode()) - cdef avro_reader_options avro_opts = move( + cdef avro_reader_options avro_opts = ( avro_reader_options.builder(source_info.c_obj) .columns(c_columns) .skip_rows(skip_rows) diff --git a/python/pylibcudf/pylibcudf/io/csv.pyx b/python/pylibcudf/pylibcudf/io/csv.pyx index b53d6771cd6..2c61cc42d82 100644 --- a/python/pylibcudf/pylibcudf/io/csv.pyx +++ b/python/pylibcudf/pylibcudf/io/csv.pyx @@ -168,7 +168,7 @@ def read_csv( cdef vector[data_type] c_dtypes_list cdef map[string, data_type] c_dtypes_map - cdef csv_reader_options options = move( + cdef csv_reader_options options = ( csv_reader_options.builder(source_info.c_obj) .compression(compression) .mangle_dupe_cols(mangle_dupe_cols) diff --git a/python/pylibcudf/pylibcudf/io/json.pyx b/python/pylibcudf/pylibcudf/io/json.pyx index 29e49083bc6..65f78f830f1 100644 --- a/python/pylibcudf/pylibcudf/io/json.pyx +++ b/python/pylibcudf/pylibcudf/io/json.pyx @@ -59,7 +59,7 @@ cdef json_reader_options _setup_json_reader_options( json_recovery_mode_t recovery_mode): cdef vector[data_type] types_vec - cdef json_reader_options opts = move( + cdef json_reader_options opts = ( json_reader_options.builder(source_info.c_obj) .compression(compression) .lines(lines) diff --git a/python/pylibcudf/pylibcudf/io/orc.pyx b/python/pylibcudf/pylibcudf/io/orc.pyx index 01a5e4b04a1..70e0a7995a2 100644 --- a/python/pylibcudf/pylibcudf/io/orc.pyx +++ b/python/pylibcudf/pylibcudf/io/orc.pyx @@ -252,7 +252,7 @@ cpdef TableWithMetadata read_orc( """ cdef orc_reader_options opts cdef vector[vector[size_type]] c_stripes - opts = move( + opts = ( orc_reader_options.builder(source_info.c_obj) .use_index(use_index) .build() diff --git a/python/pylibcudf/pylibcudf/io/timezone.pyx b/python/pylibcudf/pylibcudf/io/timezone.pyx index e02239d7252..f120b65fb2c 100644 --- a/python/pylibcudf/pylibcudf/io/timezone.pyx +++ b/python/pylibcudf/pylibcudf/io/timezone.pyx @@ -33,11 +33,9 @@ cpdef Table make_timezone_transition_table(str tzif_dir, str timezone_name): cdef string c_tzname = timezone_name.encode() with nogil: - c_result = move( - cpp_make_timezone_transition_table( - make_optional[string](c_tzdir), - c_tzname - ) + c_result = cpp_make_timezone_transition_table( + make_optional[string](c_tzdir), + c_tzname ) return Table.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/join.pyx b/python/pylibcudf/pylibcudf/join.pyx index b019ed8f099..bc72647ea8e 100644 --- a/python/pylibcudf/pylibcudf/join.pyx +++ b/python/pylibcudf/pylibcudf/join.pyx @@ -212,5 +212,5 @@ cpdef Table cross_join(Table left, Table right): """ cdef unique_ptr[table] result with nogil: - result = move(cpp_join.cross_join(left.view(), right.view())) + result = cpp_join.cross_join(left.view(), right.view()) return Table.from_libcudf(move(result)) diff --git a/python/pylibcudf/pylibcudf/json.pyx b/python/pylibcudf/pylibcudf/json.pyx index 4a8d11068f9..ebb82f80408 100644 --- a/python/pylibcudf/pylibcudf/json.pyx +++ b/python/pylibcudf/pylibcudf/json.pyx @@ -143,12 +143,10 @@ cpdef Column get_json_object( cdef cpp_json.get_json_object_options c_options = options.options with nogil: - c_result = move( - cpp_json.get_json_object( - col.view(), - dereference(c_json_path), - c_options - ) + c_result = cpp_json.get_json_object( + col.view(), + dereference(c_json_path), + c_options ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/labeling.pyx b/python/pylibcudf/pylibcudf/labeling.pyx index b3f6a92d85c..226a9e14172 100644 --- a/python/pylibcudf/pylibcudf/labeling.pyx +++ b/python/pylibcudf/pylibcudf/labeling.pyx @@ -54,14 +54,12 @@ cpdef Column label_bins( ) with nogil: - c_result = move( - cpp_labeling.label_bins( - input.view(), - left_edges.view(), - c_left_inclusive, - right_edges.view(), - c_right_inclusive, - ) + c_result = cpp_labeling.label_bins( + input.view(), + left_edges.view(), + c_left_inclusive, + right_edges.view(), + c_right_inclusive, ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/lists.pyx b/python/pylibcudf/pylibcudf/lists.pyx index 6f82124d06e..ecaf62d6895 100644 --- a/python/pylibcudf/pylibcudf/lists.pyx +++ b/python/pylibcudf/pylibcudf/lists.pyx @@ -69,7 +69,7 @@ cpdef Table explode_outer(Table input, size_type explode_column_idx): cdef unique_ptr[table] c_result with nogil: - c_result = move(cpp_explode.explode_outer(input.view(), explode_column_idx)) + c_result = cpp_explode.explode_outer(input.view(), explode_column_idx) return Table.from_libcudf(move(c_result)) @@ -92,7 +92,7 @@ cpdef Column concatenate_rows(Table input): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_concatenate_rows(input.view())) + c_result = cpp_concatenate_rows(input.view()) return Column.from_libcudf(move(c_result)) @@ -123,10 +123,7 @@ cpdef Column concatenate_list_elements(Column input, bool dropna): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_concatenate_list_elements( - input.view(), - null_policy, - )) + c_result = cpp_concatenate_list_elements(input.view(), null_policy) return Column.from_libcudf(move(c_result)) @@ -161,12 +158,12 @@ cpdef Column contains(Column input, ColumnOrScalar search_key): raise TypeError("Must pass a Column or Scalar") with nogil: - c_result = move(cpp_contains.contains( + c_result = cpp_contains.contains( list_view.view(), search_key.view() if ColumnOrScalar is Column else dereference( search_key.get() ), - )) + ) return Column.from_libcudf(move(c_result)) @@ -190,7 +187,7 @@ cpdef Column contains_nulls(Column input): cdef unique_ptr[column] c_result cdef ListColumnView list_view = input.list_view() with nogil: - c_result = move(cpp_contains.contains_nulls(list_view.view())) + c_result = cpp_contains.contains_nulls(list_view.view()) return Column.from_libcudf(move(c_result)) @@ -229,13 +226,13 @@ cpdef Column index_of(Column input, ColumnOrScalar search_key, bool find_first_o ) with nogil: - c_result = move(cpp_contains.index_of( + c_result = cpp_contains.index_of( list_view.view(), search_key.view() if ColumnOrScalar is Column else dereference( search_key.get() ), find_option, - )) + ) return Column.from_libcudf(move(c_result)) @@ -258,9 +255,7 @@ cpdef Column reverse(Column input): cdef ListColumnView list_view = input.list_view() with nogil: - c_result = move(cpp_reverse.reverse( - list_view.view(), - )) + c_result = cpp_reverse.reverse(list_view.view()) return Column.from_libcudf(move(c_result)) @@ -288,10 +283,10 @@ cpdef Column segmented_gather(Column input, Column gather_map_list): cdef ListColumnView list_view2 = gather_map_list.list_view() with nogil: - c_result = move(cpp_gather.segmented_gather( + c_result = cpp_gather.segmented_gather( list_view1.view(), list_view2.view(), - )) + ) return Column.from_libcudf(move(c_result)) @@ -316,10 +311,10 @@ cpdef Column extract_list_element(Column input, ColumnOrSizeType index): cdef ListColumnView list_view = input.list_view() with nogil: - c_result = move(cpp_extract_list_element( + c_result = cpp_extract_list_element( list_view.view(), index.view() if ColumnOrSizeType is Column else index, - )) + ) return Column.from_libcudf(move(c_result)) @@ -344,7 +339,7 @@ cpdef Column count_elements(Column input): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_count_elements(list_view.view())) + c_result = cpp_count_elements(list_view.view()) return Column.from_libcudf(move(c_result)) @@ -373,17 +368,14 @@ cpdef Column sequences(Column starts, Column sizes, Column steps = None): if steps is not None: with nogil: - c_result = move(cpp_filling.sequences( + c_result = cpp_filling.sequences( starts.view(), steps.view(), sizes.view(), - )) + ) else: with nogil: - c_result = move(cpp_filling.sequences( - starts.view(), - sizes.view(), - )) + c_result = cpp_filling.sequences(starts.view(), sizes.view()) return Column.from_libcudf(move(c_result)) cpdef Column sort_lists( @@ -423,17 +415,17 @@ cpdef Column sort_lists( with nogil: if stable: - c_result = move(cpp_stable_sort_lists( + c_result = cpp_stable_sort_lists( list_view.view(), c_sort_order, na_position, - )) + ) else: - c_result = move(cpp_sort_lists( + c_result = cpp_sort_lists( list_view.view(), c_sort_order, na_position, - )) + ) return Column.from_libcudf(move(c_result)) @@ -477,12 +469,12 @@ cpdef Column difference_distinct( ) with nogil: - c_result = move(cpp_set_operations.difference_distinct( + c_result = cpp_set_operations.difference_distinct( lhs_view.view(), rhs_view.view(), c_nulls_equal, c_nans_equal, - )) + ) return Column.from_libcudf(move(c_result)) @@ -525,12 +517,12 @@ cpdef Column have_overlap( ) with nogil: - c_result = move(cpp_set_operations.have_overlap( + c_result = cpp_set_operations.have_overlap( lhs_view.view(), rhs_view.view(), c_nulls_equal, c_nans_equal, - )) + ) return Column.from_libcudf(move(c_result)) @@ -573,12 +565,12 @@ cpdef Column intersect_distinct( ) with nogil: - c_result = move(cpp_set_operations.intersect_distinct( + c_result = cpp_set_operations.intersect_distinct( lhs_view.view(), rhs_view.view(), c_nulls_equal, c_nans_equal, - )) + ) return Column.from_libcudf(move(c_result)) @@ -622,12 +614,12 @@ cpdef Column union_distinct( ) with nogil: - c_result = move(cpp_set_operations.union_distinct( + c_result = cpp_set_operations.union_distinct( lhs_view.view(), rhs_view.view(), c_nulls_equal, c_nans_equal, - )) + ) return Column.from_libcudf(move(c_result)) @@ -652,10 +644,10 @@ cpdef Column apply_boolean_mask(Column input, Column boolean_mask): cdef ListColumnView list_view = input.list_view() cdef ListColumnView mask_view = boolean_mask.list_view() with nogil: - c_result = move(cpp_apply_boolean_mask( + c_result = cpp_apply_boolean_mask( list_view.view(), mask_view.view(), - )) + ) return Column.from_libcudf(move(c_result)) @@ -690,9 +682,9 @@ cpdef Column distinct(Column input, bool nulls_equal, bool nans_equal): ) with nogil: - c_result = move(cpp_distinct( + c_result = cpp_distinct( list_view.view(), c_nulls_equal, c_nans_equal, - )) + ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/merge.pyx b/python/pylibcudf/pylibcudf/merge.pyx index 6d707b67449..61a21aafdb2 100644 --- a/python/pylibcudf/pylibcudf/merge.pyx +++ b/python/pylibcudf/pylibcudf/merge.pyx @@ -47,12 +47,10 @@ cpdef Table merge ( cdef unique_ptr[table] c_result with nogil: - c_result = move( - cpp_merge.merge( - c_tables_to_merge, - c_key_cols, - c_column_order, - c_null_precedence, - ) + c_result = cpp_merge.merge( + c_tables_to_merge, + c_key_cols, + c_column_order, + c_null_precedence, ) return Table.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/null_mask.pyx b/python/pylibcudf/pylibcudf/null_mask.pyx index aae39987dac..74180951562 100644 --- a/python/pylibcudf/pylibcudf/null_mask.pyx +++ b/python/pylibcudf/pylibcudf/null_mask.pyx @@ -38,7 +38,7 @@ cpdef DeviceBuffer copy_bitmask(Column col): cdef device_buffer db with nogil: - db = move(cpp_null_mask.copy_bitmask(col.view())) + db = cpp_null_mask.copy_bitmask(col.view()) return buffer_to_python(move(db)) @@ -90,7 +90,7 @@ cpdef DeviceBuffer create_null_mask( cdef device_buffer db with nogil: - db = move(cpp_null_mask.create_null_mask(size, state)) + db = cpp_null_mask.create_null_mask(size, state) return buffer_to_python(move(db)) @@ -114,7 +114,7 @@ cpdef tuple bitmask_and(list columns): cdef pair[device_buffer, size_type] c_result with nogil: - c_result = move(cpp_null_mask.bitmask_and(c_table.view())) + c_result = cpp_null_mask.bitmask_and(c_table.view()) return buffer_to_python(move(c_result.first)), c_result.second @@ -138,6 +138,6 @@ cpdef tuple bitmask_or(list columns): cdef pair[device_buffer, size_type] c_result with nogil: - c_result = move(cpp_null_mask.bitmask_or(c_table.view())) + c_result = cpp_null_mask.bitmask_or(c_table.view()) return buffer_to_python(move(c_result.first)), c_result.second diff --git a/python/pylibcudf/pylibcudf/nvtext/edit_distance.pyx b/python/pylibcudf/pylibcudf/nvtext/edit_distance.pyx index fc98ccbc50c..dcacb2e1267 100644 --- a/python/pylibcudf/pylibcudf/nvtext/edit_distance.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/edit_distance.pyx @@ -33,7 +33,7 @@ cpdef Column edit_distance(Column input, Column targets): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_edit_distance(c_strings, c_targets)) + c_result = cpp_edit_distance(c_strings, c_targets) return Column.from_libcudf(move(c_result)) @@ -58,6 +58,6 @@ cpdef Column edit_distance_matrix(Column input): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_edit_distance_matrix(c_strings)) + c_result = cpp_edit_distance_matrix(c_strings) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/nvtext/generate_ngrams.pyx b/python/pylibcudf/pylibcudf/nvtext/generate_ngrams.pyx index 8c7a8edc01d..09859d09e9e 100644 --- a/python/pylibcudf/pylibcudf/nvtext/generate_ngrams.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/generate_ngrams.pyx @@ -40,12 +40,10 @@ cpdef Column generate_ngrams(Column input, size_type ngrams, Scalar separator): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_generate_ngrams( - c_strings, - ngrams, - c_separator[0] - ) + c_result = cpp_generate_ngrams( + c_strings, + ngrams, + c_separator[0] ) return Column.from_libcudf(move(c_result)) @@ -72,11 +70,9 @@ cpdef Column generate_character_ngrams(Column input, size_type ngrams = 2): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_generate_character_ngrams( - c_strings, - ngrams, - ) + c_result = cpp_generate_character_ngrams( + c_strings, + ngrams, ) return Column.from_libcudf(move(c_result)) @@ -102,10 +98,8 @@ cpdef Column hash_character_ngrams(Column input, size_type ngrams = 2): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_hash_character_ngrams( - c_strings, - ngrams, - ) + c_result = cpp_hash_character_ngrams( + c_strings, + ngrams, ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/nvtext/jaccard.pyx b/python/pylibcudf/pylibcudf/nvtext/jaccard.pyx index 9334d7ce751..3d8669865d9 100644 --- a/python/pylibcudf/pylibcudf/nvtext/jaccard.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/jaccard.pyx @@ -36,12 +36,10 @@ cpdef Column jaccard_index(Column input1, Column input2, size_type width): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_jaccard_index( - c_input1, - c_input2, - width - ) + c_result = cpp_jaccard_index( + c_input1, + c_input2, + width ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/nvtext/minhash.pyx b/python/pylibcudf/pylibcudf/nvtext/minhash.pyx index 5fabf6a3f89..f1e012e60e5 100644 --- a/python/pylibcudf/pylibcudf/nvtext/minhash.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/minhash.pyx @@ -46,13 +46,11 @@ cpdef Column minhash(Column input, ColumnOrScalar seeds, size_type width=4): raise TypeError("Must pass a Column or Scalar") with nogil: - c_result = move( - cpp_minhash( - input.view(), - seeds.view() if ColumnOrScalar is Column else - dereference(seeds.c_obj.get()), - width - ) + c_result = cpp_minhash( + input.view(), + seeds.view() if ColumnOrScalar is Column else + dereference(seeds.c_obj.get()), + width ) return Column.from_libcudf(move(c_result)) @@ -85,13 +83,11 @@ cpdef Column minhash64(Column input, ColumnOrScalar seeds, size_type width=4): raise TypeError("Must pass a Column or Scalar") with nogil: - c_result = move( - cpp_minhash64( - input.view(), - seeds.view() if ColumnOrScalar is Column else - dereference(seeds.c_obj.get()), - width - ) + c_result = cpp_minhash64( + input.view(), + seeds.view() if ColumnOrScalar is Column else + dereference(seeds.c_obj.get()), + width ) return Column.from_libcudf(move(c_result)) @@ -118,11 +114,9 @@ cpdef Column word_minhash(Column input, Column seeds): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_word_minhash( - input.view(), - seeds.view() - ) + c_result = cpp_word_minhash( + input.view(), + seeds.view() ) return Column.from_libcudf(move(c_result)) @@ -150,11 +144,9 @@ cpdef Column word_minhash64(Column input, Column seeds): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_word_minhash64( - input.view(), - seeds.view() - ) + c_result = cpp_word_minhash64( + input.view(), + seeds.view() ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/partitioning.pyx b/python/pylibcudf/pylibcudf/partitioning.pyx index 8fa70daab5a..3cff4843735 100644 --- a/python/pylibcudf/pylibcudf/partitioning.pyx +++ b/python/pylibcudf/pylibcudf/partitioning.pyx @@ -41,10 +41,10 @@ cpdef tuple[Table, list] hash_partition( cdef int c_num_partitions = num_partitions with nogil: - c_result = move( - cpp_partitioning.hash_partition( - input.view(), c_columns_to_hash, c_num_partitions - ) + c_result = cpp_partitioning.hash_partition( + input.view(), + c_columns_to_hash, + c_num_partitions ) return Table.from_libcudf(move(c_result.first)), list(c_result.second) @@ -74,8 +74,10 @@ cpdef tuple[Table, list] partition(Table t, Column partition_map, int num_partit cdef int c_num_partitions = num_partitions with nogil: - c_result = move( - cpp_partitioning.partition(t.view(), partition_map.view(), c_num_partitions) + c_result = cpp_partitioning.partition( + t.view(), + partition_map.view(), + c_num_partitions ) return Table.from_libcudf(move(c_result.first)), list(c_result.second) @@ -111,10 +113,8 @@ cpdef tuple[Table, list] round_robin_partition( cdef int c_start_partition = start_partition with nogil: - c_result = move( - cpp_partitioning.round_robin_partition( - input.view(), c_num_partitions, c_start_partition - ) + c_result = cpp_partitioning.round_robin_partition( + input.view(), c_num_partitions, c_start_partition ) return Table.from_libcudf(move(c_result.first)), list(c_result.second) diff --git a/python/pylibcudf/pylibcudf/quantiles.pyx b/python/pylibcudf/pylibcudf/quantiles.pyx index 3a771fbe7ef..7d92b598bd0 100644 --- a/python/pylibcudf/pylibcudf/quantiles.pyx +++ b/python/pylibcudf/pylibcudf/quantiles.pyx @@ -66,14 +66,12 @@ cpdef Column quantile( ordered_indices_view = ordered_indices.view() with nogil: - c_result = move( - cpp_quantile( - input.view(), - q, - interp, - ordered_indices_view, - exact, - ) + c_result = cpp_quantile( + input.view(), + q, + interp, + ordered_indices_view, + exact, ) return Column.from_libcudf(move(c_result)) @@ -141,15 +139,13 @@ cpdef Table quantiles( null_precedence_vec = null_precedence with nogil: - c_result = move( - cpp_quantiles( - input.view(), - q, - interp, - is_input_sorted, - column_order_vec, - null_precedence_vec, - ) + c_result = cpp_quantiles( + input.view(), + q, + interp, + is_input_sorted, + column_order_vec, + null_precedence_vec, ) return Table.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/reduce.pyx b/python/pylibcudf/pylibcudf/reduce.pyx index b0212a5b9c1..d9ec3a9bdc4 100644 --- a/python/pylibcudf/pylibcudf/reduce.pyx +++ b/python/pylibcudf/pylibcudf/reduce.pyx @@ -39,12 +39,10 @@ cpdef Scalar reduce(Column col, Aggregation agg, DataType data_type): cdef unique_ptr[scalar] result cdef const reduce_aggregation *c_agg = agg.view_underlying_as_reduce() with nogil: - result = move( - cpp_reduce.cpp_reduce( - col.view(), - dereference(c_agg), - data_type.c_obj - ) + result = cpp_reduce.cpp_reduce( + col.view(), + dereference(c_agg), + data_type.c_obj ) return Scalar.from_libcudf(move(result)) @@ -71,12 +69,10 @@ cpdef Column scan(Column col, Aggregation agg, scan_type inclusive): cdef unique_ptr[column] result cdef const scan_aggregation *c_agg = agg.view_underlying_as_scan() with nogil: - result = move( - cpp_reduce.cpp_scan( - col.view(), - dereference(c_agg), - inclusive, - ) + result = cpp_reduce.cpp_scan( + col.view(), + dereference(c_agg), + inclusive, ) return Column.from_libcudf(move(result)) @@ -99,7 +95,7 @@ cpdef tuple minmax(Column col): """ cdef pair[unique_ptr[scalar], unique_ptr[scalar]] result with nogil: - result = move(cpp_reduce.cpp_minmax(col.view())) + result = cpp_reduce.cpp_minmax(col.view()) return ( Scalar.from_libcudf(move(result.first)), diff --git a/python/pylibcudf/pylibcudf/replace.pyx b/python/pylibcudf/pylibcudf/replace.pyx index 115dee132fd..f77eba7ace5 100644 --- a/python/pylibcudf/pylibcudf/replace.pyx +++ b/python/pylibcudf/pylibcudf/replace.pyx @@ -56,28 +56,23 @@ cpdef Column replace_nulls(Column source_column, ReplacementType replacement): if isinstance(replacement, ReplacePolicy): policy = replacement with nogil: - c_result = move( - cpp_replace.replace_nulls(source_column.view(), policy) - ) + c_result = cpp_replace.replace_nulls(source_column.view(), policy) return Column.from_libcudf(move(c_result)) else: raise TypeError("replacement must be a Column, Scalar, or replace_policy") with nogil: if ReplacementType is Column: - c_result = move( - cpp_replace.replace_nulls(source_column.view(), replacement.view()) + c_result = cpp_replace.replace_nulls( + source_column.view(), + replacement.view() ) elif ReplacementType is Scalar: - c_result = move( - cpp_replace.replace_nulls( - source_column.view(), dereference(replacement.c_obj) - ) + c_result = cpp_replace.replace_nulls( + source_column.view(), dereference(replacement.c_obj) ) elif ReplacementType is replace_policy: - c_result = move( - cpp_replace.replace_nulls(source_column.view(), replacement) - ) + c_result = cpp_replace.replace_nulls(source_column.view(), replacement) else: assert False, "Internal error. Please contact pylibcudf developers" return Column.from_libcudf(move(c_result)) @@ -109,12 +104,10 @@ cpdef Column find_and_replace_all( """ cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_replace.find_and_replace_all( - source_column.view(), - values_to_replace.view(), - replacement_values.view(), - ) + c_result = cpp_replace.find_and_replace_all( + source_column.view(), + values_to_replace.view(), + replacement_values.view(), ) return Column.from_libcudf(move(c_result)) @@ -156,22 +149,18 @@ cpdef Column clamp( cdef unique_ptr[column] c_result with nogil: if lo_replace is None: - c_result = move( - cpp_replace.clamp( - source_column.view(), - dereference(lo.c_obj), - dereference(hi.c_obj), - ) + c_result = cpp_replace.clamp( + source_column.view(), + dereference(lo.c_obj), + dereference(hi.c_obj), ) else: - c_result = move( - cpp_replace.clamp( - source_column.view(), - dereference(lo.c_obj), - dereference(hi.c_obj), - dereference(lo_replace.c_obj), - dereference(hi_replace.c_obj), - ) + c_result = cpp_replace.clamp( + source_column.view(), + dereference(lo.c_obj), + dereference(hi.c_obj), + dereference(lo_replace.c_obj), + dereference(hi_replace.c_obj), ) return Column.from_libcudf(move(c_result)) @@ -199,9 +188,7 @@ cpdef Column normalize_nans_and_zeros(Column source_column, bool inplace=False): if inplace: cpp_replace.normalize_nans_and_zeros(source_column.mutable_view()) else: - c_result = move( - cpp_replace.normalize_nans_and_zeros(source_column.view()) - ) + c_result = cpp_replace.normalize_nans_and_zeros(source_column.view()) if not inplace: return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/reshape.pyx b/python/pylibcudf/pylibcudf/reshape.pyx index eb1499ebbea..6540b5198ab 100644 --- a/python/pylibcudf/pylibcudf/reshape.pyx +++ b/python/pylibcudf/pylibcudf/reshape.pyx @@ -38,7 +38,7 @@ cpdef Column interleave_columns(Table source_table): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_interleave_columns(source_table.view())) + c_result = cpp_interleave_columns(source_table.view()) return Column.from_libcudf(move(c_result)) @@ -63,6 +63,6 @@ cpdef Table tile(Table source_table, size_type count): cdef unique_ptr[table] c_result with nogil: - c_result = move(cpp_tile(source_table.view(), count)) + c_result = cpp_tile(source_table.view(), count) return Table.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/rolling.pyx b/python/pylibcudf/pylibcudf/rolling.pyx index a46540d7ffa..4fd0b005431 100644 --- a/python/pylibcudf/pylibcudf/rolling.pyx +++ b/python/pylibcudf/pylibcudf/rolling.pyx @@ -49,24 +49,21 @@ cpdef Column rolling_window( cdef const rolling_aggregation *c_agg = agg.view_underlying_as_rolling() if WindowType is Column: with nogil: - result = move( - cpp_rolling.rolling_window( - source.view(), - preceding_window.view(), - following_window.view(), - min_periods, - dereference(c_agg), - ) + result = cpp_rolling.rolling_window( + source.view(), + preceding_window.view(), + following_window.view(), + min_periods, + dereference(c_agg), ) else: with nogil: - result = move( - cpp_rolling.rolling_window( - source.view(), - preceding_window, - following_window, - min_periods, - dereference(c_agg), - ) + result = cpp_rolling.rolling_window( + source.view(), + preceding_window, + following_window, + min_periods, + dereference(c_agg), ) + return Column.from_libcudf(move(result)) diff --git a/python/pylibcudf/pylibcudf/round.pyx b/python/pylibcudf/pylibcudf/round.pyx index dc60d53b07e..689363e652d 100644 --- a/python/pylibcudf/pylibcudf/round.pyx +++ b/python/pylibcudf/pylibcudf/round.pyx @@ -39,12 +39,10 @@ cpdef Column round( """ cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_round( - source.view(), - decimal_places, - round_method - ) + c_result = cpp_round( + source.view(), + decimal_places, + round_method ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/search.pyx b/python/pylibcudf/pylibcudf/search.pyx index 814bc6553d8..1a870248046 100644 --- a/python/pylibcudf/pylibcudf/search.pyx +++ b/python/pylibcudf/pylibcudf/search.pyx @@ -41,13 +41,11 @@ cpdef Column lower_bound( cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_search.lower_bound( - haystack.view(), - needles.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_search.lower_bound( + haystack.view(), + needles.view(), + c_orders, + c_null_precedence, ) return Column.from_libcudf(move(c_result)) @@ -82,13 +80,11 @@ cpdef Column upper_bound( cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_search.upper_bound( - haystack.view(), - needles.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_search.upper_bound( + haystack.view(), + needles.view(), + c_orders, + c_null_precedence, ) return Column.from_libcudf(move(c_result)) @@ -112,10 +108,8 @@ cpdef Column contains(Column haystack, Column needles): """ cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_search.contains( - haystack.view(), - needles.view(), - ) + c_result = cpp_search.contains( + haystack.view(), + needles.view(), ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/sorting.pyx b/python/pylibcudf/pylibcudf/sorting.pyx index 42289d54bca..fc40f03e1fd 100644 --- a/python/pylibcudf/pylibcudf/sorting.pyx +++ b/python/pylibcudf/pylibcudf/sorting.pyx @@ -36,12 +36,10 @@ cpdef Column sorted_order(Table source_table, list column_order, list null_prece cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.sorted_order( - source_table.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.sorted_order( + source_table.view(), + c_orders, + c_null_precedence, ) return Column.from_libcudf(move(c_result)) @@ -74,12 +72,10 @@ cpdef Column stable_sorted_order( cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.stable_sorted_order( - source_table.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.stable_sorted_order( + source_table.view(), + c_orders, + c_null_precedence, ) return Column.from_libcudf(move(c_result)) @@ -118,15 +114,13 @@ cpdef Column rank( """ cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_sorting.rank( - input_view.view(), - method, - column_order, - null_handling, - null_precedence, - percentage, - ) + c_result = cpp_sorting.rank( + input_view.view(), + method, + column_order, + null_handling, + null_precedence, + percentage, ) return Column.from_libcudf(move(c_result)) @@ -154,12 +148,10 @@ cpdef bool is_sorted(Table tbl, list column_order, list null_precedence): cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.is_sorted( - tbl.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.is_sorted( + tbl.view(), + c_orders, + c_null_precedence, ) return c_result @@ -197,14 +189,12 @@ cpdef Table segmented_sort_by_key( cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.segmented_sort_by_key( - values.view(), - keys.view(), - segment_offsets.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.segmented_sort_by_key( + values.view(), + keys.view(), + segment_offsets.view(), + c_orders, + c_null_precedence, ) return Table.from_libcudf(move(c_result)) @@ -243,14 +233,12 @@ cpdef Table stable_segmented_sort_by_key( cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.stable_segmented_sort_by_key( - values.view(), - keys.view(), - segment_offsets.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.stable_segmented_sort_by_key( + values.view(), + keys.view(), + segment_offsets.view(), + c_orders, + c_null_precedence, ) return Table.from_libcudf(move(c_result)) @@ -285,13 +273,11 @@ cpdef Table sort_by_key( cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.sort_by_key( - values.view(), - keys.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.sort_by_key( + values.view(), + keys.view(), + c_orders, + c_null_precedence, ) return Table.from_libcudf(move(c_result)) @@ -326,13 +312,11 @@ cpdef Table stable_sort_by_key( cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.stable_sort_by_key( - values.view(), - keys.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.stable_sort_by_key( + values.view(), + keys.view(), + c_orders, + c_null_precedence, ) return Table.from_libcudf(move(c_result)) @@ -360,12 +344,10 @@ cpdef Table sort(Table source_table, list column_order, list null_precedence): cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.sort( - source_table.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.sort( + source_table.view(), + c_orders, + c_null_precedence, ) return Table.from_libcudf(move(c_result)) @@ -393,11 +375,9 @@ cpdef Table stable_sort(Table source_table, list column_order, list null_precede cdef vector[order] c_orders = column_order cdef vector[null_order] c_null_precedence = null_precedence with nogil: - c_result = move( - cpp_sorting.stable_sort( - source_table.view(), - c_orders, - c_null_precedence, - ) + c_result = cpp_sorting.stable_sort( + source_table.view(), + c_orders, + c_null_precedence, ) return Table.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/stream_compaction.pyx b/python/pylibcudf/pylibcudf/stream_compaction.pyx index d5475ea79d5..2145398a191 100644 --- a/python/pylibcudf/pylibcudf/stream_compaction.pyx +++ b/python/pylibcudf/pylibcudf/stream_compaction.pyx @@ -44,10 +44,8 @@ cpdef Table drop_nulls(Table source_table, list keys, size_type keep_threshold): cdef unique_ptr[table] c_result cdef vector[size_type] c_keys = keys with nogil: - c_result = move( - cpp_stream_compaction.drop_nulls( - source_table.view(), c_keys, keep_threshold - ) + c_result = cpp_stream_compaction.drop_nulls( + source_table.view(), c_keys, keep_threshold ) return Table.from_libcudf(move(c_result)) @@ -74,10 +72,8 @@ cpdef Table drop_nans(Table source_table, list keys, size_type keep_threshold): cdef unique_ptr[table] c_result cdef vector[size_type] c_keys = keys with nogil: - c_result = move( - cpp_stream_compaction.drop_nulls( - source_table.view(), c_keys, keep_threshold - ) + c_result = cpp_stream_compaction.drop_nulls( + source_table.view(), c_keys, keep_threshold ) return Table.from_libcudf(move(c_result)) @@ -101,10 +97,8 @@ cpdef Table apply_boolean_mask(Table source_table, Column boolean_mask): """ cdef unique_ptr[table] c_result with nogil: - c_result = move( - cpp_stream_compaction.apply_boolean_mask( - source_table.view(), boolean_mask.view() - ) + c_result = cpp_stream_compaction.apply_boolean_mask( + source_table.view(), boolean_mask.view() ) return Table.from_libcudf(move(c_result)) @@ -144,10 +138,8 @@ cpdef Table unique( cdef unique_ptr[table] c_result cdef vector[size_type] c_keys = keys with nogil: - c_result = move( - cpp_stream_compaction.unique( - input.view(), c_keys, keep, nulls_equal - ) + c_result = cpp_stream_compaction.unique( + input.view(), c_keys, keep, nulls_equal ) return Table.from_libcudf(move(c_result)) @@ -185,10 +177,8 @@ cpdef Table distinct( cdef unique_ptr[table] c_result cdef vector[size_type] c_keys = keys with nogil: - c_result = move( - cpp_stream_compaction.distinct( - input.view(), c_keys, keep, nulls_equal, nans_equal - ) + c_result = cpp_stream_compaction.distinct( + input.view(), c_keys, keep, nulls_equal, nans_equal ) return Table.from_libcudf(move(c_result)) @@ -221,10 +211,8 @@ cpdef Column distinct_indices( """ cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_stream_compaction.distinct_indices( - input.view(), keep, nulls_equal, nans_equal - ) + c_result = cpp_stream_compaction.distinct_indices( + input.view(), keep, nulls_equal, nans_equal ) return Column.from_libcudf(move(c_result)) @@ -262,10 +250,8 @@ cpdef Table stable_distinct( cdef unique_ptr[table] c_result cdef vector[size_type] c_keys = keys with nogil: - c_result = move( - cpp_stream_compaction.stable_distinct( - input.view(), c_keys, keep, nulls_equal, nans_equal - ) + c_result = cpp_stream_compaction.stable_distinct( + input.view(), c_keys, keep, nulls_equal, nans_equal ) return Table.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/attributes.pyx b/python/pylibcudf/pylibcudf/strings/attributes.pyx index 36bee7bd1d9..8e46a32835d 100644 --- a/python/pylibcudf/pylibcudf/strings/attributes.pyx +++ b/python/pylibcudf/pylibcudf/strings/attributes.pyx @@ -25,7 +25,7 @@ cpdef Column count_characters(Column source_strings): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_attributes.count_characters(source_strings.view())) + c_result = cpp_attributes.count_characters(source_strings.view()) return Column.from_libcudf(move(c_result)) @@ -48,7 +48,7 @@ cpdef Column count_bytes(Column source_strings): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_attributes.count_bytes(source_strings.view())) + c_result = cpp_attributes.count_bytes(source_strings.view()) return Column.from_libcudf(move(c_result)) @@ -71,6 +71,6 @@ cpdef Column code_points(Column source_strings): cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_attributes.code_points(source_strings.view())) + c_result = cpp_attributes.code_points(source_strings.view()) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/char_types.pyx b/python/pylibcudf/pylibcudf/strings/char_types.pyx index 6a24d79bc4b..cb04efe5e8f 100644 --- a/python/pylibcudf/pylibcudf/strings/char_types.pyx +++ b/python/pylibcudf/pylibcudf/strings/char_types.pyx @@ -38,12 +38,10 @@ cpdef Column all_characters_of_type( cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_char_types.all_characters_of_type( - source_strings.view(), - types, - verify_types, - ) + c_result = cpp_char_types.all_characters_of_type( + source_strings.view(), + types, + verify_types, ) return Column.from_libcudf(move(c_result)) @@ -81,13 +79,11 @@ cpdef Column filter_characters_of_type( cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_char_types.filter_characters_of_type( - source_strings.view(), - types_to_remove, - dereference(c_replacement), - types_to_keep, - ) + c_result = cpp_char_types.filter_characters_of_type( + source_strings.view(), + types_to_remove, + dereference(c_replacement), + types_to_keep, ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/contains.pyx b/python/pylibcudf/pylibcudf/strings/contains.pyx index 82bd1fbea32..d4b1130241d 100644 --- a/python/pylibcudf/pylibcudf/strings/contains.pyx +++ b/python/pylibcudf/pylibcudf/strings/contains.pyx @@ -38,10 +38,10 @@ cpdef Column contains_re( cdef unique_ptr[column] result with nogil: - result = move(cpp_contains.contains_re( + result = cpp_contains.contains_re( input.view(), prog.c_obj.get()[0] - )) + ) return Column.from_libcudf(move(result)) @@ -71,10 +71,10 @@ cpdef Column count_re( cdef unique_ptr[column] result with nogil: - result = move(cpp_contains.count_re( + result = cpp_contains.count_re( input.view(), prog.c_obj.get()[0] - )) + ) return Column.from_libcudf(move(result)) @@ -105,10 +105,10 @@ cpdef Column matches_re( cdef unique_ptr[column] result with nogil: - result = move(cpp_contains.matches_re( + result = cpp_contains.matches_re( input.view(), prog.c_obj.get()[0] - )) + ) return Column.from_libcudf(move(result)) @@ -149,19 +149,19 @@ cpdef Column like(Column input, ColumnOrScalar pattern, Scalar escape_character= if ColumnOrScalar is Column: with nogil: - result = move(cpp_contains.like( + result = cpp_contains.like( input.view(), pattern.view(), dereference(c_escape_character) - )) + ) elif ColumnOrScalar is Scalar: c_pattern = (pattern.c_obj.get()) with nogil: - result = move(cpp_contains.like( + result = cpp_contains.like( input.view(), dereference(c_pattern), dereference(c_escape_character) - )) + ) else: raise ValueError("pattern must be a Column or a Scalar") diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_booleans.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_booleans.pyx index 0c10f821ab6..dc12b291b11 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_booleans.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_booleans.pyx @@ -39,11 +39,9 @@ cpdef Column to_booleans(Column input, Scalar true_string): ) with nogil: - c_result = move( - cpp_convert_booleans.to_booleans( - input.view(), - dereference(c_true_string) - ) + c_result = cpp_convert_booleans.to_booleans( + input.view(), + dereference(c_true_string) ) return Column.from_libcudf(move(c_result)) @@ -80,12 +78,10 @@ cpdef Column from_booleans(Column booleans, Scalar true_string, Scalar false_str ) with nogil: - c_result = move( - cpp_convert_booleans.from_booleans( - booleans.view(), - dereference(c_true_string), - dereference(c_false_string), - ) + c_result = cpp_convert_booleans.from_booleans( + booleans.view(), + dereference(c_true_string), + dereference(c_false_string), ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_durations.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_durations.pyx index 76c5809c3d5..31980ace418 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_durations.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_durations.pyx @@ -43,12 +43,10 @@ cpdef Column to_durations( cdef string c_format = format.encode() with nogil: - c_result = move( - cpp_convert_durations.to_durations( - input.view(), - duration_type.c_obj, - c_format - ) + c_result = cpp_convert_durations.to_durations( + input.view(), + duration_type.c_obj, + c_format ) return Column.from_libcudf(move(c_result)) @@ -84,11 +82,9 @@ cpdef Column from_durations( cdef string c_format = format.encode() with nogil: - c_result = move( - cpp_convert_durations.from_durations( - durations.view(), - c_format - ) + c_result = cpp_convert_durations.from_durations( + durations.view(), + c_format ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_fixed_point.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_fixed_point.pyx index 60a8fca8baf..962a47dfadf 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_fixed_point.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_fixed_point.pyx @@ -33,11 +33,9 @@ cpdef Column to_fixed_point(Column input, DataType output_type): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_fixed_point.to_fixed_point( - input.view(), - output_type.c_obj, - ) + c_result = cpp_fixed_point.to_fixed_point( + input.view(), + output_type.c_obj, ) return Column.from_libcudf(move(c_result)) @@ -62,11 +60,7 @@ cpdef Column from_fixed_point(Column input): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_fixed_point.from_fixed_point( - input.view(), - ) - ) + c_result = cpp_fixed_point.from_fixed_point(input.view()) return Column.from_libcudf(move(c_result)) @@ -97,11 +91,9 @@ cpdef Column is_fixed_point(Column input, DataType decimal_type=None): decimal_type = DataType(type_id.DECIMAL64) with nogil: - c_result = move( - cpp_fixed_point.is_fixed_point( - input.view(), - decimal_type.c_obj, - ) + c_result = cpp_fixed_point.is_fixed_point( + input.view(), + decimal_type.c_obj, ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_floats.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_floats.pyx index 8081aadb085..1296f4f9db5 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_floats.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_floats.pyx @@ -33,11 +33,9 @@ cpdef Column to_floats(Column strings, DataType output_type): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_convert_floats.to_floats( - strings.view(), - output_type.c_obj, - ) + c_result = cpp_convert_floats.to_floats( + strings.view(), + output_type.c_obj, ) return Column.from_libcudf(move(c_result)) @@ -63,11 +61,7 @@ cpdef Column from_floats(Column floats): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_convert_floats.from_floats( - floats.view(), - ) - ) + c_result = cpp_convert_floats.from_floats(floats.view()) return Column.from_libcudf(move(c_result)) @@ -92,10 +86,6 @@ cpdef Column is_float(Column input): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_convert_floats.is_float( - input.view(), - ) - ) + c_result = cpp_convert_floats.is_float(input.view()) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_ipv4.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_ipv4.pyx index f2a980d4269..834781f95f3 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_ipv4.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_ipv4.pyx @@ -26,11 +26,7 @@ cpdef Column ipv4_to_integers(Column input): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_convert_ipv4.ipv4_to_integers( - input.view() - ) - ) + c_result = cpp_convert_ipv4.ipv4_to_integers(input.view()) return Column.from_libcudf(move(c_result)) @@ -54,11 +50,7 @@ cpdef Column integers_to_ipv4(Column integers): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_convert_ipv4.integers_to_ipv4( - integers.view() - ) - ) + c_result = cpp_convert_ipv4.integers_to_ipv4(integers.view()) return Column.from_libcudf(move(c_result)) @@ -83,10 +75,6 @@ cpdef Column is_ipv4(Column input): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_convert_ipv4.is_ipv4( - input.view() - ) - ) + c_result = cpp_convert_ipv4.is_ipv4(input.view()) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_lists.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_lists.pyx index 3fbc08a9ab5..cbfe5f5aa8b 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_lists.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_lists.pyx @@ -61,12 +61,10 @@ cpdef Column format_list_column( separators = make_empty_column(type_id.STRING) with nogil: - c_result = move( - cpp_convert_lists.format_list_column( - input.view(), - dereference(c_na_rep), - separators.view() - ) + c_result = cpp_convert_lists.format_list_column( + input.view(), + dereference(c_na_rep), + separators.view() ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_urls.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_urls.pyx index a5e080e53b7..82f8a75f1d9 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_urls.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_urls.pyx @@ -26,11 +26,7 @@ cpdef Column url_encode(Column input): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_convert_urls.url_encode( - input.view() - ) - ) + c_result = cpp_convert_urls.url_encode(input.view()) return Column.from_libcudf(move(c_result)) @@ -54,10 +50,6 @@ cpdef Column url_decode(Column input): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_convert_urls.url_decode( - input.view() - ) - ) + c_result = cpp_convert_urls.url_decode(input.view()) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/extract.pyx b/python/pylibcudf/pylibcudf/strings/extract.pyx index dcb11ca10ce..b56eccc8287 100644 --- a/python/pylibcudf/pylibcudf/strings/extract.pyx +++ b/python/pylibcudf/pylibcudf/strings/extract.pyx @@ -33,11 +33,9 @@ cpdef Table extract(Column input, RegexProgram prog): cdef unique_ptr[table] c_result with nogil: - c_result = move( - cpp_extract.extract( - input.view(), - prog.c_obj.get()[0] - ) + c_result = cpp_extract.extract( + input.view(), + prog.c_obj.get()[0] ) return Table.from_libcudf(move(c_result)) @@ -66,11 +64,9 @@ cpdef Column extract_all_record(Column input, RegexProgram prog): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_extract.extract_all_record( - input.view(), - prog.c_obj.get()[0] - ) + c_result = cpp_extract.extract_all_record( + input.view(), + prog.c_obj.get()[0] ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/find.pyx b/python/pylibcudf/pylibcudf/strings/find.pyx index 22d370bf7e8..6fc6dca24fd 100644 --- a/python/pylibcudf/pylibcudf/strings/find.pyx +++ b/python/pylibcudf/pylibcudf/strings/find.pyx @@ -50,22 +50,18 @@ cpdef Column find( cdef unique_ptr[column] result if ColumnOrScalar is Column: with nogil: - result = move( - cpp_find.find( - input.view(), - target.view(), - start - ) + result = cpp_find.find( + input.view(), + target.view(), + start ) elif ColumnOrScalar is Scalar: with nogil: - result = move( - cpp_find.find( - input.view(), - dereference((target.c_obj.get())), - start, - stop - ) + result = cpp_find.find( + input.view(), + dereference((target.c_obj.get())), + start, + stop ) else: raise ValueError(f"Invalid target {target}") @@ -104,13 +100,11 @@ cpdef Column rfind( """ cdef unique_ptr[column] result with nogil: - result = move( - cpp_find.rfind( - input.view(), - dereference((target.c_obj.get())), - start, - stop - ) + result = cpp_find.rfind( + input.view(), + dereference((target.c_obj.get())), + start, + stop ) return Column.from_libcudf(move(result)) @@ -149,19 +143,15 @@ cpdef Column contains( cdef unique_ptr[column] result if ColumnOrScalar is Column: with nogil: - result = move( - cpp_find.contains( - input.view(), - target.view() - ) + result = cpp_find.contains( + input.view(), + target.view() ) elif ColumnOrScalar is Scalar: with nogil: - result = move( - cpp_find.contains( - input.view(), - dereference((target.c_obj.get())) - ) + result = cpp_find.contains( + input.view(), + dereference((target.c_obj.get())) ) else: raise ValueError(f"Invalid target {target}") @@ -204,19 +194,15 @@ cpdef Column starts_with( if ColumnOrScalar is Column: with nogil: - result = move( - cpp_find.starts_with( - input.view(), - target.view() - ) + result = cpp_find.starts_with( + input.view(), + target.view() ) elif ColumnOrScalar is Scalar: with nogil: - result = move( - cpp_find.starts_with( - input.view(), - dereference((target.c_obj.get())) - ) + result = cpp_find.starts_with( + input.view(), + dereference((target.c_obj.get())) ) else: raise ValueError(f"Invalid target {target}") @@ -256,19 +242,15 @@ cpdef Column ends_with( cdef unique_ptr[column] result if ColumnOrScalar is Column: with nogil: - result = move( - cpp_find.ends_with( - input.view(), - target.view() - ) + result = cpp_find.ends_with( + input.view(), + target.view() ) elif ColumnOrScalar is Scalar: with nogil: - result = move( - cpp_find.ends_with( - input.view(), - dereference((target.c_obj.get())) - ) + result = cpp_find.ends_with( + input.view(), + dereference((target.c_obj.get())) ) else: raise ValueError(f"Invalid target {target}") diff --git a/python/pylibcudf/pylibcudf/strings/find_multiple.pyx b/python/pylibcudf/pylibcudf/strings/find_multiple.pyx index 413fc1cb79d..672aa606bd0 100644 --- a/python/pylibcudf/pylibcudf/strings/find_multiple.pyx +++ b/python/pylibcudf/pylibcudf/strings/find_multiple.pyx @@ -29,11 +29,9 @@ cpdef Column find_multiple(Column input, Column targets): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_find_multiple.find_multiple( - input.view(), - targets.view() - ) + c_result = cpp_find_multiple.find_multiple( + input.view(), + targets.view() ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/findall.pyx b/python/pylibcudf/pylibcudf/strings/findall.pyx index 5212dc4594d..89fa4302824 100644 --- a/python/pylibcudf/pylibcudf/strings/findall.pyx +++ b/python/pylibcudf/pylibcudf/strings/findall.pyx @@ -30,11 +30,9 @@ cpdef Column findall(Column input, RegexProgram pattern): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_findall.findall( - input.view(), - pattern.c_obj.get()[0] - ) + c_result = cpp_findall.findall( + input.view(), + pattern.c_obj.get()[0] ) return Column.from_libcudf(move(c_result)) @@ -62,11 +60,9 @@ cpdef Column find_re(Column input, RegexProgram pattern): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_findall.find_re( - input.view(), - pattern.c_obj.get()[0] - ) + c_result = cpp_findall.find_re( + input.view(), + pattern.c_obj.get()[0] ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/padding.pyx b/python/pylibcudf/pylibcudf/strings/padding.pyx index 24daaaa3838..f6950eecf60 100644 --- a/python/pylibcudf/pylibcudf/strings/padding.pyx +++ b/python/pylibcudf/pylibcudf/strings/padding.pyx @@ -33,13 +33,11 @@ cpdef Column pad(Column input, size_type width, side_type side, str fill_char): cdef string c_fill_char = fill_char.encode("utf-8") with nogil: - c_result = move( - cpp_padding.pad( - input.view(), - width, - side, - c_fill_char, - ) + c_result = cpp_padding.pad( + input.view(), + width, + side, + c_fill_char, ) return Column.from_libcudf(move(c_result)) @@ -65,11 +63,9 @@ cpdef Column zfill(Column input, size_type width): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_padding.zfill( - input.view(), - width, - ) + c_result = cpp_padding.zfill( + input.view(), + width, ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/repeat.pyx b/python/pylibcudf/pylibcudf/strings/repeat.pyx index 5f627218f6e..fb2bb13c666 100644 --- a/python/pylibcudf/pylibcudf/strings/repeat.pyx +++ b/python/pylibcudf/pylibcudf/strings/repeat.pyx @@ -31,19 +31,15 @@ cpdef Column repeat_strings(Column input, ColumnorSizeType repeat_times): if ColumnorSizeType is Column: with nogil: - c_result = move( - cpp_repeat.repeat_strings( - input.view(), - repeat_times.view() - ) + c_result = cpp_repeat.repeat_strings( + input.view(), + repeat_times.view() ) elif ColumnorSizeType is size_type: with nogil: - c_result = move( - cpp_repeat.repeat_strings( - input.view(), - repeat_times - ) + c_result = cpp_repeat.repeat_strings( + input.view(), + repeat_times ) else: raise ValueError("repeat_times must be size_type or integer") diff --git a/python/pylibcudf/pylibcudf/strings/replace.pyx b/python/pylibcudf/pylibcudf/strings/replace.pyx index 9d0ebf4a814..6db7f04fcbb 100644 --- a/python/pylibcudf/pylibcudf/strings/replace.pyx +++ b/python/pylibcudf/pylibcudf/strings/replace.pyx @@ -55,12 +55,12 @@ cpdef Column replace( repl_str = (repl.c_obj.get()) with nogil: - c_result = move(cpp_replace( + c_result = cpp_replace( input.view(), target_str[0], repl_str[0], maxrepl, - )) + ) return Column.from_libcudf(move(c_result)) @@ -98,11 +98,11 @@ cpdef Column replace_multiple( cdef unique_ptr[column] c_result with nogil: - c_result = move(cpp_replace_multiple( + c_result = cpp_replace_multiple( input.view(), target.view(), repl.view(), - )) + ) return Column.from_libcudf(move(c_result)) @@ -151,11 +151,11 @@ cpdef Column replace_slice( cdef const string_scalar* scalar_str = (repl.c_obj.get()) with nogil: - c_result = move(cpp_replace_slice( + c_result = cpp_replace_slice( input.view(), scalar_str[0], start, stop - )) + ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/split/partition.pyx b/python/pylibcudf/pylibcudf/strings/split/partition.pyx index ecc959e65b0..0fb4f186c41 100644 --- a/python/pylibcudf/pylibcudf/strings/split/partition.pyx +++ b/python/pylibcudf/pylibcudf/strings/split/partition.pyx @@ -45,11 +45,9 @@ cpdef Table partition(Column input, Scalar delimiter=None): ) with nogil: - c_result = move( - cpp_partition.partition( - input.view(), - dereference(c_delimiter) - ) + c_result = cpp_partition.partition( + input.view(), + dereference(c_delimiter) ) return Table.from_libcudf(move(c_result)) @@ -85,11 +83,9 @@ cpdef Table rpartition(Column input, Scalar delimiter=None): ) with nogil: - c_result = move( - cpp_partition.rpartition( - input.view(), - dereference(c_delimiter) - ) + c_result = cpp_partition.rpartition( + input.view(), + dereference(c_delimiter) ) return Table.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/split/split.pyx b/python/pylibcudf/pylibcudf/strings/split/split.pyx index a7d7f39fc47..e3827f6645e 100644 --- a/python/pylibcudf/pylibcudf/strings/split/split.pyx +++ b/python/pylibcudf/pylibcudf/strings/split/split.pyx @@ -44,12 +44,10 @@ cpdef Table split(Column strings_column, Scalar delimiter, size_type maxsplit): ) with nogil: - c_result = move( - cpp_split.split( - strings_column.view(), - dereference(c_delimiter), - maxsplit, - ) + c_result = cpp_split.split( + strings_column.view(), + dereference(c_delimiter), + maxsplit, ) return Table.from_libcudf(move(c_result)) @@ -85,12 +83,10 @@ cpdef Table rsplit(Column strings_column, Scalar delimiter, size_type maxsplit): ) with nogil: - c_result = move( - cpp_split.rsplit( - strings_column.view(), - dereference(c_delimiter), - maxsplit, - ) + c_result = cpp_split.rsplit( + strings_column.view(), + dereference(c_delimiter), + maxsplit, ) return Table.from_libcudf(move(c_result)) @@ -124,12 +120,10 @@ cpdef Column split_record(Column strings, Scalar delimiter, size_type maxsplit): ) with nogil: - c_result = move( - cpp_split.split_record( - strings.view(), - dereference(c_delimiter), - maxsplit, - ) + c_result = cpp_split.split_record( + strings.view(), + dereference(c_delimiter), + maxsplit, ) return Column.from_libcudf(move(c_result)) @@ -165,12 +159,10 @@ cpdef Column rsplit_record(Column strings, Scalar delimiter, size_type maxsplit) ) with nogil: - c_result = move( - cpp_split.rsplit_record( - strings.view(), - dereference(c_delimiter), - maxsplit, - ) + c_result = cpp_split.rsplit_record( + strings.view(), + dereference(c_delimiter), + maxsplit, ) return Column.from_libcudf(move(c_result)) @@ -203,12 +195,10 @@ cpdef Table split_re(Column input, RegexProgram prog, size_type maxsplit): cdef unique_ptr[table] c_result with nogil: - c_result = move( - cpp_split.split_re( - input.view(), - prog.c_obj.get()[0], - maxsplit, - ) + c_result = cpp_split.split_re( + input.view(), + prog.c_obj.get()[0], + maxsplit, ) return Table.from_libcudf(move(c_result)) @@ -241,12 +231,10 @@ cpdef Table rsplit_re(Column input, RegexProgram prog, size_type maxsplit): cdef unique_ptr[table] c_result with nogil: - c_result = move( - cpp_split.rsplit_re( - input.view(), - prog.c_obj.get()[0], - maxsplit, - ) + c_result = cpp_split.rsplit_re( + input.view(), + prog.c_obj.get()[0], + maxsplit, ) return Table.from_libcudf(move(c_result)) @@ -278,12 +266,10 @@ cpdef Column split_record_re(Column input, RegexProgram prog, size_type maxsplit cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_split.split_record_re( - input.view(), - prog.c_obj.get()[0], - maxsplit, - ) + c_result = cpp_split.split_record_re( + input.view(), + prog.c_obj.get()[0], + maxsplit, ) return Column.from_libcudf(move(c_result)) @@ -315,12 +301,10 @@ cpdef Column rsplit_record_re(Column input, RegexProgram prog, size_type maxspli cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_split.rsplit_record_re( - input.view(), - prog.c_obj.get()[0], - maxsplit, - ) + c_result = cpp_split.rsplit_record_re( + input.view(), + prog.c_obj.get()[0], + maxsplit, ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/translate.pyx b/python/pylibcudf/pylibcudf/strings/translate.pyx index a62c7ec4528..d85da8e6cdd 100644 --- a/python/pylibcudf/pylibcudf/strings/translate.pyx +++ b/python/pylibcudf/pylibcudf/strings/translate.pyx @@ -62,11 +62,9 @@ cpdef Column translate(Column input, dict chars_table): ) with nogil: - c_result = move( - cpp_translate.translate( - input.view(), - c_chars_table - ) + c_result = cpp_translate.translate( + input.view(), + c_chars_table ) return Column.from_libcudf(move(c_result)) @@ -111,12 +109,10 @@ cpdef Column filter_characters( ) with nogil: - c_result = move( - cpp_translate.filter_characters( - input.view(), - c_characters_to_filter, - keep_characters, - dereference(c_replacement), - ) + c_result = cpp_translate.filter_characters( + input.view(), + c_characters_to_filter, + keep_characters, + dereference(c_replacement), ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/strings/wrap.pyx b/python/pylibcudf/pylibcudf/strings/wrap.pyx index 11e31f54eee..2ced250f837 100644 --- a/python/pylibcudf/pylibcudf/strings/wrap.pyx +++ b/python/pylibcudf/pylibcudf/strings/wrap.pyx @@ -32,11 +32,9 @@ cpdef Column wrap(Column input, size_type width): cdef unique_ptr[column] c_result with nogil: - c_result = move( - cpp_wrap.wrap( - input.view(), - width, - ) + c_result = cpp_wrap.wrap( + input.view(), + width, ) return Column.from_libcudf(move(c_result)) diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index 5f77b89a605..d0d6f2343d0 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -49,9 +49,7 @@ cdef class Table: calling libcudf algorithms, and should generally not be needed by users (even direct pylibcudf Cython users). """ - cdef vector[unique_ptr[column]] c_columns = move( - dereference(libcudf_tbl).release() - ) + cdef vector[unique_ptr[column]] c_columns = dereference(libcudf_tbl).release() cdef vector[unique_ptr[column]].size_type i return Table([ diff --git a/python/pylibcudf/pylibcudf/transform.pyx b/python/pylibcudf/pylibcudf/transform.pyx index 74134caeb78..bce9702752a 100644 --- a/python/pylibcudf/pylibcudf/transform.pyx +++ b/python/pylibcudf/pylibcudf/transform.pyx @@ -35,7 +35,7 @@ cpdef tuple[gpumemoryview, int] nans_to_nulls(Column input): cdef pair[unique_ptr[device_buffer], size_type] c_result with nogil: - c_result = move(cpp_transform.nans_to_nulls(input.view())) + c_result = cpp_transform.nans_to_nulls(input.view()) return ( gpumemoryview(DeviceBuffer.c_from_unique_ptr(move(c_result.first))), @@ -59,7 +59,7 @@ cpdef tuple[gpumemoryview, int] bools_to_mask(Column input): cdef pair[unique_ptr[device_buffer], size_type] c_result with nogil: - c_result = move(cpp_transform.bools_to_mask(input.view())) + c_result = cpp_transform.bools_to_mask(input.view()) return ( gpumemoryview(DeviceBuffer.c_from_unique_ptr(move(c_result.first))), @@ -88,7 +88,7 @@ cpdef Column mask_to_bools(Py_ssize_t bitmask, int begin_bit, int end_bit): cdef bitmask_type * bitmask_ptr = int_to_bitmask_ptr(bitmask) with nogil: - c_result = move(cpp_transform.mask_to_bools(bitmask_ptr, begin_bit, end_bit)) + c_result = cpp_transform.mask_to_bools(bitmask_ptr, begin_bit, end_bit) return Column.from_libcudf(move(c_result)) @@ -119,10 +119,8 @@ cpdef Column transform(Column input, str unary_udf, DataType output_type, bool i cdef bool c_is_ptx = is_ptx with nogil: - c_result = move( - cpp_transform.transform( - input.view(), c_unary_udf, output_type.c_obj, c_is_ptx - ) + c_result = cpp_transform.transform( + input.view(), c_unary_udf, output_type.c_obj, c_is_ptx ) return Column.from_libcudf(move(c_result)) @@ -144,7 +142,7 @@ cpdef tuple[Table, Column] encode(Table input): cdef pair[unique_ptr[table], unique_ptr[column]] c_result with nogil: - c_result = move(cpp_transform.encode(input.view())) + c_result = cpp_transform.encode(input.view()) return ( Table.from_libcudf(move(c_result.first)), @@ -172,7 +170,7 @@ cpdef Table one_hot_encode(Column input, Column categories): cdef Table owner_table with nogil: - c_result = move(cpp_transform.one_hot_encode(input.view(), categories.view())) + c_result = cpp_transform.one_hot_encode(input.view(), categories.view()) owner_table = Table( [Column.from_libcudf(move(c_result.first))] * c_result.second.num_columns() diff --git a/python/pylibcudf/pylibcudf/transpose.pyx b/python/pylibcudf/pylibcudf/transpose.pyx index a708f6cc37f..a24f937ced3 100644 --- a/python/pylibcudf/pylibcudf/transpose.pyx +++ b/python/pylibcudf/pylibcudf/transpose.pyx @@ -29,7 +29,7 @@ cpdef Table transpose(Table input_table): cdef Table owner_table with nogil: - c_result = move(cpp_transpose.transpose(input_table.view())) + c_result = cpp_transpose.transpose(input_table.view()) owner_table = Table( [Column.from_libcudf(move(c_result.first))] * c_result.second.num_columns() diff --git a/python/pylibcudf/pylibcudf/unary.pyx b/python/pylibcudf/pylibcudf/unary.pyx index 839360ef406..53e8c382b5e 100644 --- a/python/pylibcudf/pylibcudf/unary.pyx +++ b/python/pylibcudf/pylibcudf/unary.pyx @@ -34,7 +34,7 @@ cpdef Column unary_operation(Column input, unary_operator op): cdef unique_ptr[column] result with nogil: - result = move(cpp_unary.unary_operation(input.view(), op)) + result = cpp_unary.unary_operation(input.view(), op) return Column.from_libcudf(move(result)) @@ -57,7 +57,7 @@ cpdef Column is_null(Column input): cdef unique_ptr[column] result with nogil: - result = move(cpp_unary.is_null(input.view())) + result = cpp_unary.is_null(input.view()) return Column.from_libcudf(move(result)) @@ -80,7 +80,7 @@ cpdef Column is_valid(Column input): cdef unique_ptr[column] result with nogil: - result = move(cpp_unary.is_valid(input.view())) + result = cpp_unary.is_valid(input.view()) return Column.from_libcudf(move(result)) @@ -105,7 +105,7 @@ cpdef Column cast(Column input, DataType data_type): cdef unique_ptr[column] result with nogil: - result = move(cpp_unary.cast(input.view(), data_type.c_obj)) + result = cpp_unary.cast(input.view(), data_type.c_obj) return Column.from_libcudf(move(result)) @@ -128,7 +128,7 @@ cpdef Column is_nan(Column input): cdef unique_ptr[column] result with nogil: - result = move(cpp_unary.is_nan(input.view())) + result = cpp_unary.is_nan(input.view()) return Column.from_libcudf(move(result)) @@ -151,7 +151,7 @@ cpdef Column is_not_nan(Column input): cdef unique_ptr[column] result with nogil: - result = move(cpp_unary.is_not_nan(input.view())) + result = cpp_unary.is_not_nan(input.view()) return Column.from_libcudf(move(result)) From f1cbbcc1c8586bf68403f9abbc1a38fc527bdef4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Oct 2024 12:16:54 -0700 Subject: [PATCH 11/12] Reenable huge pages for arrow host copying (#17097) It is unclear whether the performance gains here are entirely from huge pages themselves or whether invoking madvise with huge pages is primarily serving to trigger an eager population of the pages (huge or not). We attempted to provide alternate flags to `madvise` like `MADV_WILLNEED` and that was not sufficient to recover performance, so either huge pages themselves are doing something special or specifying huge pages is causing `madvise` to trigger a page migration that no other flag does. In any case, this change returns us to the performance before the switch to the C data interface, and this code is lifted straight out of our old implementation so I am comfortable making use of it and knowing that it is not problematic. We should explore further optimizations in this direction, though. Resolves #17075. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Mark Harris (https://github.com/harrism) URL: https://github.com/rapidsai/cudf/pull/17097 --- cpp/src/interop/to_arrow_host.cu | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cpp/src/interop/to_arrow_host.cu b/cpp/src/interop/to_arrow_host.cu index 79fb7550044..8ec0904f1ba 100644 --- a/cpp/src/interop/to_arrow_host.cu +++ b/cpp/src/interop/to_arrow_host.cu @@ -44,6 +44,7 @@ #include #include #include +#include #include @@ -52,6 +53,30 @@ namespace detail { namespace { +/* + Enable Transparent Huge Pages (THP) for large (>4MB) allocations. + `buf` is returned untouched. + Enabling THP can improve performance of device-host memory transfers + significantly, see . +*/ +void enable_hugepage(ArrowBuffer* buffer) +{ + if (buffer->size_bytes < (1u << 22u)) { // Smaller than 4 MB + return; + } + +#ifdef MADV_HUGEPAGE + auto const pagesize = sysconf(_SC_PAGESIZE); + void* addr = const_cast(buffer->data); + auto length{static_cast(buffer->size_bytes)}; + if (std::align(pagesize, pagesize, addr, length)) { + // Intentionally not checking for errors that may be returned by older kernel versions; + // optimistically tries enabling huge pages. + madvise(addr, length, MADV_HUGEPAGE); + } +#endif +} + struct dispatch_to_arrow_host { cudf::column_view column; rmm::cuda_stream_view stream; @@ -62,6 +87,7 @@ struct dispatch_to_arrow_host { if (!column.has_nulls()) { return NANOARROW_OK; } NANOARROW_RETURN_NOT_OK(ArrowBitmapResize(bitmap, static_cast(column.size()), 0)); + enable_hugepage(&bitmap->buffer); CUDF_CUDA_TRY(cudaMemcpyAsync(bitmap->buffer.data, (column.offset() > 0) ? cudf::detail::copy_bitmask(column, stream, mr).data() @@ -76,6 +102,7 @@ struct dispatch_to_arrow_host { int populate_data_buffer(device_span input, ArrowBuffer* buffer) const { NANOARROW_RETURN_NOT_OK(ArrowBufferResize(buffer, input.size_bytes(), 1)); + enable_hugepage(buffer); CUDF_CUDA_TRY(cudaMemcpyAsync( buffer->data, input.data(), input.size_bytes(), cudaMemcpyDefault, stream.value())); return NANOARROW_OK; From b513df8a0e077f89fb50ee6e1b3a11d316b8a63a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 16 Oct 2024 16:08:22 -0500 Subject: [PATCH 12/12] Include timezone file path in error message (#17102) Resolves https://github.com/rapidsai/cudf/issues/8795. Also needed for https://github.com/rapidsai/cudf/pull/16998. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - David Wendt (https://github.com/davidwendt) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/17102 --- cpp/src/datetime/timezone.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/datetime/timezone.cpp b/cpp/src/datetime/timezone.cpp index a6b6cbbf0b5..2196ee97fee 100644 --- a/cpp/src/datetime/timezone.cpp +++ b/cpp/src/datetime/timezone.cpp @@ -138,7 +138,7 @@ struct timezone_file { std::filesystem::path{tzif_dir.value_or(tzif_system_directory)} / timezone_name; std::ifstream fin; fin.open(tz_filename, ios_base::in | ios_base::binary | ios_base::ate); - CUDF_EXPECTS(fin, "Failed to open the timezone file."); + CUDF_EXPECTS(fin, "Failed to open the timezone file '" + tz_filename.string() + "'"); auto const file_size = fin.tellg(); fin.seekg(0);