Move strings_udf code into cuDF

.gitattributes

@@ -1,5 +1,4 @@
 python/cudf/cudf/_version.py export-subst
-python/strings_udf/strings_udf/_version.py export-subst
 python/cudf_kafka/cudf_kafka/_version.py export-subst
 python/custreamz/custreamz/_version.py export-subst
 python/dask_cudf/dask_cudf/_version.py export-subst

.gitignore

@@ -36,8 +36,6 @@ python/cudf_kafka/*/_lib/**/*.cpp
 python/cudf_kafka/*/_lib/**/*.h
 python/custreamz/*/_lib/**/*.cpp
 python/custreamz/*/_lib/**/*.h
-python/strings_udf/strings_udf/_lib/*.cpp
-python/strings_udf/strings_udf/*.ptx
 .Python
 env/
 develop-eggs/

build.sh

@@ -17,7 +17,7 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd $(dirname $0); pwd)
 
-VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz strings_udf -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats"
+VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp  --show_depr_warn --ptds -h --build_metrics --incl_cache_stats"
 HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"<args>\\\"]
    clean                         - remove all existing build artifacts and configuration (start
                                    over)
@@ -335,14 +335,6 @@ if buildAll || hasArg cudf; then
     fi
 fi
 
-if buildAll || hasArg strings_udf; then
-
-    cd ${REPODIR}/python/strings_udf
-    python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1}
-    if [[ ${INSTALL_TARGET} != "" ]]; then
-        python setup.py install --single-version-externally-managed --record=record.txt  -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1}
-    fi
-fi
 
 # Build and install the dask_cudf Python package
 if buildAll || hasArg dask_cudf; then

ci/build_python.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -38,10 +38,5 @@ rapids-mamba-retry mambabuild \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/custreamz
 
-rapids-mamba-retry mambabuild \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  conda/recipes/strings_udf
 
 rapids-upload-conda-to-s3 python

ci/release/update-version.sh

@@ -40,8 +40,6 @@ sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g'
 # Python update
 sed_runner 's/'"cudf_version .*)"'/'"cudf_version ${NEXT_FULL_TAG})"'/g' python/cudf/CMakeLists.txt
 
-# Strings UDF update
-sed_runner 's/'"strings_udf_version .*)"'/'"strings_udf_version ${NEXT_FULL_TAG})"'/g' python/strings_udf/CMakeLists.txt
 
 # cpp libcudf_kafka update
 sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/libcudf_kafka/CMakeLists.txt

ci/test_python_other.sh

@@ -44,41 +44,5 @@ pytest \
   custreamz
 popd
 
-set -e
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  strings_udf
-set +e
-
-rapids-logger "pytest strings_udf"
-pushd python/strings_udf/strings_udf
-pytest \
-  --cache-clear \
-  --junitxml="${RAPIDS_TESTS_DIR}/junit-strings-udf.xml" \
-  --numprocesses=8 \
-  --dist=loadscope \
-  --cov-config=.coveragerc \
-  --cov=strings_udf \
-  --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/strings-udf-coverage.xml" \
-  --cov-report=term \
-  tests
-popd
-
-rapids-logger "pytest cudf with strings_udf"
-pushd python/cudf/cudf
-pytest \
-  --cache-clear \
-  --ignore="benchmarks" \
-  --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-strings-udf.xml" \
-  --numprocesses=8 \
-  --dist=loadscope \
-  --cov-config=../.coveragerc \
-  --cov=cudf \
-  --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-strings-udf-coverage.xml" \
-  --cov-report=term \
-  tests/test_udf_masked_ops.py
-popd
-
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}

conda/recipes/cudf/meta.yaml

@@ -72,6 +72,7 @@ requirements:
     - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}
     - nvtx >=0.2.1
     - packaging
+    - ptxcompiler >=0.7.0
     - cachetools
     - cubinlinker  # CUDA enhanced compatibility.
     - cuda-python >=11.7.1,<12.0

python/cudf/CMakeLists.txt

@@ -120,7 +120,7 @@ endif()
 rapids_cython_init()
 
 add_subdirectory(cudf/_lib)
-add_subdirectory(udf_cpp/groupby)
+add_subdirectory(udf_cpp)
 
 include(cmake/Modules/ProtobufHelpers.cmake)
 codegen_protoc(cudf/utils/metadata/orc_column_statistics.proto)

python/cudf/cudf/__init__.py

@@ -91,10 +91,9 @@
     # cuDF requires a stronger set of conditions than what is
     # checked by patch_numba_linker_if_needed due to the PTX
     # files needed for JIT Groupby Apply and string UDFs
-    from cudf.core.udf.groupby_utils import dev_func_ptx
-    from cudf.core.udf.utils import _setup_numba_linker
+    from cudf.core.udf.utils import _PTX_FILE, _setup_numba_linker
 
-    _setup_numba_linker(dev_func_ptx)
+    _setup_numba_linker(_PTX_FILE)
 
     del patch_numba_linker_if_needed
 

python/cudf/cudf/_lib/CMakeLists.txt

@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -46,6 +46,7 @@ set(cython_sources
     sort.pyx
     stream_compaction.pyx
     string_casting.pyx
+    strings_udf.pyx
     text.pyx
     transform.pyx
     transpose.pyx
@@ -61,6 +62,8 @@ rapids_cython_create_modules(
   LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cudf
 )
 
+target_link_libraries(strings_udf cudf_strings_udf)
+
 # TODO: Finding NumPy currently requires finding Development due to a bug in CMake. This bug was
 # fixed in https://gitlab.kitware.com/cmake/cmake/-/merge_requests/7410 and will be available in
 # CMake 3.24, so we can remove the Development component once we upgrade to CMake 3.24.

python/cudf/cudf/_lib/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 import numpy as np
 
 from . import (
@@ -33,6 +33,7 @@
     stream_compaction,
     string_casting,
     strings,
+    strings_udf,
     text,
     transpose,
     unary,

python/strings_udf/strings_udf/_lib/cpp/strings_udf.pxd → python/cudf/cudf/_lib/cpp/strings_udf.pxd

@@ -1,14 +1,15 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
 from libc.stdint cimport uint8_t, uint16_t
 from libcpp.memory cimport unique_ptr
 from libcpp.string cimport string
 from libcpp.vector cimport vector
 
+from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer
+
 from cudf._lib.cpp.column.column cimport column
 from cudf._lib.cpp.column.column_view cimport column_view
 from cudf._lib.cpp.types cimport size_type
-from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer
 
 
 cdef extern from "cudf/strings/udf/udf_string.hpp" namespace \

python/strings_udf/strings_udf/_lib/cudf_jit_udf.pyx → python/cudf/cudf/_lib/strings_udf.pyx

@@ -1,15 +1,25 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+
+from libc.stdint cimport uint8_t, uint16_t, uintptr_t
+
+from cudf._lib.cpp.strings_udf cimport (
+    get_character_cases_table as cpp_get_character_cases_table,
+    get_character_flags_table as cpp_get_character_flags_table,
+    get_special_case_mapping_table as cpp_get_special_case_mapping_table,
+)
+
+import numpy as np
 
 from libcpp.memory cimport unique_ptr
 from libcpp.utility cimport move
 
 from cudf.core.buffer import as_buffer
 
-from cudf._lib.column cimport Column
-from cudf._lib.cpp.column.column cimport column, column_view
 from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer
 
-from strings_udf._lib.cpp.strings_udf cimport (
+from cudf._lib.column cimport Column
+from cudf._lib.cpp.column.column cimport column, column_view
+from cudf._lib.cpp.strings_udf cimport (
     column_from_udf_string_array as cpp_column_from_udf_string_array,
     free_udf_string_array as cpp_free_udf_string_array,
     to_string_view_array as cpp_to_string_view_array,
@@ -39,3 +49,18 @@ def column_from_udf_string_array(DeviceBuffer d_buffer):
     result = Column.from_unique_ptr(move(c_result))
 
     return result
+
+
+def get_character_flags_table_ptr():
+    cdef const uint8_t* tbl_ptr = cpp_get_character_flags_table()
+    return np.uintp(<uintptr_t>tbl_ptr)
+
+
+def get_character_cases_table_ptr():
+    cdef const uint16_t* tbl_ptr = cpp_get_character_cases_table()
+    return np.uintp(<uintptr_t>tbl_ptr)
+
+
+def get_special_case_mapping_table_ptr():
+    cdef const void* tbl_ptr = cpp_get_special_case_mapping_table()
+    return np.uintp(<uintptr_t>tbl_ptr)

python/cudf/cudf/core/dataframe.py

@@ -4192,11 +4192,8 @@ def apply(
         For more information, see the `cuDF guide to user defined functions
         <https://docs.rapids.ai/api/cudf/stable/user_guide/guide-to-udfs.html>`__.
 
-        Support for use of string data within UDFs is provided through the
-        `strings_udf <https://anaconda.org/rapidsai-nightly/strings_udf>`__
-        RAPIDS library. Supported operations on strings include the subset of
-        functions and string methods that expect an input string but do not
-        return a string. Refer to caveats in the UDF guide referenced above.
+        Some string functions and methods are supported. Refer to the guide
+        to UDFs for details.
 
         Parameters
         ----------

python/cudf/cudf/core/indexed_frame.py

@@ -2112,7 +2112,6 @@ def _apply(self, func, kernel_getter, *args, **kwargs):
         """Apply `func` across the rows of the frame."""
         if kwargs:
             raise ValueError("UDFs using **kwargs are not yet supported.")
-
         try:
             kernel, retty = _compile_or_get(
                 self, func, args, kernel_getter=kernel_getter
@@ -2130,7 +2129,6 @@ def _apply(self, func, kernel_getter, *args, **kwargs):
         output_args = [(ans_col, ans_mask), len(self)]
         input_args = _get_input_args_from_frame(self)
         launch_args = output_args + input_args + list(args)
-
         try:
             kernel.forall(len(self))(*launch_args)
         except Exception as e:

python/cudf/cudf/core/series.py

@@ -2298,11 +2298,8 @@ def apply(self, func, convert_dtype=True, args=(), **kwargs):
         For more information, see the `cuDF guide to user defined functions
         <https://docs.rapids.ai/api/cudf/stable/user_guide/guide-to-udfs.html>`__.
 
-        Support for use of string data within UDFs is provided through the
-        `strings_udf <https://anaconda.org/rapidsai-nightly/strings_udf>`__
-        RAPIDS library. Supported operations on strings include the subset of
-        functions and string methods that expect an input string but do not
-        return a string. Refer to caveats in the UDF guide referenced above.
+        Some string functions and methods are supported. Refer to the guide
+        to UDFs for details.
 
         Parameters
         ----------