From f90ae52b52bf7aaa75925c45719da190f13711d8 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Wed, 22 Feb 2023 08:14:33 -0600 Subject: [PATCH] Move `strings_udf` code into cuDF (#12669) With the merge of https://github.com/rapidsai/cudf/pull/11452 we have the machinery to build and deploy PTX libraries of shim functions as part of cuDF's build process. With this there is no reason to keep the `strings_udf` code separate anymore. This PR removes the separate package and all of it's related CI plumbing as well as supports the strings feature by default, just like GroupBy. Authors: - https://github.com/brandon-b-miller - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12669 --- .gitattributes | 1 - .gitignore | 2 - build.sh | 10 +- ci/build_python.sh | 7 +- ci/release/update-version.sh | 2 - ci/test_python_other.sh | 36 - conda/recipes/cudf/meta.yaml | 1 + conda/recipes/strings_udf/build.sh | 4 - .../strings_udf/conda_build_config.yaml | 14 - conda/recipes/strings_udf/meta.yaml | 78 - .../source/user_guide/guide-to-udfs.ipynb | 192 +- python/cudf/CMakeLists.txt | 2 +- python/cudf/cudf/__init__.py | 5 +- python/cudf/cudf/_lib/CMakeLists.txt | 5 +- python/cudf/cudf/_lib/__init__.py | 3 +- .../cudf}/_lib/cpp/strings_udf.pxd | 5 +- .../cudf/_lib/strings_udf.pyx} | 33 +- python/cudf/cudf/core/dataframe.py | 7 +- python/cudf/cudf/core/indexed_frame.py | 2 - python/cudf/cudf/core/series.py | 7 +- python/cudf/cudf/core/udf/__init__.py | 86 +- python/cudf/cudf/core/udf/groupby_utils.py | 5 - python/cudf/cudf/core/udf/masked_lowering.py | 13 +- python/cudf/cudf/core/udf/masked_typing.py | 325 ++- python/cudf/cudf/core/udf/row_function.py | 27 +- python/cudf/cudf/core/udf/scalar_function.py | 7 +- python/cudf/cudf/core/udf/strings_lowering.py | 561 +++- python/cudf/cudf/core/udf/strings_typing.py | 313 ++- python/cudf/cudf/core/udf/utils.py | 188 +- .../cudf}/tests/test_string_udfs.py | 24 +- python/cudf/cudf/tests/test_udf_masked_ops.py | 5 +- python/cudf/setup.cfg | 3 +- .../cpp => cudf/udf_cpp}/CMakeLists.txt | 40 +- python/cudf/udf_cpp/groupby/CMakeLists.txt | 79 - python/cudf/udf_cpp/groupby/function.cu | 323 --- .../src/strings/udf => cudf/udf_cpp}/shim.cu | 310 ++- .../include/cudf/strings/udf/case.cuh | 2 +- .../include/cudf/strings/udf/char_types.cuh | 2 +- .../include/cudf/strings/udf/numeric.cuh | 2 +- .../strings}/include/cudf/strings/udf/pad.cuh | 2 +- .../include/cudf/strings/udf/replace.cuh | 2 +- .../include/cudf/strings/udf/search.cuh | 2 +- .../include/cudf/strings/udf/split.cuh | 2 +- .../include/cudf/strings/udf/starts_with.cuh | 2 +- .../include/cudf/strings/udf/strip.cuh | 2 +- .../include/cudf/strings/udf/udf_apis.hpp | 2 +- .../include/cudf/strings/udf/udf_string.cuh | 2 +- .../include/cudf/strings/udf/udf_string.hpp | 2 +- .../strings}/src/strings/udf/udf_apis.cu | 2 +- python/strings_udf/CMakeLists.txt | 46 - python/strings_udf/setup.cfg | 41 - python/strings_udf/setup.py | 82 - python/strings_udf/strings_udf/__init__.py | 44 - .../strings_udf/_lib/CMakeLists.txt | 21 - .../strings_udf/strings_udf/_lib/__init__.py | 0 .../strings_udf/_lib/cpp/__init__.pxd | 0 .../strings_udf/strings_udf/_lib/tables.pyx | 26 - python/strings_udf/strings_udf/_typing.py | 278 -- python/strings_udf/strings_udf/_version.py | 711 ------ python/strings_udf/strings_udf/lowering.py | 517 ---- python/strings_udf/versioneer.py | 2245 ----------------- 61 files changed, 1613 insertions(+), 5149 deletions(-) delete mode 100644 conda/recipes/strings_udf/build.sh delete mode 100644 conda/recipes/strings_udf/conda_build_config.yaml delete mode 100644 conda/recipes/strings_udf/meta.yaml rename python/{strings_udf/strings_udf => cudf/cudf}/_lib/cpp/strings_udf.pxd (96%) rename python/{strings_udf/strings_udf/_lib/cudf_jit_udf.pyx => cudf/cudf/_lib/strings_udf.pyx} (59%) rename python/{strings_udf/strings_udf => cudf/cudf}/tests/test_string_udfs.py (94%) rename python/{strings_udf/cpp => cudf/udf_cpp}/CMakeLists.txt (75%) delete mode 100644 python/cudf/udf_cpp/groupby/CMakeLists.txt delete mode 100644 python/cudf/udf_cpp/groupby/function.cu rename python/{strings_udf/cpp/src/strings/udf => cudf/udf_cpp}/shim.cu (55%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/case.cuh (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/char_types.cuh (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/numeric.cuh (97%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/pad.cuh (98%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/replace.cuh (97%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/search.cuh (97%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/split.cuh (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/starts_with.cuh (98%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/strip.cuh (98%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/udf_apis.hpp (97%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/udf_string.cuh (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/udf_string.hpp (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/src/strings/udf/udf_apis.cu (98%) delete mode 100644 python/strings_udf/CMakeLists.txt delete mode 100644 python/strings_udf/setup.cfg delete mode 100644 python/strings_udf/setup.py delete mode 100644 python/strings_udf/strings_udf/__init__.py delete mode 100644 python/strings_udf/strings_udf/_lib/CMakeLists.txt delete mode 100644 python/strings_udf/strings_udf/_lib/__init__.py delete mode 100644 python/strings_udf/strings_udf/_lib/cpp/__init__.pxd delete mode 100644 python/strings_udf/strings_udf/_lib/tables.pyx delete mode 100644 python/strings_udf/strings_udf/_typing.py delete mode 100644 python/strings_udf/strings_udf/_version.py delete mode 100644 python/strings_udf/strings_udf/lowering.py delete mode 100644 python/strings_udf/versioneer.py diff --git a/.gitattributes b/.gitattributes index ed8e5e1425a..fbfe7434d50 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,4 @@ python/cudf/cudf/_version.py export-subst -python/strings_udf/strings_udf/_version.py export-subst python/cudf_kafka/cudf_kafka/_version.py export-subst python/custreamz/custreamz/_version.py export-subst python/dask_cudf/dask_cudf/_version.py export-subst diff --git a/.gitignore b/.gitignore index 0f81dcb6f2b..2d83aad7712 100644 --- a/.gitignore +++ b/.gitignore @@ -36,8 +36,6 @@ python/cudf_kafka/*/_lib/**/*.cpp python/cudf_kafka/*/_lib/**/*.h python/custreamz/*/_lib/**/*.cpp python/custreamz/*/_lib/**/*.h -python/strings_udf/strings_udf/_lib/*.cpp -python/strings_udf/strings_udf/*.ptx .Python env/ develop-eggs/ diff --git a/build.sh b/build.sh index 22a62df7182..bee66d819b4 100755 --- a/build.sh +++ b/build.sh @@ -17,7 +17,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz strings_udf -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" +VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] clean - remove all existing build artifacts and configuration (start over) @@ -337,14 +337,6 @@ if buildAll || hasArg cudf; then fi fi -if buildAll || hasArg strings_udf; then - - cd ${REPODIR}/python/strings_udf - python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} - if [[ ${INSTALL_TARGET} != "" ]]; then - python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} - fi -fi # Build and install the dask_cudf Python package if buildAll || hasArg dask_cudf; then diff --git a/ci/build_python.sh b/ci/build_python.sh index 8756d759cc7..ec34d63b282 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. set -euo pipefail @@ -38,10 +38,5 @@ rapids-mamba-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/custreamz -rapids-mamba-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/strings_udf rapids-upload-conda-to-s3 python diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d2be7d5f222..15d81127450 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -40,8 +40,6 @@ sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' # Python update sed_runner 's/'"cudf_version .*)"'/'"cudf_version ${NEXT_FULL_TAG})"'/g' python/cudf/CMakeLists.txt -# Strings UDF update -sed_runner 's/'"strings_udf_version .*)"'/'"strings_udf_version ${NEXT_FULL_TAG})"'/g' python/strings_udf/CMakeLists.txt # cpp libcudf_kafka update sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/libcudf_kafka/CMakeLists.txt diff --git a/ci/test_python_other.sh b/ci/test_python_other.sh index b79cd44cdbe..25ed615df84 100755 --- a/ci/test_python_other.sh +++ b/ci/test_python_other.sh @@ -44,41 +44,5 @@ pytest \ custreamz popd -set -e -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - strings_udf -set +e - -rapids-logger "pytest strings_udf" -pushd python/strings_udf/strings_udf -pytest \ - --cache-clear \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-strings-udf.xml" \ - --numprocesses=8 \ - --dist=loadscope \ - --cov-config=.coveragerc \ - --cov=strings_udf \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/strings-udf-coverage.xml" \ - --cov-report=term \ - tests -popd - -rapids-logger "pytest cudf with strings_udf" -pushd python/cudf/cudf -pytest \ - --cache-clear \ - --ignore="benchmarks" \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-strings-udf.xml" \ - --numprocesses=8 \ - --dist=loadscope \ - --cov-config=../.coveragerc \ - --cov=cudf \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-strings-udf-coverage.xml" \ - --cov-report=term \ - tests/test_udf_masked_ops.py -popd - rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 0d5b5d16e08..27073eb323b 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -72,6 +72,7 @@ requirements: - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - nvtx >=0.2.1 - packaging + - ptxcompiler >=0.7.0 - cachetools - cubinlinker # CUDA enhanced compatibility. - cuda-python >=11.7.1,<12.0 diff --git a/conda/recipes/strings_udf/build.sh b/conda/recipes/strings_udf/build.sh deleted file mode 100644 index 2de1325347b..00000000000 --- a/conda/recipes/strings_udf/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh strings_udf diff --git a/conda/recipes/strings_udf/conda_build_config.yaml b/conda/recipes/strings_udf/conda_build_config.yaml deleted file mode 100644 index 4feac647e8c..00000000000 --- a/conda/recipes/strings_udf/conda_build_config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -c_compiler_version: - - 9 - -cxx_compiler_version: - - 9 - -sysroot_version: - - "2.17" - -cmake_version: - - ">=3.23.1,!=3.25.0" - -cuda_compiler: - - nvcc diff --git a/conda/recipes/strings_udf/meta.yaml b/conda/recipes/strings_udf/meta.yaml deleted file mode 100644 index 93316a92c22..00000000000 --- a/conda/recipes/strings_udf/meta.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: strings_udf - version: {{ version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda_{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - PARALLEL_LEVEL - - CMAKE_GENERATOR - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=strings-udf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=strings-udf-linux64 # [linux64] - - SCCACHE_BUCKET - - SCCACHE_REGION - - SCCACHE_IDLE_TIMEOUT - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - ignore_run_exports: - # libcudf's run_exports pinning is looser than we would like - - libcudf - ignore_run_exports_from: - - {{ compiler('cuda') }} - -requirements: - build: - - cmake {{ cmake_version }} - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ compiler('cuda') }} {{ cuda_version }} - - ninja - - sysroot_{{ target_platform }} {{ sysroot_version }} - host: - - python - - cython >=0.29,<0.30 - - scikit-build >=0.13.1 - - setuptools - - numba >=0.54 - - libcudf ={{ version }} - - cudf ={{ version }} - - cudatoolkit ={{ cuda_version }} - run: - - python - - typing_extensions - - numba >=0.54 - - numpy - - libcudf ={{ version }} - - cudf ={{ version }} - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - cachetools - - ptxcompiler >=0.7.0 # CUDA enhanced compatibility. See https://github.com/rapidsai/ptxcompiler - -test: - requires: - - cudatoolkit ={{ cuda_version }} - imports: - - strings_udf - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: strings_udf library diff --git a/docs/cudf/source/user_guide/guide-to-udfs.ipynb b/docs/cudf/source/user_guide/guide-to-udfs.ipynb index e6ef5e86416..943fc980a31 100644 --- a/docs/cudf/source/user_guide/guide-to-udfs.ipynb +++ b/docs/cudf/source/user_guide/guide-to-udfs.ipynb @@ -336,30 +336,13 @@ "### String data" ] }, - { - "cell_type": "markdown", - "id": "c0980218", - "metadata": {}, - "source": [ - "Experimental support for a subset of string functionality is available for `apply` through the [strings_udf](https://anaconda.org/rapidsai-nightly/strings_udf) package, which is separately installable from RAPIDS conda channels. The following several cells show an example of its usage if present. If it is not present, this notebook may be restarted after a conda install of `strings_udf` to proceed through the example." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "b91a60f7", - "metadata": {}, - "outputs": [], - "source": [ - "string_udfs_enabled = cudf.core.udf._STRING_UDFS_ENABLED" - ] - }, { "cell_type": "markdown", "id": "81762aea", "metadata": {}, "source": [ - "Currently, the following string operations are provided through `strings_udf`:\",\n", + "Experimental support for a subset of string functionality is available for `apply`. The following string operations are currently supported:\n", + "\n", "- `str.count`\n", "- `str.startswith`\n", "- `str.endswith`\n", @@ -382,12 +365,12 @@ "- `upper`\n", "- `lower`\n", "- `+` (string concatenation)\n", - "- `replace" + "- `replace`" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "id": "d7d1abd7", "metadata": {}, "outputs": [], @@ -397,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "id": "e8538ba0", "metadata": {}, "outputs": [], @@ -416,7 +399,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "id": "23524fd8", "metadata": {}, "outputs": [ @@ -432,10 +415,8 @@ } ], "source": [ - "# Requires strings_udf package, or will raise a compilation error\n", - "if string_udfs_enabled:\n", - " result = sr.apply(f)\n", - " print(result)" + "result = sr.apply(f)\n", + "print(result)" ] }, { @@ -456,14 +437,13 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "id": "b26ec6dc", "metadata": {}, "outputs": [], "source": [ - "if string_udfs_enabled:\n", - " from strings_udf import set_malloc_heap_size\n", - " set_malloc_heap_size(int(2e9))" + "from cudf.core.udf.utils import set_malloc_heap_size\n", + "set_malloc_heap_size(int(2e9))" ] }, { @@ -487,7 +467,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "id": "732434f6", "metadata": {}, "outputs": [], @@ -497,7 +477,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "id": "4f5997e5", "metadata": {}, "outputs": [], @@ -523,7 +503,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "id": "ea6008a6", "metadata": {}, "outputs": [], @@ -543,7 +523,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "id": "183a82ed", "metadata": {}, "outputs": [ @@ -623,7 +603,7 @@ "4 979 982 1011 9790.0" ] }, - "execution_count": 20, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -672,7 +652,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "id": "73653918", "metadata": {}, "outputs": [], @@ -691,7 +671,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "id": "077feb75", "metadata": {}, "outputs": [ @@ -747,7 +727,7 @@ "2 3 6" ] }, - "execution_count": 22, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -770,7 +750,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "id": "091e39e1", "metadata": {}, "outputs": [ @@ -783,7 +763,7 @@ "dtype: int64" ] }, - "execution_count": 23, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -802,7 +782,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "id": "bd345fab", "metadata": {}, "outputs": [ @@ -815,7 +795,7 @@ "dtype: object" ] }, - "execution_count": 24, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -842,7 +822,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "id": "b70f4b3b", "metadata": {}, "outputs": [ @@ -894,7 +874,7 @@ "2 3" ] }, - "execution_count": 25, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -913,7 +893,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "id": "0313c8df", "metadata": {}, "outputs": [ @@ -926,7 +906,7 @@ "dtype: int64" ] }, - "execution_count": 26, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -945,7 +925,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "id": "96a7952a", "metadata": {}, "outputs": [ @@ -1001,7 +981,7 @@ "2 3 1" ] }, - "execution_count": 27, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -1024,7 +1004,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "id": "e0815f60", "metadata": {}, "outputs": [ @@ -1037,7 +1017,7 @@ "dtype: int64" ] }, - "execution_count": 28, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1056,7 +1036,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "id": "495efd14", "metadata": {}, "outputs": [ @@ -1112,7 +1092,7 @@ "2 3 3.14" ] }, - "execution_count": 29, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -1130,7 +1110,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "id": "678b0b5a", "metadata": {}, "outputs": [ @@ -1143,7 +1123,7 @@ "dtype: float64" ] }, - "execution_count": 30, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -1175,7 +1155,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "id": "acf48d56", "metadata": {}, "outputs": [ @@ -1227,7 +1207,7 @@ "2 5" ] }, - "execution_count": 31, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -1248,7 +1228,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "id": "78a98172", "metadata": {}, "outputs": [ @@ -1261,7 +1241,7 @@ "dtype: float64" ] }, - "execution_count": 32, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -1280,7 +1260,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "id": "142c30a9", "metadata": {}, "outputs": [ @@ -1348,7 +1328,7 @@ "2 3 6 4 8 6" ] }, - "execution_count": 33, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -1369,7 +1349,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "id": "fee9198a", "metadata": {}, "outputs": [ @@ -1382,7 +1362,7 @@ "dtype: float64" ] }, - "execution_count": 34, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -1409,17 +1389,7 @@ }, { "cell_type": "code", - "execution_count": 35, - "id": "3bfb117a", - "metadata": {}, - "outputs": [], - "source": [ - "string_udfs_enabled = cudf.core.udf._STRING_UDFS_ENABLED" - ] - }, - { - "cell_type": "code", - "execution_count": 36, + "execution_count": 34, "id": "cccd59f7", "metadata": {}, "outputs": [ @@ -1475,7 +1445,7 @@ "2 Example 3" ] }, - "execution_count": 36, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1490,7 +1460,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 35, "id": "35737fd9", "metadata": {}, "outputs": [], @@ -1507,7 +1477,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 36, "id": "4ede4d5b", "metadata": {}, "outputs": [ @@ -1523,9 +1493,8 @@ } ], "source": [ - "if string_udfs_enabled:\n", - " result = str_df.apply(f, axis=1)\n", - " print(result)" + "result = str_df.apply(f, axis=1)\n", + "print(result)" ] }, { @@ -1548,7 +1517,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 37, "id": "90cbcd85", "metadata": {}, "outputs": [], @@ -1579,7 +1548,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 38, "id": "e782daff", "metadata": {}, "outputs": [ @@ -1651,7 +1620,7 @@ "2 3 6 4 8 6 9.0" ] }, - "execution_count": 40, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -1685,7 +1654,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 39, "id": "befd8333", "metadata": {}, "outputs": [ @@ -1759,7 +1728,7 @@ "4 979 982 1011" ] }, - "execution_count": 41, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -1786,7 +1755,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 40, "id": "d1f3dcaf", "metadata": {}, "outputs": [ @@ -1866,7 +1835,7 @@ "4 979 982 1011 1961.0" ] }, - "execution_count": 42, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -1901,7 +1870,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 41, "id": "6bc6aea3", "metadata": {}, "outputs": [ @@ -1917,7 +1886,7 @@ "dtype: float64" ] }, - "execution_count": 43, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -1929,7 +1898,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 42, "id": "a4c31df1", "metadata": {}, "outputs": [ @@ -1939,7 +1908,7 @@ "Rolling [window=3,min_periods=3,center=False]" ] }, - "execution_count": 44, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -1959,7 +1928,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 43, "id": "eb5a081b", "metadata": {}, "outputs": [], @@ -1985,7 +1954,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 44, "id": "ddec3263", "metadata": {}, "outputs": [ @@ -2001,7 +1970,7 @@ "dtype: float64" ] }, - "execution_count": 46, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -2020,7 +1989,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 45, "id": "8b61094a", "metadata": {}, "outputs": [ @@ -2088,7 +2057,7 @@ "4 59.0 59.0" ] }, - "execution_count": 47, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -2102,7 +2071,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 46, "id": "bb8c3019", "metadata": {}, "outputs": [ @@ -2200,7 +2169,7 @@ "9 100.0 100.0" ] }, - "execution_count": 48, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -2224,7 +2193,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 47, "id": "3dc272ab", "metadata": {}, "outputs": [ @@ -2304,7 +2273,7 @@ "4 -0.970850 False Sarah 0.342905" ] }, - "execution_count": 49, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -2316,7 +2285,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 48, "id": "c0578e0a", "metadata": {}, "outputs": [], @@ -2334,7 +2303,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 49, "id": "19f0f7fe", "metadata": {}, "outputs": [], @@ -2363,7 +2332,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 50, "id": "c43426c3", "metadata": {}, "outputs": [ @@ -2494,7 +2463,7 @@ "9 -0.725581 True George 0.405245 0.271319" ] }, - "execution_count": 52, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -2526,7 +2495,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 51, "id": "aa6a8509", "metadata": {}, "outputs": [ @@ -2536,7 +2505,7 @@ "array([ 1., 2., 3., 4., 10.])" ] }, - "execution_count": 53, + "execution_count": 51, "metadata": {}, "output_type": "execute_result" } @@ -2559,7 +2528,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 52, "id": "0bb8bf93", "metadata": {}, "outputs": [ @@ -2574,7 +2543,7 @@ "dtype: int32" ] }, - "execution_count": 54, + "execution_count": 52, "metadata": {}, "output_type": "execute_result" } @@ -2601,7 +2570,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 53, "id": "ce60b639", "metadata": {}, "outputs": [ @@ -2611,7 +2580,7 @@ "array([ 5., 10., 15., 20., 50.])" ] }, - "execution_count": 55, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -2655,6 +2624,7 @@ "- CuPy NDArrays\n", "- Numba DeviceNDArrays\n", "- Generalized NA UDFs\n", + "- String UDFs\n", "\n", "\n", "For more information please see the [cuDF](https://docs.rapids.ai/api/cudf/nightly/), [Numba.cuda](https://numba.readthedocs.io/en/stable/cuda/index.html), and [CuPy](https://docs-cupy.chainer.org/en/stable/) documentation." @@ -2677,7 +2647,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.15" + "version": "3.10.8" } }, "nbformat": 4, diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 7210d398c6b..c528eb69575 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -111,7 +111,7 @@ endif() rapids_cython_init() add_subdirectory(cudf/_lib) -add_subdirectory(udf_cpp/groupby) +add_subdirectory(udf_cpp) include(cmake/Modules/ProtobufHelpers.cmake) codegen_protoc(cudf/utils/metadata/orc_column_statistics.proto) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index b86fb72d955..05f61ee4f5a 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -91,10 +91,9 @@ # cuDF requires a stronger set of conditions than what is # checked by patch_numba_linker_if_needed due to the PTX # files needed for JIT Groupby Apply and string UDFs - from cudf.core.udf.groupby_utils import dev_func_ptx - from cudf.core.udf.utils import _setup_numba_linker + from cudf.core.udf.utils import _PTX_FILE, _setup_numba_linker - _setup_numba_linker(dev_func_ptx) + _setup_numba_linker(_PTX_FILE) del patch_numba_linker_if_needed diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 90d2d59cc66..4b785563484 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -46,6 +46,7 @@ set(cython_sources sort.pyx stream_compaction.pyx string_casting.pyx + strings_udf.pyx text.pyx transform.pyx transpose.pyx @@ -61,6 +62,8 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cudf ) +target_link_libraries(strings_udf cudf_strings_udf) + # TODO: Finding NumPy currently requires finding Development due to a bug in CMake. This bug was # fixed in https://gitlab.kitware.com/cmake/cmake/-/merge_requests/7410 and will be available in # CMake 3.24, so we can remove the Development component once we upgrade to CMake 3.24. diff --git a/python/cudf/cudf/_lib/__init__.py b/python/cudf/cudf/_lib/__init__.py index 8ecb9a57426..b101db9a744 100644 --- a/python/cudf/cudf/_lib/__init__.py +++ b/python/cudf/cudf/_lib/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import numpy as np from . import ( @@ -33,6 +33,7 @@ stream_compaction, string_casting, strings, + strings_udf, text, transpose, unary, diff --git a/python/strings_udf/strings_udf/_lib/cpp/strings_udf.pxd b/python/cudf/cudf/_lib/cpp/strings_udf.pxd similarity index 96% rename from python/strings_udf/strings_udf/_lib/cpp/strings_udf.pxd rename to python/cudf/cudf/_lib/cpp/strings_udf.pxd index b3bf6465db6..7d45bc858f5 100644 --- a/python/strings_udf/strings_udf/_lib/cpp/strings_udf.pxd +++ b/python/cudf/cudf/_lib/cpp/strings_udf.pxd @@ -1,14 +1,15 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. from libc.stdint cimport uint8_t, uint16_t from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector +from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer + from cudf._lib.cpp.column.column cimport column from cudf._lib.cpp.column.column_view cimport column_view from cudf._lib.cpp.types cimport size_type -from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer cdef extern from "cudf/strings/udf/udf_string.hpp" namespace \ diff --git a/python/strings_udf/strings_udf/_lib/cudf_jit_udf.pyx b/python/cudf/cudf/_lib/strings_udf.pyx similarity index 59% rename from python/strings_udf/strings_udf/_lib/cudf_jit_udf.pyx rename to python/cudf/cudf/_lib/strings_udf.pyx index bf459f22c16..3d465f9172b 100644 --- a/python/strings_udf/strings_udf/_lib/cudf_jit_udf.pyx +++ b/python/cudf/cudf/_lib/strings_udf.pyx @@ -1,15 +1,25 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +from libc.stdint cimport uint8_t, uint16_t, uintptr_t + +from cudf._lib.cpp.strings_udf cimport ( + get_character_cases_table as cpp_get_character_cases_table, + get_character_flags_table as cpp_get_character_flags_table, + get_special_case_mapping_table as cpp_get_special_case_mapping_table, +) + +import numpy as np from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from cudf.core.buffer import as_buffer -from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column, column_view from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer -from strings_udf._lib.cpp.strings_udf cimport ( +from cudf._lib.column cimport Column +from cudf._lib.cpp.column.column cimport column, column_view +from cudf._lib.cpp.strings_udf cimport ( column_from_udf_string_array as cpp_column_from_udf_string_array, free_udf_string_array as cpp_free_udf_string_array, to_string_view_array as cpp_to_string_view_array, @@ -39,3 +49,18 @@ def column_from_udf_string_array(DeviceBuffer d_buffer): result = Column.from_unique_ptr(move(c_result)) return result + + +def get_character_flags_table_ptr(): + cdef const uint8_t* tbl_ptr = cpp_get_character_flags_table() + return np.uintp(tbl_ptr) + + +def get_character_cases_table_ptr(): + cdef const uint16_t* tbl_ptr = cpp_get_character_cases_table() + return np.uintp(tbl_ptr) + + +def get_special_case_mapping_table_ptr(): + cdef const void* tbl_ptr = cpp_get_special_case_mapping_table() + return np.uintp(tbl_ptr) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index c32ab19fb69..d43621d3d36 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -4192,11 +4192,8 @@ def apply( For more information, see the `cuDF guide to user defined functions `__. - Support for use of string data within UDFs is provided through the - `strings_udf `__ - RAPIDS library. Supported operations on strings include the subset of - functions and string methods that expect an input string but do not - return a string. Refer to caveats in the UDF guide referenced above. + Some string functions and methods are supported. Refer to the guide + to UDFs for details. Parameters ---------- diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 25c89c34040..43277fb55ff 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -2112,7 +2112,6 @@ def _apply(self, func, kernel_getter, *args, **kwargs): """Apply `func` across the rows of the frame.""" if kwargs: raise ValueError("UDFs using **kwargs are not yet supported.") - try: kernel, retty = _compile_or_get( self, func, args, kernel_getter=kernel_getter @@ -2130,7 +2129,6 @@ def _apply(self, func, kernel_getter, *args, **kwargs): output_args = [(ans_col, ans_mask), len(self)] input_args = _get_input_args_from_frame(self) launch_args = output_args + input_args + list(args) - try: kernel.forall(len(self))(*launch_args) except Exception as e: diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 43b2f38ca46..60655c5a6f9 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -2327,11 +2327,8 @@ def apply(self, func, convert_dtype=True, args=(), **kwargs): For more information, see the `cuDF guide to user defined functions `__. - Support for use of string data within UDFs is provided through the - `strings_udf `__ - RAPIDS library. Supported operations on strings include the subset of - functions and string methods that expect an input string but do not - return a string. Refer to caveats in the UDF guide referenced above. + Some string functions and methods are supported. Refer to the guide + to UDFs for details. Parameters ---------- diff --git a/python/cudf/cudf/core/udf/__init__.py b/python/cudf/cudf/core/udf/__init__.py index 06ceecf0a35..85d454652b7 100644 --- a/python/cudf/cudf/core/udf/__init__.py +++ b/python/cudf/cudf/core/udf/__init__.py @@ -1,81 +1,9 @@ # Copyright (c) 2022-2023, NVIDIA CORPORATION. - -from functools import lru_cache - -from numba import types -from numba.cuda.cudaimpl import lower as cuda_lower - -from cudf.core.dtypes import dtype -from cudf.core.udf import api, row_function, utils -from cudf.utils.dtypes import STRING_TYPES - -from . import groupby_lowering, groupby_typing, masked_lowering, masked_typing - -_units = ["ns", "ms", "us", "s"] -_datetime_cases = {types.NPDatetime(u) for u in _units} -_timedelta_cases = {types.NPTimedelta(u) for u in _units} -_supported_masked_types = ( - types.integer_domain - | types.real_domain - | _datetime_cases - | _timedelta_cases - | {types.boolean} +from . import ( + groupby_lowering, + groupby_typing, + masked_lowering, + masked_typing, + strings_lowering, + strings_typing, ) -_STRING_UDFS_ENABLED = False -cudf_str_dtype = dtype(str) - - -try: - import strings_udf - from strings_udf import ptxpath - - if ptxpath: - utils.ptx_files.append(ptxpath) - - from strings_udf._lib.cudf_jit_udf import ( - column_from_udf_string_array, - column_to_string_view_array, - ) - from strings_udf._typing import ( - str_view_arg_handler, - string_view, - udf_string, - ) - - from . import strings_typing # isort: skip - from . import strings_lowering # isort: skip - - cuda_lower(api.Masked, string_view, types.boolean)( - masked_lowering.masked_constructor - ) - utils.JIT_SUPPORTED_TYPES |= STRING_TYPES - _supported_masked_types |= {string_view, udf_string} - - @lru_cache(maxsize=None) - def set_initial_malloc_heap_size(): - strings_udf.set_malloc_heap_size() - - def column_to_string_view_array_init_heap(col): - # lazily allocate heap only when a string needs to be returned - set_initial_malloc_heap_size() - return column_to_string_view_array(col) - - utils.launch_arg_getters[ - cudf_str_dtype - ] = column_to_string_view_array_init_heap - utils.output_col_getters[cudf_str_dtype] = column_from_udf_string_array - utils.masked_array_types[cudf_str_dtype] = string_view - row_function.itemsizes[cudf_str_dtype] = string_view.size_bytes - - utils.arg_handlers.append(str_view_arg_handler) - - masked_typing.MASKED_INIT_MAP[udf_string] = udf_string - - _STRING_UDFS_ENABLED = True - -except ImportError as e: - # allow cuDF to work without strings_udf - pass - -masked_typing._register_masked_constructor_typing(_supported_masked_types) -masked_lowering._register_masked_constructor_lowering(_supported_masked_types) diff --git a/python/cudf/cudf/core/udf/groupby_utils.py b/python/cudf/cudf/core/udf/groupby_utils.py index a1174835db9..dc31cf43292 100644 --- a/python/cudf/cudf/core/udf/groupby_utils.py +++ b/python/cudf/cudf/core/udf/groupby_utils.py @@ -1,6 +1,5 @@ # Copyright (c) 2022-2023, NVIDIA CORPORATION. -import os import cupy as cp import numpy as np @@ -22,16 +21,12 @@ from cudf.core.udf.utils import ( _get_extensionty_size, _get_kernel, - _get_ptx_file, _get_udf_return_type, _supported_cols_from_frame, _supported_dtypes_from_frame, ) from cudf.utils.utils import _cudf_nvtx_annotate -dev_func_ptx = _get_ptx_file(os.path.dirname(__file__), "function_") -cudf.core.udf.utils.ptx_files.append(dev_func_ptx) - def _get_frame_groupby_type(dtype, index_dtype): """ diff --git a/python/cudf/cudf/core/udf/masked_lowering.py b/python/cudf/cudf/core/udf/masked_lowering.py index 37f3117e756..74b414ce36a 100644 --- a/python/cudf/cudf/core/udf/masked_lowering.py +++ b/python/cudf/cudf/core/udf/masked_lowering.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import operator @@ -18,7 +18,11 @@ comparison_ops, unary_ops, ) -from cudf.core.udf.masked_typing import MaskedType, NAType +from cudf.core.udf.masked_typing import ( + MaskedType, + NAType, + _supported_masked_types, +) @cuda_lowering_registry.lower_constant(NAType) @@ -381,9 +385,8 @@ def masked_constructor(context, builder, sig, args): return masked._getvalue() -def _register_masked_constructor_lowering(supported_masked_types): - for ty in supported_masked_types: - lower_builtin(api.Masked, ty, types.boolean)(masked_constructor) +for ty in _supported_masked_types: + lower_builtin(api.Masked, ty, types.boolean)(masked_constructor) # Allows us to make an instance of MaskedType a global variable diff --git a/python/cudf/cudf/core/udf/masked_typing.py b/python/cudf/cudf/core/udf/masked_typing.py index 7baf2d585e2..30caa641701 100644 --- a/python/cudf/cudf/core/udf/masked_typing.py +++ b/python/cudf/cudf/core/udf/masked_typing.py @@ -1,7 +1,6 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import operator -from typing import Any, Dict from numba import types from numba.core.extending import ( @@ -27,6 +26,18 @@ comparison_ops, unary_ops, ) +from cudf.core.udf.strings_typing import ( + StringView, + UDFString, + bool_binary_funcs, + id_unary_funcs, + int_binary_funcs, + size_type, + string_return_attrs, + string_unary_funcs, + string_view, + udf_string, +) from cudf.utils.dtypes import ( DATETIME_TYPES, NUMERIC_TYPES, @@ -34,19 +45,33 @@ TIMEDELTA_TYPES, ) +SUPPORTED_NUMPY_TYPES = ( + NUMERIC_TYPES | DATETIME_TYPES | TIMEDELTA_TYPES | STRING_TYPES +) +supported_type_str = "\n".join(sorted(list(SUPPORTED_NUMPY_TYPES) + ["bool"])) + +_units = ["ns", "ms", "us", "s"] +_datetime_cases = {types.NPDatetime(u) for u in _units} +_timedelta_cases = {types.NPTimedelta(u) for u in _units} +_supported_masked_types = ( + types.integer_domain + | types.real_domain + | _datetime_cases + | _timedelta_cases + | {types.boolean} + | {string_view, udf_string} +) + + SUPPORTED_NUMBA_TYPES = ( types.Number, types.Boolean, types.NPDatetime, types.NPTimedelta, + StringView, + UDFString, ) -SUPPORTED_NUMPY_TYPES = ( - NUMERIC_TYPES | DATETIME_TYPES | TIMEDELTA_TYPES | STRING_TYPES -) -supported_type_str = "\n".join(sorted(list(SUPPORTED_NUMPY_TYPES) + ["bool"])) -MASKED_INIT_MAP: Dict[Any, Any] = {} - def _format_error_string(err): """ @@ -56,31 +81,19 @@ def _format_error_string(err): def _type_to_masked_type(t): - result = MASKED_INIT_MAP.get(t) - if result is None: - if isinstance(t, SUPPORTED_NUMBA_TYPES): - return t - else: - # Unsupported Dtype. Numba tends to print out the type info - # for whatever operands and operation failed to type and then - # output its own error message. Putting the message in the repr - # then is one way of getting the true cause to the user - err = _format_error_string( - "Unsupported MaskedType. This is usually caused by " - "attempting to use a column of unsupported dtype in a UDF. " - f"Supported dtypes are:\n{supported_type_str}" - ) - return types.Poison(err) + if isinstance(t, SUPPORTED_NUMBA_TYPES): + return t else: - return result - - -MASKED_INIT_MAP[types.pyobject] = types.Poison( - _format_error_string( - "strings_udf library required for usage of string dtypes " - "inside user defined functions." - ) -) + # Unsupported Dtype. Numba tends to print out the type info + # for whatever operands and operation failed to type and then + # output its own error message. Putting the message in the repr + # then is one way of getting the true cause to the user + err = _format_error_string( + "Unsupported MaskedType. This is usually caused by " + "attempting to use a column of unsupported dtype in a UDF. " + f"Supported dtypes are:\n{supported_type_str}" + ) + return types.Poison(err) # Masked scalars of all types @@ -169,30 +182,30 @@ def typeof_masked(val, c): # Implemented typing for Masked(value, valid) - the construction of a Masked # type in a kernel. -def _register_masked_constructor_typing(supported_masked_types): - class MaskedConstructor(ConcreteTemplate): - key = api.Masked - cases = [ - nb_signature(MaskedType(t), t, types.boolean) - for t in supported_masked_types - ] - - cuda_decl_registry.register(MaskedConstructor) - - # Typing for `api.Masked` - @cuda_decl_registry.register_attr - class ClassesTemplate(AttributeTemplate): - key = types.Module(api) - - def resolve_Masked(self, mod): - return types.Function(MaskedConstructor) - - # Registration of the global is also needed for Numba to type api.Masked - cuda_decl_registry.register_global(api, types.Module(api)) - # For typing bare Masked (as in `from .api import Masked` - cuda_decl_registry.register_global( - api.Masked, types.Function(MaskedConstructor) - ) +@cuda_decl_registry.register +class MaskedConstructor(ConcreteTemplate): + key = api.Masked + cases = [ + nb_signature(MaskedType(t), t, types.boolean) + for t in _supported_masked_types + ] + + +# Typing for `api.Masked` +@cuda_decl_registry.register_attr +class ClassesTemplate(AttributeTemplate): + key = types.Module(api) + + def resolve_Masked(self, mod): + return types.Function(MaskedConstructor) + + +# Registration of the global is also needed for Numba to type api.Masked +cuda_decl_registry.register_global(api, types.Module(api)) +# For typing bare Masked (as in `from .api import Masked` +cuda_decl_registry.register_global( + api.Masked, types.Function(MaskedConstructor) +) # Provide access to `m.value` and `m.valid` in a kernel for a Masked `m`. @@ -423,3 +436,203 @@ def generic(self, args, kws): for unary_op in unary_ops: cuda_decl_registry.register_global(unary_op)(MaskedScalarUnaryOp) + + +# Strings functions and utilities +def _is_valid_string_arg(ty): + return ( + isinstance(ty, MaskedType) and isinstance(ty.value_type, StringView) + ) or isinstance(ty, types.StringLiteral) + + +def register_masked_string_function(func): + """ + Helper function wrapping numba's low level extension API. Provides + the boilerplate needed to associate a signature with a function or + operator to be overloaded. + """ + + def deco(generic): + class MaskedStringFunction(AbstractTemplate): + pass + + MaskedStringFunction.generic = generic + cuda_decl_registry.register_global(func)(MaskedStringFunction) + + return deco + + +@register_masked_string_function(len) +def len_typing(self, args, kws): + if isinstance(args[0], MaskedType) and isinstance( + args[0].value_type, StringView + ): + return nb_signature(MaskedType(size_type), args[0]) + elif isinstance(args[0], types.StringLiteral) and len(args) == 1: + return nb_signature(size_type, args[0]) + + +@register_masked_string_function(operator.add) +def concat_typing(self, args, kws): + if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): + return nb_signature( + MaskedType(udf_string), + MaskedType(string_view), + MaskedType(string_view), + ) + + +@register_masked_string_function(operator.contains) +def contains_typing(self, args, kws): + if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): + return nb_signature( + MaskedType(types.boolean), + MaskedType(string_view), + MaskedType(string_view), + ) + + +class MaskedStringViewCmpOp(AbstractTemplate): + """ + return the boolean result of `cmpop` between to strings + since the typing is the same for every comparison operator, + we can reuse this class for all of them. + """ + + def generic(self, args, kws): + if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): + return nb_signature( + MaskedType(types.boolean), + MaskedType(string_view), + MaskedType(string_view), + ) + + +for op in comparison_ops: + cuda_decl_registry.register_global(op)(MaskedStringViewCmpOp) + + +def create_masked_binary_attr(attrname, retty): + """ + Helper function wrapping numba's low level extension API. Provides + the boilerplate needed to register a binary function of two masked + string objects as an attribute of one, e.g. `string.func(other)`. + """ + + class MaskedStringViewBinaryAttr(AbstractTemplate): + key = attrname + + def generic(self, args, kws): + return nb_signature( + MaskedType(retty), MaskedType(string_view), recvr=self.this + ) + + def attr(self, mod): + return types.BoundFunction( + MaskedStringViewBinaryAttr, + MaskedType(string_view), + ) + + return attr + + +def create_masked_unary_attr(attrname, retty): + """ + Helper function wrapping numba's low level extension API. Provides + the boilerplate needed to register a unary function of a masked + string object as an attribute, e.g. `string.func()`. + """ + + class MaskedStringViewIdentifierAttr(AbstractTemplate): + key = attrname + + def generic(self, args, kws): + return nb_signature(MaskedType(retty), recvr=self.this) + + def attr(self, mod): + return types.BoundFunction( + MaskedStringViewIdentifierAttr, + MaskedType(string_view), + ) + + return attr + + +class MaskedStringViewCount(AbstractTemplate): + key = "MaskedType.count" + + def generic(self, args, kws): + return nb_signature( + MaskedType(size_type), MaskedType(string_view), recvr=self.this + ) + + +class MaskedStringViewReplace(AbstractTemplate): + key = "MaskedType.replace" + + def generic(self, args, kws): + return nb_signature( + MaskedType(udf_string), + MaskedType(string_view), + MaskedType(string_view), + recvr=self.this, + ) + + +class MaskedStringViewAttrs(AttributeTemplate): + key = MaskedType(string_view) + + def resolve_replace(self, mod): + return types.BoundFunction( + MaskedStringViewReplace, MaskedType(string_view) + ) + + def resolve_count(self, mod): + return types.BoundFunction( + MaskedStringViewCount, MaskedType(string_view) + ) + + def resolve_value(self, mod): + return string_view + + def resolve_valid(self, mod): + return types.boolean + + +# Build attributes for `MaskedType(string_view)` +for func in bool_binary_funcs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_binary_attr(f"MaskedType.{func}", types.boolean), + ) + +for func in int_binary_funcs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_binary_attr(f"MaskedType.{func}", size_type), + ) + +for func in string_return_attrs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_binary_attr(f"MaskedType.{func}", udf_string), + ) + +for func in id_unary_funcs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_unary_attr(f"MaskedType.{func}", types.boolean), + ) + +for func in string_unary_funcs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_unary_attr(f"MaskedType.{func}", udf_string), + ) + +cuda_decl_registry.register_attr(MaskedStringViewAttrs) diff --git a/python/cudf/cudf/core/udf/row_function.py b/python/cudf/cudf/core/udf/row_function.py index 8d887a37706..bfb04e38e7d 100644 --- a/python/cudf/cudf/core/udf/row_function.py +++ b/python/cudf/cudf/core/udf/row_function.py @@ -1,6 +1,5 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import math -from typing import Any, Dict import numpy as np from numba import cuda @@ -9,6 +8,7 @@ from cudf.core.udf.api import Masked, pack_return from cudf.core.udf.masked_typing import MaskedType +from cudf.core.udf.strings_typing import string_view from cudf.core.udf.templates import ( masked_input_initializer_template, row_initializer_template, @@ -18,6 +18,7 @@ from cudf.core.udf.utils import ( _all_dtypes_from_frame, _construct_signature, + _get_extensionty_size, _get_kernel, _get_udf_return_type, _mask_get, @@ -25,8 +26,6 @@ _supported_dtypes_from_frame, ) -itemsizes: Dict[Any, int] = {} - def _get_frame_row_type(dtype): """ @@ -47,8 +46,12 @@ def _get_frame_row_type(dtype): offset = 0 sizes = [ - itemsizes.get(val[0], val[0].itemsize) for val in dtype.fields.values() + _get_extensionty_size(string_view) + if val[0] == np.dtype("O") + else val[0].itemsize + for val in dtype.fields.values() ] + for i, (name, info) in enumerate(dtype.fields.items()): # *info* consists of the element dtype, its offset from the beginning # of the record, and an optional "title" containing metadata. @@ -56,7 +59,13 @@ def _get_frame_row_type(dtype): # instead, we compute the correct offset based on the masked type. elemdtype = info[0] title = info[2] if len(info) == 3 else None - ty = numpy_support.from_dtype(elemdtype) + + ty = ( + # columns of dtype string start life as string_view + string_view + if elemdtype == np.dtype("O") + else numpy_support.from_dtype(elemdtype) + ) infos = { "type": MaskedType(ty), "offset": offset, @@ -65,7 +74,11 @@ def _get_frame_row_type(dtype): fields.append((name, infos)) # increment offset by itemsize plus one byte for validity - itemsize = itemsizes.get(elemdtype, elemdtype.itemsize) + itemsize = ( + _get_extensionty_size(string_view) + if elemdtype == np.dtype("O") + else elemdtype.itemsize + ) offset += itemsize + 1 # Align the next member of the struct to be a multiple of the diff --git a/python/cudf/cudf/core/udf/scalar_function.py b/python/cudf/cudf/core/udf/scalar_function.py index 31599f4151e..ff7fad3fb82 100644 --- a/python/cudf/cudf/core/udf/scalar_function.py +++ b/python/cudf/cudf/core/udf/scalar_function.py @@ -1,10 +1,11 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from numba import cuda from numba.np import numpy_support from cudf.core.udf.api import Masked, pack_return from cudf.core.udf.masked_typing import MaskedType +from cudf.core.udf.strings_typing import string_view from cudf.core.udf.templates import ( masked_input_initializer_template, scalar_kernel_template, @@ -48,7 +49,9 @@ def f(x, c, k): def _get_scalar_kernel(sr, func, args): - sr_type = MaskedType(numpy_support.from_dtype(sr.dtype)) + sr_type = MaskedType( + string_view if sr.dtype == "O" else numpy_support.from_dtype(sr.dtype) + ) scalar_return_type = _get_udf_return_type(sr_type, func, args) sig = _construct_signature(sr, scalar_return_type, args=args) diff --git a/python/cudf/cudf/core/udf/strings_lowering.py b/python/cudf/cudf/core/udf/strings_lowering.py index ec956cdd65d..00905f72cda 100644 --- a/python/cudf/cudf/core/udf/strings_lowering.py +++ b/python/cudf/cudf/core/udf/strings_lowering.py @@ -1,38 +1,521 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import operator +from functools import partial -from numba import types +from numba import cuda, types from numba.core import cgutils +from numba.core.datamodel import default_manager from numba.core.typing import signature as nb_signature -from numba.cuda.cudaimpl import lower as cuda_lower - -from strings_udf._typing import size_type, string_view, udf_string -from strings_udf.lowering import ( - contains_impl, - count_impl, - endswith_impl, - find_impl, - isalnum_impl, - isalpha_impl, - isdecimal_impl, - isdigit_impl, - islower_impl, - isspace_impl, - istitle_impl, - isupper_impl, - len_impl, - lower_impl, - lstrip_impl, - replace_impl, - rfind_impl, - rstrip_impl, - startswith_impl, - strip_impl, - upper_impl, +from numba.cuda.cudadrv import nvvm +from numba.cuda.cudaimpl import ( + lower as cuda_lower, + registry as cuda_lowering_registry, ) +from cudf._lib.strings_udf import ( + get_character_cases_table_ptr, + get_character_flags_table_ptr, + get_special_case_mapping_table_ptr, +) from cudf.core.udf.masked_typing import MaskedType +from cudf.core.udf.strings_typing import size_type, string_view, udf_string + +_STR_VIEW_PTR = types.CPointer(string_view) +_UDF_STRING_PTR = types.CPointer(udf_string) + + +# CUDA function declarations +# read-only (input is a string_view, output is a fixed with type) +_string_view_len = cuda.declare_device("len", size_type(_STR_VIEW_PTR)) +_concat_string_view = cuda.declare_device( + "concat", types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR) +) + +_string_view_replace = cuda.declare_device( + "replace", + types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), +) + + +def _declare_binary_func(lhs, rhs, out, name): + # Declare a binary function + return cuda.declare_device( + name, + out(lhs, rhs), + ) + + +def _declare_strip_func(name): + return cuda.declare_device( + name, size_type(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR) + ) + + +# A binary function of the form f(string, string) -> bool +_declare_bool_str_str_func = partial( + _declare_binary_func, _STR_VIEW_PTR, _STR_VIEW_PTR, types.boolean +) + +_declare_size_type_str_str_func = partial( + _declare_binary_func, _STR_VIEW_PTR, _STR_VIEW_PTR, size_type +) + +_string_view_contains = _declare_bool_str_str_func("contains") +_string_view_eq = _declare_bool_str_str_func("eq") +_string_view_ne = _declare_bool_str_str_func("ne") +_string_view_ge = _declare_bool_str_str_func("ge") +_string_view_le = _declare_bool_str_str_func("le") +_string_view_gt = _declare_bool_str_str_func("gt") +_string_view_lt = _declare_bool_str_str_func("lt") +_string_view_startswith = _declare_bool_str_str_func("startswith") +_string_view_endswith = _declare_bool_str_str_func("endswith") +_string_view_find = _declare_size_type_str_str_func("find") +_string_view_rfind = _declare_size_type_str_str_func("rfind") +_string_view_contains = _declare_bool_str_str_func("contains") +_string_view_strip = _declare_strip_func("strip") +_string_view_lstrip = _declare_strip_func("lstrip") +_string_view_rstrip = _declare_strip_func("rstrip") + + +# A binary function of the form f(string, int) -> bool +_declare_bool_str_int_func = partial( + _declare_binary_func, _STR_VIEW_PTR, types.int64, types.boolean +) + + +def _declare_upper_or_lower(func): + return cuda.declare_device( + func, + types.void( + _UDF_STRING_PTR, + _STR_VIEW_PTR, + types.uintp, + types.uintp, + types.uintp, + ), + ) + + +_string_view_isdigit = _declare_bool_str_int_func("pyisdigit") +_string_view_isalnum = _declare_bool_str_int_func("pyisalnum") +_string_view_isalpha = _declare_bool_str_int_func("pyisalpha") +_string_view_isdecimal = _declare_bool_str_int_func("pyisdecimal") +_string_view_isnumeric = _declare_bool_str_int_func("pyisnumeric") +_string_view_isspace = _declare_bool_str_int_func("pyisspace") +_string_view_isupper = _declare_bool_str_int_func("pyisupper") +_string_view_islower = _declare_bool_str_int_func("pyislower") +_string_view_istitle = _declare_bool_str_int_func("pyistitle") +_string_view_upper = _declare_upper_or_lower("upper") +_string_view_lower = _declare_upper_or_lower("lower") + + +_string_view_count = cuda.declare_device( + "pycount", + size_type(_STR_VIEW_PTR, _STR_VIEW_PTR), +) + + +# casts +@cuda_lowering_registry.lower_cast(types.StringLiteral, string_view) +def cast_string_literal_to_string_view(context, builder, fromty, toty, val): + """ + Cast a literal to a string_view + """ + # create an empty string_view + sv = cgutils.create_struct_proxy(string_view)(context, builder) + + # set the empty strview data pointer to point to the literal value + s = context.insert_const_string(builder.module, fromty.literal_value) + sv.data = context.insert_addrspace_conv( + builder, s, nvvm.ADDRSPACE_CONSTANT + ) + sv.length = context.get_constant(size_type, len(fromty.literal_value)) + sv.bytes = context.get_constant( + size_type, len(fromty.literal_value.encode("UTF-8")) + ) + + return sv._getvalue() + + +@cuda_lowering_registry.lower_cast(string_view, udf_string) +def cast_string_view_to_udf_string(context, builder, fromty, toty, val): + sv_ptr = builder.alloca(default_manager[fromty].get_value_type()) + udf_str_ptr = builder.alloca(default_manager[toty].get_value_type()) + builder.store(val, sv_ptr) + _ = context.compile_internal( + builder, + call_create_udf_string_from_string_view, + nb_signature(types.void, _STR_VIEW_PTR, types.CPointer(udf_string)), + (sv_ptr, udf_str_ptr), + ) + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + + return result._getvalue() + + +# utilities +_create_udf_string_from_string_view = cuda.declare_device( + "udf_string_from_string_view", + types.void(types.CPointer(string_view), types.CPointer(udf_string)), +) + + +def call_create_udf_string_from_string_view(sv, udf_str): + _create_udf_string_from_string_view(sv, udf_str) + + +# String function implementations +def call_len_string_view(st): + return _string_view_len(st) + + +@cuda_lower(len, string_view) +def len_impl(context, builder, sig, args): + sv_ptr = builder.alloca(args[0].type) + builder.store(args[0], sv_ptr) + result = context.compile_internal( + builder, + call_len_string_view, + nb_signature(size_type, _STR_VIEW_PTR), + (sv_ptr,), + ) + + return result + + +def call_concat_string_view(result, lhs, rhs): + return _concat_string_view(result, lhs, rhs) + + +@cuda_lower(operator.add, string_view, string_view) +def concat_impl(context, builder, sig, args): + lhs_ptr = builder.alloca(args[0].type) + rhs_ptr = builder.alloca(args[1].type) + builder.store(args[0], lhs_ptr) + builder.store(args[1], rhs_ptr) + + udf_str_ptr = builder.alloca(default_manager[udf_string].get_value_type()) + _ = context.compile_internal( + builder, + call_concat_string_view, + types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), + (udf_str_ptr, lhs_ptr, rhs_ptr), + ) + + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + return result._getvalue() + + +def call_string_view_replace(result, src, to_replace, replacement): + return _string_view_replace(result, src, to_replace, replacement) + + +@cuda_lower("StringView.replace", string_view, string_view, string_view) +def replace_impl(context, builder, sig, args): + src_ptr = builder.alloca(args[0].type) + to_replace_ptr = builder.alloca(args[1].type) + replacement_ptr = builder.alloca(args[2].type) + + builder.store(args[0], src_ptr) + builder.store(args[1], to_replace_ptr), + builder.store(args[2], replacement_ptr) + + udf_str_ptr = builder.alloca(default_manager[udf_string].get_value_type()) + + _ = context.compile_internal( + builder, + call_string_view_replace, + types.void( + _UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR + ), + (udf_str_ptr, src_ptr, to_replace_ptr, replacement_ptr), + ) + + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + return result._getvalue() + + +def create_binary_string_func(binary_func, retty): + """ + Provide a wrapper around numba's low-level extension API which + produces the boilerplate needed to implement a binary function + of two strings. + """ + + def deco(cuda_func): + @cuda_lower(binary_func, string_view, string_view) + def binary_func_impl(context, builder, sig, args): + lhs_ptr = builder.alloca(args[0].type) + rhs_ptr = builder.alloca(args[1].type) + builder.store(args[0], lhs_ptr) + builder.store(args[1], rhs_ptr) + + # these conditional statements should compile out + if retty != udf_string: + # binary function of two strings yielding a fixed-width type + # example: str.startswith(other) -> bool + # shim functions can return the value through nb_retval + result = context.compile_internal( + builder, + cuda_func, + nb_signature(retty, _STR_VIEW_PTR, _STR_VIEW_PTR), + (lhs_ptr, rhs_ptr), + ) + return result + else: + # binary function of two strings yielding a new string + # example: str.strip(other) -> str + # shim functions can not return a struct due to C linkage + # so we create a new udf_string and pass a pointer to it + # for the shim function to write the output to. The return + # value of compile_internal is therefore discarded (although + # this may change in the future if we need to return error + # codes, for instance). + udf_str_ptr = builder.alloca( + default_manager[udf_string].get_value_type() + ) + _ = context.compile_internal( + builder, + cuda_func, + size_type(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), + (udf_str_ptr, lhs_ptr, rhs_ptr), + ) + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + return result._getvalue() + + return binary_func_impl + + return deco + + +@create_binary_string_func(operator.contains, types.boolean) +def contains_impl(st, substr): + return _string_view_contains(st, substr) + + +@create_binary_string_func(operator.eq, types.boolean) +def eq_impl(st, rhs): + return _string_view_eq(st, rhs) + + +@create_binary_string_func(operator.ne, types.boolean) +def ne_impl(st, rhs): + return _string_view_ne(st, rhs) + + +@create_binary_string_func(operator.ge, types.boolean) +def ge_impl(st, rhs): + return _string_view_ge(st, rhs) + + +@create_binary_string_func(operator.le, types.boolean) +def le_impl(st, rhs): + return _string_view_le(st, rhs) + + +@create_binary_string_func(operator.gt, types.boolean) +def gt_impl(st, rhs): + return _string_view_gt(st, rhs) + + +@create_binary_string_func(operator.lt, types.boolean) +def lt_impl(st, rhs): + return _string_view_lt(st, rhs) + + +@create_binary_string_func("StringView.strip", udf_string) +def strip_impl(result, to_strip, strip_char): + return _string_view_strip(result, to_strip, strip_char) + + +@create_binary_string_func("StringView.lstrip", udf_string) +def lstrip_impl(result, to_strip, strip_char): + return _string_view_lstrip(result, to_strip, strip_char) + + +@create_binary_string_func("StringView.rstrip", udf_string) +def rstrip_impl(result, to_strip, strip_char): + return _string_view_rstrip(result, to_strip, strip_char) + + +@create_binary_string_func("StringView.startswith", types.boolean) +def startswith_impl(sv, substr): + return _string_view_startswith(sv, substr) + + +@create_binary_string_func("StringView.endswith", types.boolean) +def endswith_impl(sv, substr): + return _string_view_endswith(sv, substr) + + +@create_binary_string_func("StringView.count", size_type) +def count_impl(st, substr): + return _string_view_count(st, substr) + + +@create_binary_string_func("StringView.find", size_type) +def find_impl(sv, substr): + return _string_view_find(sv, substr) + + +@create_binary_string_func("StringView.rfind", size_type) +def rfind_impl(sv, substr): + return _string_view_rfind(sv, substr) + + +def create_unary_identifier_func(id_func): + """ + Provide a wrapper around numba's low-level extension API which + produces the boilerplate needed to implement a unary function + of a string. + """ + + def deco(cuda_func): + @cuda_lower(id_func, string_view) + def id_func_impl(context, builder, sig, args): + str_ptr = builder.alloca(args[0].type) + builder.store(args[0], str_ptr) + + # Lookup table required for conversion functions + # must be resolved at runtime after context initialization, + # therefore cannot be a global variable + tbl_ptr = context.get_constant( + types.uintp, get_character_flags_table_ptr() + ) + result = context.compile_internal( + builder, + cuda_func, + nb_signature(types.boolean, _STR_VIEW_PTR, types.uintp), + (str_ptr, tbl_ptr), + ) + + return result + + return id_func_impl + + return deco + + +def create_upper_or_lower(id_func): + """ + Provide a wrapper around numba's low-level extension API which + produces the boilerplate needed to implement either the upper + or lower attrs of a string view. + """ + + def deco(cuda_func): + @cuda_lower(id_func, string_view) + def id_func_impl(context, builder, sig, args): + str_ptr = builder.alloca(args[0].type) + builder.store(args[0], str_ptr) + + # Lookup table required for conversion functions + # must be resolved at runtime after context initialization, + # therefore cannot be a global variable + flags_tbl_ptr = context.get_constant( + types.uintp, get_character_flags_table_ptr() + ) + cases_tbl_ptr = context.get_constant( + types.uintp, get_character_cases_table_ptr() + ) + special_tbl_ptr = context.get_constant( + types.uintp, get_special_case_mapping_table_ptr() + ) + udf_str_ptr = builder.alloca( + default_manager[udf_string].get_value_type() + ) + + _ = context.compile_internal( + builder, + cuda_func, + types.void( + _UDF_STRING_PTR, + _STR_VIEW_PTR, + types.uintp, + types.uintp, + types.uintp, + ), + ( + udf_str_ptr, + str_ptr, + flags_tbl_ptr, + cases_tbl_ptr, + special_tbl_ptr, + ), + ) + + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + return result._getvalue() + + return id_func_impl + + return deco + + +@create_upper_or_lower("StringView.upper") +def upper_impl(result, st, flags, cases, special): + return _string_view_upper(result, st, flags, cases, special) + + +@create_upper_or_lower("StringView.lower") +def lower_impl(result, st, flags, cases, special): + return _string_view_lower(result, st, flags, cases, special) + + +@create_unary_identifier_func("StringView.isdigit") +def isdigit_impl(st, tbl): + return _string_view_isdigit(st, tbl) + + +@create_unary_identifier_func("StringView.isalnum") +def isalnum_impl(st, tbl): + return _string_view_isalnum(st, tbl) + + +@create_unary_identifier_func("StringView.isalpha") +def isalpha_impl(st, tbl): + return _string_view_isalpha(st, tbl) + + +@create_unary_identifier_func("StringView.isnumeric") +def isnumeric_impl(st, tbl): + return _string_view_isnumeric(st, tbl) + + +@create_unary_identifier_func("StringView.isdecimal") +def isdecimal_impl(st, tbl): + return _string_view_isdecimal(st, tbl) + + +@create_unary_identifier_func("StringView.isspace") +def isspace_impl(st, tbl): + return _string_view_isspace(st, tbl) + + +@create_unary_identifier_func("StringView.isupper") +def isupper_impl(st, tbl): + return _string_view_isupper(st, tbl) + + +@create_unary_identifier_func("StringView.islower") +def islower_impl(st, tbl): + return _string_view_islower(st, tbl) + + +@create_unary_identifier_func("StringView.istitle") +def istitle_impl(st, tbl): + return _string_view_istitle(st, tbl) @cuda_lower(len, MaskedType(string_view)) @@ -85,7 +568,7 @@ def masked_string_view_replace_impl(context, builder, sig, args): return ret._getvalue() -def create_binary_string_func(op, cuda_func, retty): +def create_masked_binary_string_func(op, cuda_func, retty): """ Provide a wrapper around numba's low-level extension API which produces the boilerplate needed to implement a binary function @@ -165,19 +648,23 @@ def upper_or_lower_impl(context, builder, sig, args): cuda_lower(op, MaskedType(string_view))(upper_or_lower_impl) -create_binary_string_func("MaskedType.strip", strip_impl, udf_string) -create_binary_string_func("MaskedType.lstrip", lstrip_impl, udf_string) -create_binary_string_func("MaskedType.rstrip", rstrip_impl, udf_string) -create_binary_string_func( +create_masked_binary_string_func("MaskedType.strip", strip_impl, udf_string) +create_masked_binary_string_func("MaskedType.lstrip", lstrip_impl, udf_string) +create_masked_binary_string_func("MaskedType.rstrip", rstrip_impl, udf_string) +create_masked_binary_string_func( "MaskedType.startswith", startswith_impl, types.boolean, ) -create_binary_string_func("MaskedType.endswith", endswith_impl, types.boolean) -create_binary_string_func("MaskedType.find", find_impl, size_type) -create_binary_string_func("MaskedType.rfind", rfind_impl, size_type) -create_binary_string_func("MaskedType.count", count_impl, size_type) -create_binary_string_func(operator.contains, contains_impl, types.boolean) +create_masked_binary_string_func( + "MaskedType.endswith", endswith_impl, types.boolean +) +create_masked_binary_string_func("MaskedType.find", find_impl, size_type) +create_masked_binary_string_func("MaskedType.rfind", rfind_impl, size_type) +create_masked_binary_string_func("MaskedType.count", count_impl, size_type) +create_masked_binary_string_func( + operator.contains, contains_impl, types.boolean +) create_masked_unary_identifier_func("MaskedType.isalnum", isalnum_impl) diff --git a/python/cudf/cudf/core/udf/strings_typing.py b/python/cudf/cudf/core/udf/strings_typing.py index e373b8b018d..7c8429d9997 100644 --- a/python/cudf/cudf/core/udf/strings_typing.py +++ b/python/cudf/cudf/core/udf/strings_typing.py @@ -1,226 +1,273 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import operator +import numpy as np from numba import types +from numba.core.extending import models, register_model from numba.core.typing import signature as nb_signature from numba.core.typing.templates import AbstractTemplate, AttributeTemplate from numba.cuda.cudadecl import registry as cuda_decl_registry -from strings_udf._typing import ( - StringView, - bool_binary_funcs, - id_unary_funcs, - int_binary_funcs, - size_type, - string_return_attrs, - string_unary_funcs, - string_view, - udf_string, -) +import rmm -from cudf.core.udf import masked_typing -from cudf.core.udf._ops import comparison_ops -from cudf.core.udf.masked_typing import MaskedType +# libcudf size_type +size_type = types.int32 -masked_typing.MASKED_INIT_MAP[types.pyobject] = string_view -masked_typing.MASKED_INIT_MAP[string_view] = string_view +# String object definitions +class UDFString(types.Type): -def _is_valid_string_arg(ty): - return ( - isinstance(ty, MaskedType) and isinstance(ty.value_type, StringView) - ) or isinstance(ty, types.StringLiteral) + np_dtype = np.dtype("object") + def __init__(self): + super().__init__(name="udf_string") -def register_string_function(func): - """ - Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to associate a signature with a function or - operator to be overloaded. - """ + @property + def return_type(self): + return self - def deco(generic): - class MaskedStringFunction(AbstractTemplate): - pass - MaskedStringFunction.generic = generic - cuda_decl_registry.register_global(func)(MaskedStringFunction) +class StringView(types.Type): - return deco + np_dtype = np.dtype("object") + def __init__(self): + super().__init__(name="string_view") -@register_string_function(len) -def len_typing(self, args, kws): - if isinstance(args[0], MaskedType) and isinstance( - args[0].value_type, StringView - ): - return nb_signature(MaskedType(size_type), args[0]) - elif isinstance(args[0], types.StringLiteral) and len(args) == 1: - return nb_signature(size_type, args[0]) + @property + def return_type(self): + return UDFString() -@register_string_function(operator.add) -def concat_typing(self, args, kws): - if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): - return nb_signature( - MaskedType(udf_string), - MaskedType(string_view), - MaskedType(string_view), - ) +@register_model(StringView) +class stringview_model(models.StructModel): + # from string_view.hpp: + _members = ( + # const char* _data{} + # Pointer to device memory contain char array for this string + ("data", types.CPointer(types.char)), + # size_type _bytes{}; + # Number of bytes in _data for this string + ("bytes", size_type), + # mutable size_type _length{}; + # Number of characters in this string (computed) + ("length", size_type), + ) + def __init__(self, dmm, fe_type): + super().__init__(dmm, fe_type, self._members) -@register_string_function(operator.contains) -def contains_typing(self, args, kws): - if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): - return nb_signature( - MaskedType(types.boolean), - MaskedType(string_view), - MaskedType(string_view), - ) + +@register_model(UDFString) +class udf_string_model(models.StructModel): + # from udf_string.hpp: + # private: + # char* m_data{}; + # cudf::size_type m_bytes{}; + # cudf::size_type m_size{}; + + _members = ( + ("m_data", types.CPointer(types.char)), + ("m_bytes", size_type), + ("m_size", size_type), + ) + + def __init__(self, dmm, fe_type): + super().__init__(dmm, fe_type, self._members) -class MaskedStringViewCmpOp(AbstractTemplate): +any_string_ty = (StringView, UDFString, types.StringLiteral) +string_view = StringView() +udf_string = UDFString() + + +class StrViewArgHandler: """ - return the boolean result of `cmpop` between to strings - since the typing is the same for every comparison operator, - we can reuse this class for all of them. + As part of Numba's preprocessing step, incoming function arguments are + modified based on the associated type for that argument that was used + to JIT the kernel. However it only knows how to handle built in array + types natively. With string UDFs, the jitted type is string_view*, + which numba does not know how to handle. + + This class converts string_view* to raw pointer arguments, which Numba + knows how to use. + + See numba.cuda.compiler._prepare_args for details. + """ + + def prepare_args(self, ty, val, **kwargs): + if isinstance(ty, types.CPointer) and isinstance( + ty.dtype, (StringView, UDFString) + ): + return types.uint64, val.ptr if isinstance( + val, rmm._lib.device_buffer.DeviceBuffer + ) else val.get_ptr(mode="read") + else: + return ty, val + + +str_view_arg_handler = StrViewArgHandler() + + +# String functions +@cuda_decl_registry.register_global(len) +class StringLength(AbstractTemplate): + """ + provide the length of a cudf::string_view like struct """ def generic(self, args, kws): - if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): - return nb_signature( - MaskedType(types.boolean), - MaskedType(string_view), - MaskedType(string_view), - ) + if isinstance(args[0], any_string_ty) and len(args) == 1: + # length: + # string_view -> int32 + # udf_string -> int32 + # literal -> int32 + return nb_signature(size_type, args[0]) -for op in comparison_ops: - cuda_decl_registry.register_global(op)(MaskedStringViewCmpOp) +def register_stringview_binaryop(op, retty): + """ + Helper function wrapping numba's low level extension API. Provides + the boilerplate needed to associate a signature with a function or + operator expecting a string. + """ + + class StringViewBinaryOp(AbstractTemplate): + def generic(self, args, kws): + if isinstance(args[0], any_string_ty) and isinstance( + args[1], any_string_ty + ): + return nb_signature(retty, string_view, string_view) + cuda_decl_registry.register_global(op)(StringViewBinaryOp) -def create_masked_binary_attr(attrname, retty): + +def create_binary_attr(attrname, retty): """ Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to register a binary function of two masked - string objects as an attribute of one, e.g. `string.func(other)`. + the boilerplate needed to register a binary function of two string + objects as an attribute of one, e.g. `string.func(other)`. """ - class MaskedStringViewBinaryAttr(AbstractTemplate): - key = attrname + class StringViewBinaryAttr(AbstractTemplate): + key = f"StringView.{attrname}" def generic(self, args, kws): - return nb_signature( - MaskedType(retty), MaskedType(string_view), recvr=self.this - ) + return nb_signature(retty, string_view, recvr=self.this) def attr(self, mod): - return types.BoundFunction( - MaskedStringViewBinaryAttr, - MaskedType(string_view), - ) + return types.BoundFunction(StringViewBinaryAttr, string_view) return attr -def create_masked_unary_attr(attrname, retty): +def create_identifier_attr(attrname, retty): """ Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to register a unary function of a masked - string object as an attribute, e.g. `string.func()`. + the boilerplate needed to register a unary function of a string + object as an attribute, e.g. `string.func()`. """ - class MaskedStringViewIdentifierAttr(AbstractTemplate): - key = attrname + class StringViewIdentifierAttr(AbstractTemplate): + key = f"StringView.{attrname}" def generic(self, args, kws): - return nb_signature(MaskedType(retty), recvr=self.this) + return nb_signature(retty, recvr=self.this) def attr(self, mod): - return types.BoundFunction( - MaskedStringViewIdentifierAttr, - MaskedType(string_view), - ) + return types.BoundFunction(StringViewIdentifierAttr, string_view) return attr -class MaskedStringViewCount(AbstractTemplate): - key = "MaskedType.count" +class StringViewCount(AbstractTemplate): + key = "StringView.count" def generic(self, args, kws): - return nb_signature( - MaskedType(size_type), MaskedType(string_view), recvr=self.this - ) + return nb_signature(size_type, string_view, recvr=self.this) -class MaskedStringViewReplace(AbstractTemplate): - key = "MaskedType.replace" +class StringViewReplace(AbstractTemplate): + key = "StringView.replace" def generic(self, args, kws): return nb_signature( - MaskedType(udf_string), - MaskedType(string_view), - MaskedType(string_view), - recvr=self.this, + udf_string, string_view, string_view, recvr=self.this ) -class MaskedStringViewAttrs(AttributeTemplate): - key = MaskedType(string_view) - - def resolve_replace(self, mod): - return types.BoundFunction( - MaskedStringViewReplace, MaskedType(string_view) - ) +class StringViewAttrs(AttributeTemplate): + key = string_view def resolve_count(self, mod): - return types.BoundFunction( - MaskedStringViewCount, MaskedType(string_view) - ) - - def resolve_value(self, mod): - return string_view - - def resolve_valid(self, mod): - return types.boolean + return types.BoundFunction(StringViewCount, string_view) + def resolve_replace(self, mod): + return types.BoundFunction(StringViewReplace, string_view) + + +bool_binary_funcs = ["startswith", "endswith"] +int_binary_funcs = ["find", "rfind"] +id_unary_funcs = [ + "isalpha", + "isalnum", + "isdecimal", + "isdigit", + "isupper", + "islower", + "isspace", + "isnumeric", + "istitle", +] +string_unary_funcs = ["upper", "lower"] +string_return_attrs = ["strip", "lstrip", "rstrip"] -# Build attributes for `MaskedType(string_view)` for func in bool_binary_funcs: setattr( - MaskedStringViewAttrs, + StringViewAttrs, f"resolve_{func}", - create_masked_binary_attr(f"MaskedType.{func}", types.boolean), + create_binary_attr(func, types.boolean), ) -for func in int_binary_funcs: +for func in string_return_attrs: setattr( - MaskedStringViewAttrs, + StringViewAttrs, f"resolve_{func}", - create_masked_binary_attr(f"MaskedType.{func}", size_type), + create_binary_attr(func, udf_string), ) -for func in string_return_attrs: + +for func in int_binary_funcs: setattr( - MaskedStringViewAttrs, - f"resolve_{func}", - create_masked_binary_attr(f"MaskedType.{func}", udf_string), + StringViewAttrs, f"resolve_{func}", create_binary_attr(func, size_type) ) for func in id_unary_funcs: setattr( - MaskedStringViewAttrs, + StringViewAttrs, f"resolve_{func}", - create_masked_unary_attr(f"MaskedType.{func}", types.boolean), + create_identifier_attr(func, types.boolean), ) for func in string_unary_funcs: setattr( - MaskedStringViewAttrs, + StringViewAttrs, f"resolve_{func}", - create_masked_unary_attr(f"MaskedType.{func}", udf_string), + create_identifier_attr(func, udf_string), ) -cuda_decl_registry.register_attr(MaskedStringViewAttrs) +cuda_decl_registry.register_attr(StringViewAttrs) + +register_stringview_binaryop(operator.eq, types.boolean) +register_stringview_binaryop(operator.ne, types.boolean) +register_stringview_binaryop(operator.lt, types.boolean) +register_stringview_binaryop(operator.gt, types.boolean) +register_stringview_binaryop(operator.le, types.boolean) +register_stringview_binaryop(operator.ge, types.boolean) + +# st in other +register_stringview_binaryop(operator.contains, types.boolean) + +# st + other +register_stringview_binaryop(operator.add, udf_string) diff --git a/python/cudf/cudf/core/udf/utils.py b/python/cudf/cudf/core/udf/utils.py index 3ee1d8edcbd..edc1a16353f 100644 --- a/python/cudf/cudf/core/udf/utils.py +++ b/python/cudf/cudf/core/udf/utils.py @@ -2,13 +2,14 @@ import glob import os -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict import cachetools import cupy as cp import llvmlite.binding as ll import numpy as np from cubinlinker.patch import _numba_version_ok, get_logger, new_patched_linker +from cuda import cudart from numba import cuda, typeof from numba.core.datamodel import default_manager from numba.core.errors import TypingError @@ -19,32 +20,105 @@ import rmm +from cudf._lib.strings_udf import ( + column_from_udf_string_array, + column_to_string_view_array, +) from cudf.core.column.column import as_column +from cudf.core.dtypes import dtype from cudf.core.udf.masked_typing import MaskedType +from cudf.core.udf.strings_typing import ( + str_view_arg_handler, + string_view, + udf_string, +) from cudf.utils import cudautils from cudf.utils.dtypes import ( BOOL_TYPES, DATETIME_TYPES, NUMERIC_TYPES, + STRING_TYPES, TIMEDELTA_TYPES, ) -from cudf.utils.utils import _cudf_nvtx_annotate +from cudf.utils.utils import _cudf_nvtx_annotate, initfunc + +# Maximum size of a string column is 2 GiB +_STRINGS_UDF_DEFAULT_HEAP_SIZE = os.environ.get( + "STRINGS_UDF_HEAP_SIZE", 2**31 +) +_heap_size = 0 +_cudf_str_dtype = dtype(str) + logger = get_logger() JIT_SUPPORTED_TYPES = ( - NUMERIC_TYPES | BOOL_TYPES | DATETIME_TYPES | TIMEDELTA_TYPES + NUMERIC_TYPES + | BOOL_TYPES + | DATETIME_TYPES + | TIMEDELTA_TYPES + | STRING_TYPES ) libcudf_bitmask_type = numpy_support.from_dtype(np.dtype("int32")) MASK_BITSIZE = np.dtype("int32").itemsize * 8 precompiled: cachetools.LRUCache = cachetools.LRUCache(maxsize=32) -arg_handlers: List[Any] = [] -ptx_files: List[Any] = [] -masked_array_types: Dict[Any, Any] = {} launch_arg_getters: Dict[Any, Any] = {} -output_col_getters: Dict[Any, Any] = {} + + +def _get_best_ptx_file(archs, max_compute_capability): + """ + Determine of the available PTX files which one is + the most recent up to and including the device cc + """ + filtered_archs = [x for x in archs if x[0] <= max_compute_capability] + if filtered_archs: + return max(filtered_archs, key=lambda y: y[0]) + else: + return None + + +def _get_ptx_file(path, prefix): + if "RAPIDS_NO_INITIALIZE" in os.environ: + # cc=60 ptx is always built + cc = int(os.environ.get("STRINGS_UDF_CC", "60")) + else: + dev = cuda.get_current_device() + + # Load the highest compute capability file available that is less than + # the current device's. + cc = int("".join(str(x) for x in dev.compute_capability)) + files = glob.glob(os.path.join(path, f"{prefix}*.ptx")) + if len(files) == 0: + raise RuntimeError(f"Missing PTX files for cc={cc}") + regular_sms = [] + + for f in files: + file_name = os.path.basename(f) + sm_number = file_name.rstrip(".ptx").lstrip(prefix) + if sm_number.endswith("a"): + processed_sm_number = int(sm_number.rstrip("a")) + if processed_sm_number == cc: + return f + else: + regular_sms.append((int(sm_number), f)) + + regular_result = None + + if regular_sms: + regular_result = _get_best_ptx_file(regular_sms, cc) + + if regular_result is None: + raise RuntimeError( + "This cuDF installation is missing the necessary PTX " + f"files that are <={cc}." + ) + else: + return regular_result[1] + + +_PTX_FILE = _get_ptx_file(os.path.dirname(__file__), "shim_") @_cudf_nvtx_annotate @@ -129,9 +203,8 @@ def _masked_array_type_from_col(col): array of bools representing a mask. """ - col_type = masked_array_types.get(col.dtype) - if col_type: - col_type = CPointer(col_type) + if col.dtype == _cudf_str_dtype: + col_type = CPointer(string_view) else: nb_scalar_ty = numpy_support.from_dtype(col.dtype) col_type = nb_scalar_ty[::1] @@ -239,7 +312,9 @@ def _get_kernel(kernel_string, globals_, sig, func): globals_["f_"] = f_ exec(kernel_string, globals_) _kernel = globals_["_kernel"] - kernel = cuda.jit(sig, link=ptx_files, extensions=arg_handlers)(_kernel) + kernel = cuda.jit( + sig, link=[_PTX_FILE], extensions=[str_view_arg_handler] + )(_kernel) return kernel @@ -248,9 +323,8 @@ def _get_input_args_from_frame(fr): args = [] offsets = [] for col in _supported_cols_from_frame(fr).values(): - getter = launch_arg_getters.get(col.dtype) - if getter: - data = getter(col) + if col.dtype == _cudf_str_dtype: + data = column_to_string_view_array_init_heap(col) else: data = col.data if col.mask is not None: @@ -264,69 +338,18 @@ def _get_input_args_from_frame(fr): return args + offsets -def _return_arr_from_dtype(dt, size): - if extensionty := masked_array_types.get(dt): - return rmm.DeviceBuffer(size=size * extensionty.return_type.size_bytes) - return cp.empty(size, dtype=dt) +def _return_arr_from_dtype(dtype, size): + if dtype == _cudf_str_dtype: + return rmm.DeviceBuffer(size=size * _get_extensionty_size(udf_string)) + return cp.empty(size, dtype=dtype) def _post_process_output_col(col, retty): - if getter := output_col_getters.get(retty): - col = getter(col) + if retty == _cudf_str_dtype: + return column_from_udf_string_array(col) return as_column(col, retty) -def _get_best_ptx_file(archs, max_compute_capability): - """ - Determine of the available PTX files which one is - the most recent up to and including the device cc - """ - filtered_archs = [x for x in archs if x[0] <= max_compute_capability] - if filtered_archs: - return max(filtered_archs, key=lambda y: y[0]) - else: - return None - - -def _get_ptx_file(path, prefix): - if "RAPIDS_NO_INITIALIZE" in os.environ: - # cc=60 ptx is always built - cc = int(os.environ.get("STRINGS_UDF_CC", "60")) - else: - dev = cuda.get_current_device() - - # Load the highest compute capability file available that is less than - # the current device's. - cc = int("".join(str(x) for x in dev.compute_capability)) - files = glob.glob(os.path.join(path, f"{prefix}*.ptx")) - if len(files) == 0: - raise RuntimeError(f"Missing PTX files for cc={cc}") - regular_sms = [] - - for f in files: - file_name = os.path.basename(f) - sm_number = file_name.rstrip(".ptx").lstrip(prefix) - if sm_number.endswith("a"): - processed_sm_number = int(sm_number.rstrip("a")) - if processed_sm_number == cc: - return f - else: - regular_sms.append((int(sm_number), f)) - - regular_result = None - - if regular_sms: - regular_result = _get_best_ptx_file(regular_sms, cc) - - if regular_result is None: - raise RuntimeError( - "This cuDF installation is missing the necessary PTX " - f"files that are <={cc}." - ) - else: - return regular_result[1] - - def _get_extensionty_size(ty): """ Return the size of an extension type in bytes @@ -420,3 +443,26 @@ def maybe_patch_numba_linker( Linker.new = new_patched_linker else: logger.debug("Cannot patch Numba Linker - unsupported version") + + +@initfunc +def set_malloc_heap_size(size=None): + """ + Heap size control for strings_udf, size in bytes. + """ + global _heap_size + if size is None: + size = _STRINGS_UDF_DEFAULT_HEAP_SIZE + if size != _heap_size: + (ret,) = cudart.cudaDeviceSetLimit( + cudart.cudaLimit.cudaLimitMallocHeapSize, size + ) + if ret.value != 0: + raise RuntimeError("Unable to set cudaMalloc heap size") + + _heap_size = size + + +def column_to_string_view_array_init_heap(col): + # lazily allocate heap only when a string needs to be returned + return column_to_string_view_array(col) diff --git a/python/strings_udf/strings_udf/tests/test_string_udfs.py b/python/cudf/cudf/tests/test_string_udfs.py similarity index 94% rename from python/strings_udf/strings_udf/tests/test_string_udfs.py rename to python/cudf/cudf/tests/test_string_udfs.py index b8de821e101..a8de63be0f5 100644 --- a/python/strings_udf/strings_udf/tests/test_string_udfs.py +++ b/python/cudf/cudf/tests/test_string_udfs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import numba import numpy as np @@ -8,16 +8,20 @@ from numba.core.typing import signature as nb_signature from numba.types import CPointer, void -import cudf import rmm -from cudf.testing._utils import assert_eq -import strings_udf -from strings_udf._lib.cudf_jit_udf import ( +import cudf +from cudf._lib.strings_udf import ( column_from_udf_string_array, column_to_string_view_array, ) -from strings_udf._typing import str_view_arg_handler, string_view, udf_string +from cudf.core.udf.strings_typing import ( + str_view_arg_handler, + string_view, + udf_string, +) +from cudf.core.udf.utils import _PTX_FILE, _get_extensionty_size +from cudf.testing._utils import assert_eq def get_kernel(func, dtype, size): @@ -36,9 +40,7 @@ def get_kernel(func, dtype, size): outty = numba.np.numpy_support.from_dtype(dtype)[::1] sig = nb_signature(void, CPointer(string_view), outty) - @cuda.jit( - sig, link=[strings_udf.ptxpath], extensions=[str_view_arg_handler] - ) + @cuda.jit(sig, link=[_PTX_FILE], extensions=[str_view_arg_handler]) def kernel(input_strings, output_col): id = cuda.grid(1) if id < size: @@ -59,7 +61,9 @@ def run_udf_test(data, func, dtype): comparing it with the equivalent pandas result """ if dtype == "str": - output = rmm.DeviceBuffer(size=len(data) * udf_string.size_bytes) + output = rmm.DeviceBuffer( + size=len(data) * _get_extensionty_size(udf_string) + ) else: dtype = np.dtype(dtype) output = cudf.core.column.column_empty(len(data), dtype=dtype) diff --git a/python/cudf/cudf/tests/test_udf_masked_ops.py b/python/cudf/cudf/tests/test_udf_masked_ops.py index 091f44176ab..4d54c3181b2 100644 --- a/python/cudf/cudf/tests/test_udf_masked_ops.py +++ b/python/cudf/cudf/tests/test_udf_masked_ops.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import math import operator @@ -8,7 +8,6 @@ import cudf from cudf.core.missing import NA -from cudf.core.udf import _STRING_UDFS_ENABLED from cudf.core.udf._ops import ( arith_ops, bitwise_ops, @@ -757,8 +756,6 @@ def func(x): run_masked_udf_series(func, data, check_dtype=False) -# only run string udf tests if library exists and is enabled -@pytest.mark.skipif(not _STRING_UDFS_ENABLED, reason="String UDFs not enabled") class TestStringUDFs: def test_string_udf_len(self, str_udf_data): def func(row): diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg index 8a648097ac8..59dd5d0179e 100644 --- a/python/cudf/setup.cfg +++ b/python/cudf/setup.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. # See the docstring in versioneer.py for instructions. Note that you must # re-run 'versioneer.py setup' after changing this section, and commit the @@ -25,7 +25,6 @@ known_dask= dask_cuda known_rapids= rmm - strings_udf known_first_party= cudf default_section=THIRDPARTY diff --git a/python/strings_udf/cpp/CMakeLists.txt b/python/cudf/udf_cpp/CMakeLists.txt similarity index 75% rename from python/strings_udf/cpp/CMakeLists.txt rename to python/cudf/udf_cpp/CMakeLists.txt index 3e58d10d6e2..0c07236682f 100644 --- a/python/strings_udf/cpp/CMakeLists.txt +++ b/python/cudf/udf_cpp/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -16,38 +16,28 @@ cmake_minimum_required(VERSION 3.23.1) include(rapids-cmake) include(rapids-cpm) -include(rapids-cuda) include(rapids-find) -rapids_cuda_init_architectures(strings-udf-cpp) - -# Create a project so that we can enable CUDA architectures in this file. -project( - strings-udf-cpp - VERSION ${strings_udf_version} - LANGUAGES CUDA -) - rapids_cpm_init() rapids_find_package( CUDAToolkit REQUIRED - BUILD_EXPORT_SET strings-udf-exports - INSTALL_EXPORT_SET strings-udf-exports + BUILD_EXPORT_SET udf-exports + INSTALL_EXPORT_SET udf-exports ) include(${rapids-cmake-dir}/cpm/libcudacxx.cmake) -rapids_cpm_libcudacxx(BUILD_EXPORT_SET strings-udf-exports INSTALL_EXPORT_SET strings-udf-exports) +rapids_cpm_libcudacxx(BUILD_EXPORT_SET udf-exports INSTALL_EXPORT_SET udf-exports) -add_library(cudf_strings_udf SHARED src/strings/udf/udf_apis.cu) +add_library(cudf_strings_udf SHARED strings/src/strings/udf/udf_apis.cu) target_include_directories( - cudf_strings_udf PUBLIC "$" + cudf_strings_udf PUBLIC "$" ) set_target_properties( cudf_strings_udf - PROPERTIES BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" + PROPERTIES BUILD_RPATH "\$ORIGIN/../" + INSTALL_RPATH "\$ORIGIN/../" CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON CUDA_STANDARD 17 @@ -62,8 +52,8 @@ target_compile_options( cudf_strings_udf PRIVATE "$<$:${UDF_CXX_FLAGS}>" "$<$:${UDF_CUDA_FLAGS}>" ) -target_link_libraries(cudf_strings_udf PUBLIC cudf::cudf CUDA::nvrtc) -install(TARGETS cudf_strings_udf DESTINATION ./strings_udf/_lib/) +target_link_libraries(cudf_strings_udf PUBLIC cudf::cudf) +install(TARGETS cudf_strings_udf DESTINATION ./cudf/_lib/) # This function will copy the generated PTX file from its generator-specific location in the build # tree into a specified location in the build tree from which we can install it. @@ -104,18 +94,20 @@ list(REMOVE_DUPLICATES CMAKE_CUDA_ARCHITECTURES) foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES) set(tgt shim_${arch}) - add_library(${tgt} OBJECT src/strings/udf/shim.cu) + add_library(${tgt} OBJECT shim.cu) set_target_properties(${tgt} PROPERTIES CUDA_ARCHITECTURES ${arch} CUDA_PTX_COMPILATION ON) - target_include_directories(${tgt} PUBLIC include) + target_include_directories( + ${tgt} PUBLIC "$" + ) target_compile_options(${tgt} PRIVATE "$<$:${SHIM_CUDA_FLAGS}>") target_link_libraries(${tgt} PUBLIC cudf::cudf) - copy_ptx_to_location(${tgt} "${CMAKE_CURRENT_BINARY_DIR}/../strings_udf" ${tgt}.ptx) + copy_ptx_to_location(${tgt} "${CMAKE_CURRENT_BINARY_DIR}/../udf" ${tgt}.ptx) install( FILES $ - DESTINATION ./strings_udf + DESTINATION ./cudf/core/udf/ RENAME ${tgt}.ptx ) endforeach() diff --git a/python/cudf/udf_cpp/groupby/CMakeLists.txt b/python/cudf/udf_cpp/groupby/CMakeLists.txt deleted file mode 100644 index 043ab28f362..00000000000 --- a/python/cudf/udf_cpp/groupby/CMakeLists.txt +++ /dev/null @@ -1,79 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -cmake_minimum_required(VERSION 3.23.1) - -include(rapids-find) - -# This function will copy the generated PTX file from its generator-specific location in the build -# tree into a specified location in the build tree from which we can install it. -function(copy_ptx_to_location target destination) - set(cmake_generated_file - "${CMAKE_CURRENT_BINARY_DIR}/cmake/cp_${target}_$>_ptx.cmake" - ) - file( - GENERATE - OUTPUT "${cmake_generated_file}" - CONTENT - " -set(ptx_paths \"$\") -file(COPY_FILE \${ptx_paths} \"${destination}/${target}.ptx\")" - ) - - add_custom_target( - ${target}_cp_ptx ALL - COMMAND ${CMAKE_COMMAND} -P "${cmake_generated_file}" - DEPENDS $ - COMMENT "Copying PTX files to '${destination}'" - ) -endfunction() - -# Create the shim library for each architecture. -set(GROUPBY_FUNCTION_CUDA_FLAGS --expt-relaxed-constexpr) - -# always build a default PTX file in case RAPIDS_NO_INITIALIZE is set and the device cc can't be -# safely queried through a context -list(INSERT CMAKE_CUDA_ARCHITECTURES 0 "60") - -list(TRANSFORM CMAKE_CUDA_ARCHITECTURES REPLACE "-real" "") -list(TRANSFORM CMAKE_CUDA_ARCHITECTURES REPLACE "-virtual" "") -list(SORT CMAKE_CUDA_ARCHITECTURES) -list(REMOVE_DUPLICATES CMAKE_CUDA_ARCHITECTURES) - -foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES) - set(tgt function_${arch}) - - add_library(${tgt} OBJECT function.cu) - set_target_properties( - ${tgt} - PROPERTIES CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - CUDA_ARCHITECTURES ${arch} - CUDA_PTX_COMPILATION ON - CUDA_SEPARABLE_COMPILATION ON - ) - - target_include_directories(${tgt} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) - target_compile_options( - ${tgt} PRIVATE "$<$:${GROUPBY_FUNCTION_CUDA_FLAGS}>" - ) - target_link_libraries(${tgt} PUBLIC cudf::cudf) - - copy_ptx_to_location(${tgt} "${CMAKE_CURRENT_BINARY_DIR}/") - install( - FILES $ - DESTINATION ./cudf/core/udf/ - RENAME ${tgt}.ptx - ) -endforeach() diff --git a/python/cudf/udf_cpp/groupby/function.cu b/python/cudf/udf_cpp/groupby/function.cu deleted file mode 100644 index 782371b8a44..00000000000 --- a/python/cudf/udf_cpp/groupby/function.cu +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -#include - -#include -#include - -template -__device__ bool are_all_nans(cooperative_groups::thread_block const& block, - T const* data, - int64_t size) -{ - // TODO: to be refactored with CG vote functions once - // block size is known at build time - __shared__ int64_t count; - - if (block.thread_rank() == 0) { count = 0; } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - if (not std::isnan(data[idx])) { - cuda::atomic_ref ref{count}; - ref.fetch_add(1, cuda::std::memory_order_relaxed); - break; - } - } - - block.sync(); - return count == 0; -} - -template -__device__ void device_sum(cooperative_groups::thread_block const& block, - T const* data, - int64_t size, - T* sum) -{ - T local_sum = 0; - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - local_sum += data[idx]; - } - - cuda::atomic_ref ref{*sum}; - ref.fetch_add(local_sum, cuda::std::memory_order_relaxed); - - block.sync(); -} - -template -__device__ T BlockSum(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - if constexpr (std::is_floating_point_v) { - if (are_all_nans(block, data, size)) { return 0; } - } - - __shared__ T block_sum; - if (block.thread_rank() == 0) { block_sum = 0; } - block.sync(); - - device_sum(block, data, size, &block_sum); - return block_sum; -} - -template -__device__ double BlockMean(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - __shared__ T block_sum; - if (block.thread_rank() == 0) { block_sum = 0; } - block.sync(); - - device_sum(block, data, size, &block_sum); - return static_cast(block_sum) / static_cast(size); -} - -template -__device__ double BlockVar(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - __shared__ double block_var; - __shared__ T block_sum; - if (block.thread_rank() == 0) { - block_var = 0; - block_sum = 0; - } - block.sync(); - - T local_sum = 0; - double local_var = 0; - - device_sum(block, data, size, &block_sum); - - auto const mean = static_cast(block_sum) / static_cast(size); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - auto const delta = static_cast(data[idx]) - mean; - local_var += delta * delta; - } - - cuda::atomic_ref ref{block_var}; - ref.fetch_add(local_var, cuda::std::memory_order_relaxed); - block.sync(); - - if (block.thread_rank() == 0) { block_var = block_var / static_cast(size - 1); } - block.sync(); - return block_var; -} - -template -__device__ double BlockStd(T const* data, int64_t size) -{ - auto const var = BlockVar(data, size); - return sqrt(var); -} - -template -__device__ T BlockMax(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - if constexpr (std::is_floating_point_v) { - if (are_all_nans(block, data, size)) { return std::numeric_limits::quiet_NaN(); } - } - - auto local_max = cudf::DeviceMax::identity(); - __shared__ T block_max; - if (block.thread_rank() == 0) { block_max = local_max; } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - local_max = max(local_max, data[idx]); - } - - cuda::atomic_ref ref{block_max}; - ref.fetch_max(local_max, cuda::std::memory_order_relaxed); - - block.sync(); - - return block_max; -} - -template -__device__ T BlockMin(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - if constexpr (std::is_floating_point_v) { - if (are_all_nans(block, data, size)) { return std::numeric_limits::quiet_NaN(); } - } - - auto local_min = cudf::DeviceMin::identity(); - - __shared__ T block_min; - if (block.thread_rank() == 0) { block_min = local_min; } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - local_min = min(local_min, data[idx]); - } - - cuda::atomic_ref ref{block_min}; - ref.fetch_min(local_min, cuda::std::memory_order_relaxed); - - block.sync(); - - return block_min; -} - -template -__device__ int64_t BlockIdxMax(T const* data, int64_t* index, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - __shared__ T block_max; - __shared__ int64_t block_idx_max; - __shared__ bool found_max; - - auto local_max = cudf::DeviceMax::identity(); - auto local_idx_max = cudf::DeviceMin::identity(); - - if (block.thread_rank() == 0) { - block_max = local_max; - block_idx_max = local_idx_max; - found_max = false; - } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - auto const current_data = data[idx]; - if (current_data > local_max) { - local_max = current_data; - local_idx_max = index[idx]; - found_max = true; - } - } - - cuda::atomic_ref ref{block_max}; - ref.fetch_max(local_max, cuda::std::memory_order_relaxed); - block.sync(); - - if (found_max) { - if (local_max == block_max) { - cuda::atomic_ref ref_idx{block_idx_max}; - ref_idx.fetch_min(local_idx_max, cuda::std::memory_order_relaxed); - } - } else { - if (block.thread_rank() == 0) { block_idx_max = index[0]; } - } - block.sync(); - - return block_idx_max; -} - -template -__device__ int64_t BlockIdxMin(T const* data, int64_t* index, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - __shared__ T block_min; - __shared__ int64_t block_idx_min; - __shared__ bool found_min; - - auto local_min = cudf::DeviceMin::identity(); - auto local_idx_min = cudf::DeviceMin::identity(); - - if (block.thread_rank() == 0) { - block_min = local_min; - block_idx_min = local_idx_min; - found_min = false; - } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - auto const current_data = data[idx]; - if (current_data < local_min) { - local_min = current_data; - local_idx_min = index[idx]; - found_min = true; - } - } - - cuda::atomic_ref ref{block_min}; - ref.fetch_min(local_min, cuda::std::memory_order_relaxed); - block.sync(); - - if (found_min) { - if (local_min == block_min) { - cuda::atomic_ref ref_idx{block_idx_min}; - ref_idx.fetch_min(local_idx_min, cuda::std::memory_order_relaxed); - } - } else { - if (block.thread_rank() == 0) { block_idx_min = index[0]; } - } - block.sync(); - - return block_idx_min; -} - -extern "C" { -#define make_definition(name, cname, type, return_type) \ - __device__ int name##_##cname(return_type* numba_return_value, type* const data, int64_t size) \ - { \ - return_type const res = name(data, size); \ - *numba_return_value = res; \ - __syncthreads(); \ - return 0; \ - } - -make_definition(BlockSum, int64, int64_t, int64_t); -make_definition(BlockSum, float64, double, double); -make_definition(BlockMean, int64, int64_t, double); -make_definition(BlockMean, float64, double, double); -make_definition(BlockStd, int64, int64_t, double); -make_definition(BlockStd, float64, double, double); -make_definition(BlockVar, int64, int64_t, double); -make_definition(BlockVar, float64, double, double); -make_definition(BlockMin, int64, int64_t, int64_t); -make_definition(BlockMin, float64, double, double); -make_definition(BlockMax, int64, int64_t, int64_t); -make_definition(BlockMax, float64, double, double); -#undef make_definition -} - -extern "C" { -#define make_definition_idx(name, cname, type) \ - __device__ int name##_##cname( \ - int64_t* numba_return_value, type* const data, int64_t* index, int64_t size) \ - { \ - auto const res = name(data, index, size); \ - *numba_return_value = res; \ - __syncthreads(); \ - return 0; \ - } - -make_definition_idx(BlockIdxMin, int64, int64_t); -make_definition_idx(BlockIdxMin, float64, double); -make_definition_idx(BlockIdxMax, int64, int64_t); -make_definition_idx(BlockIdxMax, float64, double); -#undef make_definition_idx -} diff --git a/python/strings_udf/cpp/src/strings/udf/shim.cu b/python/cudf/udf_cpp/shim.cu similarity index 55% rename from python/strings_udf/cpp/src/strings/udf/shim.cu rename to python/cudf/udf_cpp/shim.cu index d10cc635209..9223a54654e 100644 --- a/python/strings_udf/cpp/src/strings/udf/shim.cu +++ b/python/cudf/udf_cpp/shim.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ * limitations under the License. */ +#include #include #include #include @@ -22,6 +23,13 @@ #include #include +#include + +#include + +#include +#include + using namespace cudf::strings::udf; extern "C" __device__ int len(int* nb_retval, void const* str) @@ -343,3 +351,303 @@ extern "C" __device__ int replace( return 0; } + +// Groupby Shim Functions +template +__device__ bool are_all_nans(cooperative_groups::thread_block const& block, + T const* data, + int64_t size) +{ + // TODO: to be refactored with CG vote functions once + // block size is known at build time + __shared__ int64_t count; + + if (block.thread_rank() == 0) { count = 0; } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + if (not std::isnan(data[idx])) { + cuda::atomic_ref ref{count}; + ref.fetch_add(1, cuda::std::memory_order_relaxed); + break; + } + } + + block.sync(); + return count == 0; +} + +template +__device__ void device_sum(cooperative_groups::thread_block const& block, + T const* data, + int64_t size, + T* sum) +{ + T local_sum = 0; + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + local_sum += data[idx]; + } + + cuda::atomic_ref ref{*sum}; + ref.fetch_add(local_sum, cuda::std::memory_order_relaxed); + + block.sync(); +} + +template +__device__ T BlockSum(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + if constexpr (std::is_floating_point_v) { + if (are_all_nans(block, data, size)) { return 0; } + } + + __shared__ T block_sum; + if (block.thread_rank() == 0) { block_sum = 0; } + block.sync(); + + device_sum(block, data, size, &block_sum); + return block_sum; +} + +template +__device__ double BlockMean(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + __shared__ T block_sum; + if (block.thread_rank() == 0) { block_sum = 0; } + block.sync(); + + device_sum(block, data, size, &block_sum); + return static_cast(block_sum) / static_cast(size); +} + +template +__device__ double BlockVar(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + __shared__ double block_var; + __shared__ T block_sum; + if (block.thread_rank() == 0) { + block_var = 0; + block_sum = 0; + } + block.sync(); + + T local_sum = 0; + double local_var = 0; + + device_sum(block, data, size, &block_sum); + + auto const mean = static_cast(block_sum) / static_cast(size); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + auto const delta = static_cast(data[idx]) - mean; + local_var += delta * delta; + } + + cuda::atomic_ref ref{block_var}; + ref.fetch_add(local_var, cuda::std::memory_order_relaxed); + block.sync(); + + if (block.thread_rank() == 0) { block_var = block_var / static_cast(size - 1); } + block.sync(); + return block_var; +} + +template +__device__ double BlockStd(T const* data, int64_t size) +{ + auto const var = BlockVar(data, size); + return sqrt(var); +} + +template +__device__ T BlockMax(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + if constexpr (std::is_floating_point_v) { + if (are_all_nans(block, data, size)) { return std::numeric_limits::quiet_NaN(); } + } + + auto local_max = cudf::DeviceMax::identity(); + __shared__ T block_max; + if (block.thread_rank() == 0) { block_max = local_max; } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + local_max = max(local_max, data[idx]); + } + + cuda::atomic_ref ref{block_max}; + ref.fetch_max(local_max, cuda::std::memory_order_relaxed); + + block.sync(); + + return block_max; +} + +template +__device__ T BlockMin(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + if constexpr (std::is_floating_point_v) { + if (are_all_nans(block, data, size)) { return std::numeric_limits::quiet_NaN(); } + } + + auto local_min = cudf::DeviceMin::identity(); + + __shared__ T block_min; + if (block.thread_rank() == 0) { block_min = local_min; } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + local_min = min(local_min, data[idx]); + } + + cuda::atomic_ref ref{block_min}; + ref.fetch_min(local_min, cuda::std::memory_order_relaxed); + + block.sync(); + + return block_min; +} + +template +__device__ int64_t BlockIdxMax(T const* data, int64_t* index, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + __shared__ T block_max; + __shared__ int64_t block_idx_max; + __shared__ bool found_max; + + auto local_max = cudf::DeviceMax::identity(); + auto local_idx_max = cudf::DeviceMin::identity(); + + if (block.thread_rank() == 0) { + block_max = local_max; + block_idx_max = local_idx_max; + found_max = false; + } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + auto const current_data = data[idx]; + if (current_data > local_max) { + local_max = current_data; + local_idx_max = index[idx]; + found_max = true; + } + } + + cuda::atomic_ref ref{block_max}; + ref.fetch_max(local_max, cuda::std::memory_order_relaxed); + block.sync(); + + if (found_max) { + if (local_max == block_max) { + cuda::atomic_ref ref_idx{block_idx_max}; + ref_idx.fetch_min(local_idx_max, cuda::std::memory_order_relaxed); + } + } else { + if (block.thread_rank() == 0) { block_idx_max = index[0]; } + } + block.sync(); + + return block_idx_max; +} + +template +__device__ int64_t BlockIdxMin(T const* data, int64_t* index, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + __shared__ T block_min; + __shared__ int64_t block_idx_min; + __shared__ bool found_min; + + auto local_min = cudf::DeviceMin::identity(); + auto local_idx_min = cudf::DeviceMin::identity(); + + if (block.thread_rank() == 0) { + block_min = local_min; + block_idx_min = local_idx_min; + found_min = false; + } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + auto const current_data = data[idx]; + if (current_data < local_min) { + local_min = current_data; + local_idx_min = index[idx]; + found_min = true; + } + } + + cuda::atomic_ref ref{block_min}; + ref.fetch_min(local_min, cuda::std::memory_order_relaxed); + block.sync(); + + if (found_min) { + if (local_min == block_min) { + cuda::atomic_ref ref_idx{block_idx_min}; + ref_idx.fetch_min(local_idx_min, cuda::std::memory_order_relaxed); + } + } else { + if (block.thread_rank() == 0) { block_idx_min = index[0]; } + } + block.sync(); + + return block_idx_min; +} + +extern "C" { +#define make_definition(name, cname, type, return_type) \ + __device__ int name##_##cname(return_type* numba_return_value, type* const data, int64_t size) \ + { \ + return_type const res = name(data, size); \ + *numba_return_value = res; \ + __syncthreads(); \ + return 0; \ + } + +make_definition(BlockSum, int64, int64_t, int64_t); +make_definition(BlockSum, float64, double, double); +make_definition(BlockMean, int64, int64_t, double); +make_definition(BlockMean, float64, double, double); +make_definition(BlockStd, int64, int64_t, double); +make_definition(BlockStd, float64, double, double); +make_definition(BlockVar, int64, int64_t, double); +make_definition(BlockVar, float64, double, double); +make_definition(BlockMin, int64, int64_t, int64_t); +make_definition(BlockMin, float64, double, double); +make_definition(BlockMax, int64, int64_t, int64_t); +make_definition(BlockMax, float64, double, double); +#undef make_definition +} + +extern "C" { +#define make_definition_idx(name, cname, type) \ + __device__ int name##_##cname( \ + int64_t* numba_return_value, type* const data, int64_t* index, int64_t size) \ + { \ + auto const res = name(data, index, size); \ + *numba_return_value = res; \ + __syncthreads(); \ + return 0; \ + } + +make_definition_idx(BlockIdxMin, int64, int64_t); +make_definition_idx(BlockIdxMin, float64, double); +make_definition_idx(BlockIdxMax, int64, int64_t); +make_definition_idx(BlockIdxMax, float64, double); +#undef make_definition_idx +} diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/case.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/case.cuh similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/case.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/case.cuh index 472101959a6..76477087e71 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/case.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/case.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/char_types.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/char_types.cuh similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/char_types.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/char_types.cuh index 9320686442b..d03de2c4290 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/char_types.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/char_types.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/numeric.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/numeric.cuh similarity index 97% rename from python/strings_udf/cpp/include/cudf/strings/udf/numeric.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/numeric.cuh index c8c9f6e46f4..ecd952a2307 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/numeric.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/numeric.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/pad.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/pad.cuh similarity index 98% rename from python/strings_udf/cpp/include/cudf/strings/udf/pad.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/pad.cuh index d6d4ed637e9..2f3b4fe0298 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/pad.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/pad.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/replace.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/replace.cuh similarity index 97% rename from python/strings_udf/cpp/include/cudf/strings/udf/replace.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/replace.cuh index c1f0cdc94c5..6d2135291fd 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/replace.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/replace.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/search.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/search.cuh similarity index 97% rename from python/strings_udf/cpp/include/cudf/strings/udf/search.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/search.cuh index ef15886f1f5..3300c859dba 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/search.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/search.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/split.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/split.cuh similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/split.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/split.cuh index ca31425aa62..d3131464121 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/split.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/split.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/starts_with.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/starts_with.cuh similarity index 98% rename from python/strings_udf/cpp/include/cudf/strings/udf/starts_with.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/starts_with.cuh index 38c609ae505..1c7beac71f5 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/starts_with.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/starts_with.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/strip.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/strip.cuh similarity index 98% rename from python/strings_udf/cpp/include/cudf/strings/udf/strip.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/strip.cuh index f2db3073460..521b578bc52 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/strip.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/strip.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/udf_apis.hpp b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_apis.hpp similarity index 97% rename from python/strings_udf/cpp/include/cudf/strings/udf/udf_apis.hpp rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_apis.hpp index 68834afa082..219dbe27682 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/udf_apis.hpp +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_apis.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/udf_string.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.cuh similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/udf_string.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.cuh index 724116143d8..a8970c04b74 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/udf_string.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/udf_string.hpp b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.hpp similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/udf_string.hpp rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.hpp index 2bbda357cee..59cb35df419 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/udf_string.hpp +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/src/strings/udf/udf_apis.cu b/python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu similarity index 98% rename from python/strings_udf/cpp/src/strings/udf/udf_apis.cu rename to python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu index 3e6491e32e7..bedaa8e8fff 100644 --- a/python/strings_udf/cpp/src/strings/udf/udf_apis.cu +++ b/python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/CMakeLists.txt b/python/strings_udf/CMakeLists.txt deleted file mode 100644 index 2507116f957..00000000000 --- a/python/strings_udf/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -cmake_minimum_required(VERSION 3.23.1 FATAL_ERROR) - -set(strings_udf_version 23.04.00) - -include(../../fetch_rapids.cmake) - -include(rapids-cuda) -rapids_cuda_init_architectures(strings-udf-python) - -project( - strings-udf-python - VERSION ${strings_udf_version} - LANGUAGES CXX - # TODO: Building Python extension modules via the python_extension_module requires the C - # language to be enabled here. The test project that is built in scikit-build to verify - # various linking options for the python library is hardcoded to build with C, so until - # that is fixed we need to keep C. - C - # TODO: Enabling CUDA will not be necessary once we upgrade to CMake 3.22, which will - # pull in the required languages for the C++ project even if this project does not - # require those languages. - CUDA -) - -find_package(cudf ${strings_udf_version} REQUIRED) - -add_subdirectory(cpp) - -include(rapids-cython) -rapids_cython_init() - -add_subdirectory(strings_udf/_lib) diff --git a/python/strings_udf/setup.cfg b/python/strings_udf/setup.cfg deleted file mode 100644 index 9f29b26b5e0..00000000000 --- a/python/strings_udf/setup.cfg +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -[versioneer] -VCS = git -style = pep440 -versionfile_source = strings_udf/_version.py -versionfile_build = strings_udf/_version.py -tag_prefix = v -parentdir_prefix = strings_udf- - -[isort] -line_length=79 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -known_dask= - dask - distributed - dask_cuda -known_rapids= - rmm - cudf -known_first_party= - strings_udf -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER -skip= - thirdparty - .eggs - .git - .hg - .mypy_cache - .tox - .venv - _build - buck-out - build - dist - __init__.py diff --git a/python/strings_udf/setup.py b/python/strings_udf/setup.py deleted file mode 100644 index 5b4c503e032..00000000000 --- a/python/strings_udf/setup.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -import os -import re -import shutil - -import versioneer -from setuptools import find_packages -from skbuild import setup - -install_requires = ["numba>=0.53.1", "numpy", "cudf"] - -extras_require = { - "test": [ - "pytest", - ] -} - - -def get_cuda_version_from_header(cuda_include_dir, delimiter=""): - - cuda_version = None - - with open(os.path.join(cuda_include_dir, "cuda.h"), encoding="utf-8") as f: - for line in f.readlines(): - if re.search(r"#define CUDA_VERSION ", line) is not None: - cuda_version = line - break - - if cuda_version is None: - raise TypeError("CUDA_VERSION not found in cuda.h") - cuda_version = int(cuda_version.split()[2]) - return "%d%s%d" % ( - cuda_version // 1000, - delimiter, - (cuda_version % 1000) // 10, - ) - - -CUDA_HOME = os.environ.get("CUDA_HOME", False) -if not CUDA_HOME: - path_to_cuda_gdb = shutil.which("cuda-gdb") - if path_to_cuda_gdb is None: - raise OSError( - "Could not locate CUDA. " - "Please set the environment variable " - "CUDA_HOME to the path to the CUDA installation " - "and try again." - ) - CUDA_HOME = os.path.dirname(os.path.dirname(path_to_cuda_gdb)) - -if not os.path.isdir(CUDA_HOME): - raise OSError(f"Invalid CUDA_HOME: directory does not exist: {CUDA_HOME}") - -cuda_include_dir = os.path.join(CUDA_HOME, "include") - -setup( - name="strings_udf", - version=versioneer.get_version(), - description="Strings UDF Library", - url="https://github.com/rapidsai/cudf", - author="NVIDIA Corporation", - license="Apache 2.0", - classifiers=[ - "Intended Audience :: Developers", - "Topic :: Database", - "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - packages=find_packages(include=["strings_udf", "strings_udf.*"]), - package_data={ - key: ["*.pxd"] for key in find_packages(include=["strings_udf._lib*"]) - }, - cmdclass=versioneer.get_cmdclass(), - install_requires=install_requires, - extras_require=extras_require, - zip_safe=False, -) diff --git a/python/strings_udf/strings_udf/__init__.py b/python/strings_udf/strings_udf/__init__.py deleted file mode 100644 index 66c037125e6..00000000000 --- a/python/strings_udf/strings_udf/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -import os - -from cuda import cudart -from ptxcompiler.patch import NO_DRIVER, safe_get_versions - -from cudf.core.udf.utils import _get_cuda_version_from_ptx_file, _get_ptx_file - -from . import _version - -__version__ = _version.get_versions()["version"] - - -path = os.path.dirname(__file__) - - -# Maximum size of a string column is 2 GiB -_STRINGS_UDF_DEFAULT_HEAP_SIZE = os.environ.get( - "STRINGS_UDF_HEAP_SIZE", 2**31 -) -heap_size = 0 - - -def set_malloc_heap_size(size=None): - """ - Heap size control for strings_udf, size in bytes. - """ - global heap_size - if size is None: - size = _STRINGS_UDF_DEFAULT_HEAP_SIZE - if size != heap_size: - (ret,) = cudart.cudaDeviceSetLimit( - cudart.cudaLimit.cudaLimitMallocHeapSize, size - ) - if ret.value != 0: - raise RuntimeError("Unable to set cudaMalloc heap size") - - heap_size = size - - -ptxpath = None -versions = safe_get_versions() -if versions != NO_DRIVER: - ptxpath = _get_ptx_file(path, "shim_") diff --git a/python/strings_udf/strings_udf/_lib/CMakeLists.txt b/python/strings_udf/strings_udf/_lib/CMakeLists.txt deleted file mode 100644 index 55a33a050e0..00000000000 --- a/python/strings_udf/strings_udf/_lib/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -set(cython_sources cudf_jit_udf.pyx tables.pyx) -set(linked_libraries cudf::cudf cudf_strings_udf) -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" -) diff --git a/python/strings_udf/strings_udf/_lib/__init__.py b/python/strings_udf/strings_udf/_lib/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/strings_udf/strings_udf/_lib/cpp/__init__.pxd b/python/strings_udf/strings_udf/_lib/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/strings_udf/strings_udf/_lib/tables.pyx b/python/strings_udf/strings_udf/_lib/tables.pyx deleted file mode 100644 index 6442a34f63f..00000000000 --- a/python/strings_udf/strings_udf/_lib/tables.pyx +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -from libc.stdint cimport uint8_t, uint16_t, uintptr_t - -from strings_udf._lib.cpp.strings_udf cimport ( - get_character_cases_table as cpp_get_character_cases_table, - get_character_flags_table as cpp_get_character_flags_table, - get_special_case_mapping_table as cpp_get_special_case_mapping_table, -) - -import numpy as np - - -def get_character_flags_table_ptr(): - cdef const uint8_t* tbl_ptr = cpp_get_character_flags_table() - return np.uintp(tbl_ptr) - - -def get_character_cases_table_ptr(): - cdef const uint16_t* tbl_ptr = cpp_get_character_cases_table() - return np.uintp(tbl_ptr) - - -def get_special_case_mapping_table_ptr(): - cdef const void* tbl_ptr = cpp_get_special_case_mapping_table() - return np.uintp(tbl_ptr) diff --git a/python/strings_udf/strings_udf/_typing.py b/python/strings_udf/strings_udf/_typing.py deleted file mode 100644 index fa87ad63dc2..00000000000 --- a/python/strings_udf/strings_udf/_typing.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -import operator - -import numpy as np -from numba import types -from numba.core.extending import models, register_model -from numba.core.typing import signature as nb_signature -from numba.core.typing.templates import AbstractTemplate, AttributeTemplate -from numba.cuda.cudadecl import registry as cuda_decl_registry - -import rmm -from cudf.core.udf.utils import _get_extensionty_size - -# libcudf size_type -size_type = types.int32 - - -# String object definitions -class UDFString(types.Type): - - np_dtype = np.dtype("object") - - def __init__(self): - super().__init__(name="udf_string") - self.size_bytes = _get_extensionty_size(self) - - @property - def return_type(self): - return self - - -class StringView(types.Type): - - np_dtype = np.dtype("object") - - def __init__(self): - super().__init__(name="string_view") - self.size_bytes = _get_extensionty_size(self) - - @property - def return_type(self): - return UDFString() - - -@register_model(StringView) -class stringview_model(models.StructModel): - # from string_view.hpp: - _members = ( - # const char* _data{} - # Pointer to device memory contain char array for this string - ("data", types.CPointer(types.char)), - # size_type _bytes{}; - # Number of bytes in _data for this string - ("bytes", size_type), - # mutable size_type _length{}; - # Number of characters in this string (computed) - ("length", size_type), - ) - - def __init__(self, dmm, fe_type): - super().__init__(dmm, fe_type, self._members) - - -@register_model(UDFString) -class udf_string_model(models.StructModel): - # from udf_string.hpp: - # private: - # char* m_data{}; - # cudf::size_type m_bytes{}; - # cudf::size_type m_size{}; - - _members = ( - ("m_data", types.CPointer(types.char)), - ("m_bytes", size_type), - ("m_size", size_type), - ) - - def __init__(self, dmm, fe_type): - super().__init__(dmm, fe_type, self._members) - - -any_string_ty = (StringView, UDFString, types.StringLiteral) -string_view = StringView() -udf_string = UDFString() - - -class StrViewArgHandler: - """ - As part of Numba's preprocessing step, incoming function arguments are - modified based on the associated type for that argument that was used - to JIT the kernel. However it only knows how to handle built in array - types natively. With string UDFs, the jitted type is string_view*, - which numba does not know how to handle. - - This class converts string_view* to raw pointer arguments, which Numba - knows how to use. - - See numba.cuda.compiler._prepare_args for details. - """ - - def prepare_args(self, ty, val, **kwargs): - if isinstance(ty, types.CPointer) and isinstance( - ty.dtype, (StringView, UDFString) - ): - return types.uint64, val.ptr if isinstance( - val, rmm._lib.device_buffer.DeviceBuffer - ) else val.get_ptr(mode="read") - else: - return ty, val - - -str_view_arg_handler = StrViewArgHandler() - - -# String functions -@cuda_decl_registry.register_global(len) -class StringLength(AbstractTemplate): - """ - provide the length of a cudf::string_view like struct - """ - - def generic(self, args, kws): - if isinstance(args[0], any_string_ty) and len(args) == 1: - # length: - # string_view -> int32 - # udf_string -> int32 - # literal -> int32 - return nb_signature(size_type, args[0]) - - -def register_stringview_binaryop(op, retty): - """ - Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to associate a signature with a function or - operator expecting a string. - """ - - class StringViewBinaryOp(AbstractTemplate): - def generic(self, args, kws): - if isinstance(args[0], any_string_ty) and isinstance( - args[1], any_string_ty - ): - return nb_signature(retty, string_view, string_view) - - cuda_decl_registry.register_global(op)(StringViewBinaryOp) - - -def create_binary_attr(attrname, retty): - """ - Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to register a binary function of two string - objects as an attribute of one, e.g. `string.func(other)`. - """ - - class StringViewBinaryAttr(AbstractTemplate): - key = f"StringView.{attrname}" - - def generic(self, args, kws): - return nb_signature(retty, string_view, recvr=self.this) - - def attr(self, mod): - return types.BoundFunction(StringViewBinaryAttr, string_view) - - return attr - - -def create_identifier_attr(attrname, retty): - """ - Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to register a unary function of a string - object as an attribute, e.g. `string.func()`. - """ - - class StringViewIdentifierAttr(AbstractTemplate): - key = f"StringView.{attrname}" - - def generic(self, args, kws): - return nb_signature(retty, recvr=self.this) - - def attr(self, mod): - return types.BoundFunction(StringViewIdentifierAttr, string_view) - - return attr - - -class StringViewCount(AbstractTemplate): - key = "StringView.count" - - def generic(self, args, kws): - return nb_signature(size_type, string_view, recvr=self.this) - - -class StringViewReplace(AbstractTemplate): - key = "StringView.replace" - - def generic(self, args, kws): - return nb_signature( - udf_string, string_view, string_view, recvr=self.this - ) - - -@cuda_decl_registry.register_attr -class StringViewAttrs(AttributeTemplate): - key = string_view - - def resolve_count(self, mod): - return types.BoundFunction(StringViewCount, string_view) - - def resolve_replace(self, mod): - return types.BoundFunction(StringViewReplace, string_view) - - -# Build attributes for `MaskedType(string_view)` -bool_binary_funcs = ["startswith", "endswith"] -int_binary_funcs = ["find", "rfind"] -id_unary_funcs = [ - "isalpha", - "isalnum", - "isdecimal", - "isdigit", - "isupper", - "islower", - "isspace", - "isnumeric", - "istitle", -] -string_unary_funcs = ["upper", "lower"] -string_return_attrs = ["strip", "lstrip", "rstrip"] - -for func in bool_binary_funcs: - setattr( - StringViewAttrs, - f"resolve_{func}", - create_binary_attr(func, types.boolean), - ) - -for func in string_return_attrs: - setattr( - StringViewAttrs, - f"resolve_{func}", - create_binary_attr(func, udf_string), - ) - - -for func in int_binary_funcs: - setattr( - StringViewAttrs, f"resolve_{func}", create_binary_attr(func, size_type) - ) - -for func in id_unary_funcs: - setattr( - StringViewAttrs, - f"resolve_{func}", - create_identifier_attr(func, types.boolean), - ) - -for func in string_unary_funcs: - setattr( - StringViewAttrs, - f"resolve_{func}", - create_identifier_attr(func, udf_string), - ) - -cuda_decl_registry.register_attr(StringViewAttrs) - -register_stringview_binaryop(operator.eq, types.boolean) -register_stringview_binaryop(operator.ne, types.boolean) -register_stringview_binaryop(operator.lt, types.boolean) -register_stringview_binaryop(operator.gt, types.boolean) -register_stringview_binaryop(operator.le, types.boolean) -register_stringview_binaryop(operator.ge, types.boolean) - -# st in other -register_stringview_binaryop(operator.contains, types.boolean) - -# st + other -register_stringview_binaryop(operator.add, udf_string) diff --git a/python/strings_udf/strings_udf/_version.py b/python/strings_udf/strings_udf/_version.py deleted file mode 100644 index 14ff9ec314d..00000000000 --- a/python/strings_udf/strings_udf/_version.py +++ /dev/null @@ -1,711 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.23 (https://github.com/python-versioneer/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import functools -import os -import re -import subprocess -import sys -from typing import Callable, Dict - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "v" - cfg.parentdir_prefix = "strings_udf-" - cfg.versionfile_source = "strings_udf/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen( - [command] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - **popen_kwargs, - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r"\d", r): - continue - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - f"{tag_prefix}[[:digit:]]*", - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner( - GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root - ) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post( - pieces["closest-tag"] - ) - rendered = tag_version - if post_version is not None: - rendered += ".post%d.dev%d" % ( - post_version + 1, - pieces["distance"], - ) - else: - rendered += ".post0.dev%d" % (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords( - get_keywords(), cfg.tag_prefix, verbose - ) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/python/strings_udf/strings_udf/lowering.py b/python/strings_udf/strings_udf/lowering.py deleted file mode 100644 index afc96a8380a..00000000000 --- a/python/strings_udf/strings_udf/lowering.py +++ /dev/null @@ -1,517 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -import operator -from functools import partial - -from numba import cuda, types -from numba.core import cgutils -from numba.core.datamodel import default_manager -from numba.core.typing import signature as nb_signature -from numba.cuda.cudadrv import nvvm -from numba.cuda.cudaimpl import ( - lower as cuda_lower, - registry as cuda_lowering_registry, -) - -from strings_udf._lib.tables import ( - get_character_cases_table_ptr, - get_character_flags_table_ptr, - get_special_case_mapping_table_ptr, -) -from strings_udf._typing import size_type, string_view, udf_string - -_STR_VIEW_PTR = types.CPointer(string_view) -_UDF_STRING_PTR = types.CPointer(udf_string) - - -# CUDA function declarations -# read-only (input is a string_view, output is a fixed with type) -_string_view_len = cuda.declare_device("len", size_type(_STR_VIEW_PTR)) -_concat_string_view = cuda.declare_device( - "concat", types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR) -) - -_string_view_replace = cuda.declare_device( - "replace", - types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), -) - - -def _declare_binary_func(lhs, rhs, out, name): - # Declare a binary function - return cuda.declare_device( - name, - out(lhs, rhs), - ) - - -def _declare_strip_func(name): - return cuda.declare_device( - name, size_type(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR) - ) - - -# A binary function of the form f(string, string) -> bool -_declare_bool_str_str_func = partial( - _declare_binary_func, _STR_VIEW_PTR, _STR_VIEW_PTR, types.boolean -) - -_declare_size_type_str_str_func = partial( - _declare_binary_func, _STR_VIEW_PTR, _STR_VIEW_PTR, size_type -) - -_string_view_contains = _declare_bool_str_str_func("contains") -_string_view_eq = _declare_bool_str_str_func("eq") -_string_view_ne = _declare_bool_str_str_func("ne") -_string_view_ge = _declare_bool_str_str_func("ge") -_string_view_le = _declare_bool_str_str_func("le") -_string_view_gt = _declare_bool_str_str_func("gt") -_string_view_lt = _declare_bool_str_str_func("lt") -_string_view_startswith = _declare_bool_str_str_func("startswith") -_string_view_endswith = _declare_bool_str_str_func("endswith") -_string_view_find = _declare_size_type_str_str_func("find") -_string_view_rfind = _declare_size_type_str_str_func("rfind") -_string_view_contains = _declare_bool_str_str_func("contains") -_string_view_strip = _declare_strip_func("strip") -_string_view_lstrip = _declare_strip_func("lstrip") -_string_view_rstrip = _declare_strip_func("rstrip") - - -# A binary function of the form f(string, int) -> bool -_declare_bool_str_int_func = partial( - _declare_binary_func, _STR_VIEW_PTR, types.int64, types.boolean -) - - -def _declare_upper_or_lower(func): - return cuda.declare_device( - func, - types.void( - _UDF_STRING_PTR, - _STR_VIEW_PTR, - types.uintp, - types.uintp, - types.uintp, - ), - ) - - -_string_view_isdigit = _declare_bool_str_int_func("pyisdigit") -_string_view_isalnum = _declare_bool_str_int_func("pyisalnum") -_string_view_isalpha = _declare_bool_str_int_func("pyisalpha") -_string_view_isdecimal = _declare_bool_str_int_func("pyisdecimal") -_string_view_isnumeric = _declare_bool_str_int_func("pyisnumeric") -_string_view_isspace = _declare_bool_str_int_func("pyisspace") -_string_view_isupper = _declare_bool_str_int_func("pyisupper") -_string_view_islower = _declare_bool_str_int_func("pyislower") -_string_view_istitle = _declare_bool_str_int_func("pyistitle") -_string_view_upper = _declare_upper_or_lower("upper") -_string_view_lower = _declare_upper_or_lower("lower") - - -_string_view_count = cuda.declare_device( - "pycount", - size_type(_STR_VIEW_PTR, _STR_VIEW_PTR), -) - - -# casts -@cuda_lowering_registry.lower_cast(types.StringLiteral, string_view) -def cast_string_literal_to_string_view(context, builder, fromty, toty, val): - """ - Cast a literal to a string_view - """ - # create an empty string_view - sv = cgutils.create_struct_proxy(string_view)(context, builder) - - # set the empty strview data pointer to point to the literal value - s = context.insert_const_string(builder.module, fromty.literal_value) - sv.data = context.insert_addrspace_conv( - builder, s, nvvm.ADDRSPACE_CONSTANT - ) - sv.length = context.get_constant(size_type, len(fromty.literal_value)) - sv.bytes = context.get_constant( - size_type, len(fromty.literal_value.encode("UTF-8")) - ) - - return sv._getvalue() - - -@cuda_lowering_registry.lower_cast(string_view, udf_string) -def cast_string_view_to_udf_string(context, builder, fromty, toty, val): - sv_ptr = builder.alloca(default_manager[fromty].get_value_type()) - udf_str_ptr = builder.alloca(default_manager[toty].get_value_type()) - builder.store(val, sv_ptr) - _ = context.compile_internal( - builder, - call_create_udf_string_from_string_view, - nb_signature(types.void, _STR_VIEW_PTR, types.CPointer(udf_string)), - (sv_ptr, udf_str_ptr), - ) - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - - return result._getvalue() - - -# utilities -_create_udf_string_from_string_view = cuda.declare_device( - "udf_string_from_string_view", - types.void(types.CPointer(string_view), types.CPointer(udf_string)), -) - - -def call_create_udf_string_from_string_view(sv, udf_str): - _create_udf_string_from_string_view(sv, udf_str) - - -# String function implementations -def call_len_string_view(st): - return _string_view_len(st) - - -@cuda_lower(len, string_view) -def len_impl(context, builder, sig, args): - sv_ptr = builder.alloca(args[0].type) - builder.store(args[0], sv_ptr) - result = context.compile_internal( - builder, - call_len_string_view, - nb_signature(size_type, _STR_VIEW_PTR), - (sv_ptr,), - ) - - return result - - -def call_concat_string_view(result, lhs, rhs): - return _concat_string_view(result, lhs, rhs) - - -@cuda_lower(operator.add, string_view, string_view) -def concat_impl(context, builder, sig, args): - lhs_ptr = builder.alloca(args[0].type) - rhs_ptr = builder.alloca(args[1].type) - builder.store(args[0], lhs_ptr) - builder.store(args[1], rhs_ptr) - - udf_str_ptr = builder.alloca(default_manager[udf_string].get_value_type()) - _ = context.compile_internal( - builder, - call_concat_string_view, - types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), - (udf_str_ptr, lhs_ptr, rhs_ptr), - ) - - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - return result._getvalue() - - -def call_string_view_replace(result, src, to_replace, replacement): - return _string_view_replace(result, src, to_replace, replacement) - - -@cuda_lower("StringView.replace", string_view, string_view, string_view) -def replace_impl(context, builder, sig, args): - src_ptr = builder.alloca(args[0].type) - to_replace_ptr = builder.alloca(args[1].type) - replacement_ptr = builder.alloca(args[2].type) - - builder.store(args[0], src_ptr) - builder.store(args[1], to_replace_ptr), - builder.store(args[2], replacement_ptr) - - udf_str_ptr = builder.alloca(default_manager[udf_string].get_value_type()) - - _ = context.compile_internal( - builder, - call_string_view_replace, - types.void( - _UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR - ), - (udf_str_ptr, src_ptr, to_replace_ptr, replacement_ptr), - ) - - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - return result._getvalue() - - -def create_binary_string_func(binary_func, retty): - """ - Provide a wrapper around numba's low-level extension API which - produces the boilerplate needed to implement a binary function - of two strings. - """ - - def deco(cuda_func): - @cuda_lower(binary_func, string_view, string_view) - def binary_func_impl(context, builder, sig, args): - lhs_ptr = builder.alloca(args[0].type) - rhs_ptr = builder.alloca(args[1].type) - builder.store(args[0], lhs_ptr) - builder.store(args[1], rhs_ptr) - - # these conditional statements should compile out - if retty != udf_string: - # binary function of two strings yielding a fixed-width type - # example: str.startswith(other) -> bool - # shim functions can return the value through nb_retval - result = context.compile_internal( - builder, - cuda_func, - nb_signature(retty, _STR_VIEW_PTR, _STR_VIEW_PTR), - (lhs_ptr, rhs_ptr), - ) - return result - else: - # binary function of two strings yielding a new string - # example: str.strip(other) -> str - # shim functions can not return a struct due to C linkage - # so we create a new udf_string and pass a pointer to it - # for the shim function to write the output to. The return - # value of compile_internal is therefore discarded (although - # this may change in the future if we need to return error - # codes, for instance). - udf_str_ptr = builder.alloca( - default_manager[udf_string].get_value_type() - ) - _ = context.compile_internal( - builder, - cuda_func, - size_type(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), - (udf_str_ptr, lhs_ptr, rhs_ptr), - ) - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - return result._getvalue() - - return binary_func_impl - - return deco - - -@create_binary_string_func(operator.contains, types.boolean) -def contains_impl(st, substr): - return _string_view_contains(st, substr) - - -@create_binary_string_func(operator.eq, types.boolean) -def eq_impl(st, rhs): - return _string_view_eq(st, rhs) - - -@create_binary_string_func(operator.ne, types.boolean) -def ne_impl(st, rhs): - return _string_view_ne(st, rhs) - - -@create_binary_string_func(operator.ge, types.boolean) -def ge_impl(st, rhs): - return _string_view_ge(st, rhs) - - -@create_binary_string_func(operator.le, types.boolean) -def le_impl(st, rhs): - return _string_view_le(st, rhs) - - -@create_binary_string_func(operator.gt, types.boolean) -def gt_impl(st, rhs): - return _string_view_gt(st, rhs) - - -@create_binary_string_func(operator.lt, types.boolean) -def lt_impl(st, rhs): - return _string_view_lt(st, rhs) - - -@create_binary_string_func("StringView.strip", udf_string) -def strip_impl(result, to_strip, strip_char): - return _string_view_strip(result, to_strip, strip_char) - - -@create_binary_string_func("StringView.lstrip", udf_string) -def lstrip_impl(result, to_strip, strip_char): - return _string_view_lstrip(result, to_strip, strip_char) - - -@create_binary_string_func("StringView.rstrip", udf_string) -def rstrip_impl(result, to_strip, strip_char): - return _string_view_rstrip(result, to_strip, strip_char) - - -@create_binary_string_func("StringView.startswith", types.boolean) -def startswith_impl(sv, substr): - return _string_view_startswith(sv, substr) - - -@create_binary_string_func("StringView.endswith", types.boolean) -def endswith_impl(sv, substr): - return _string_view_endswith(sv, substr) - - -@create_binary_string_func("StringView.count", size_type) -def count_impl(st, substr): - return _string_view_count(st, substr) - - -@create_binary_string_func("StringView.find", size_type) -def find_impl(sv, substr): - return _string_view_find(sv, substr) - - -@create_binary_string_func("StringView.rfind", size_type) -def rfind_impl(sv, substr): - return _string_view_rfind(sv, substr) - - -def create_unary_identifier_func(id_func): - """ - Provide a wrapper around numba's low-level extension API which - produces the boilerplate needed to implement a unary function - of a string. - """ - - def deco(cuda_func): - @cuda_lower(id_func, string_view) - def id_func_impl(context, builder, sig, args): - str_ptr = builder.alloca(args[0].type) - builder.store(args[0], str_ptr) - - # Lookup table required for conversion functions - # must be resolved at runtime after context initialization, - # therefore cannot be a global variable - tbl_ptr = context.get_constant( - types.uintp, get_character_flags_table_ptr() - ) - result = context.compile_internal( - builder, - cuda_func, - nb_signature(types.boolean, _STR_VIEW_PTR, types.uintp), - (str_ptr, tbl_ptr), - ) - - return result - - return id_func_impl - - return deco - - -def create_upper_or_lower(id_func): - """ - Provide a wrapper around numba's low-level extension API which - produces the boilerplate needed to implement either the upper - or lower attrs of a string view. - """ - - def deco(cuda_func): - @cuda_lower(id_func, string_view) - def id_func_impl(context, builder, sig, args): - str_ptr = builder.alloca(args[0].type) - builder.store(args[0], str_ptr) - - # Lookup table required for conversion functions - # must be resolved at runtime after context initialization, - # therefore cannot be a global variable - flags_tbl_ptr = context.get_constant( - types.uintp, get_character_flags_table_ptr() - ) - cases_tbl_ptr = context.get_constant( - types.uintp, get_character_cases_table_ptr() - ) - special_tbl_ptr = context.get_constant( - types.uintp, get_special_case_mapping_table_ptr() - ) - udf_str_ptr = builder.alloca( - default_manager[udf_string].get_value_type() - ) - - _ = context.compile_internal( - builder, - cuda_func, - types.void( - _UDF_STRING_PTR, - _STR_VIEW_PTR, - types.uintp, - types.uintp, - types.uintp, - ), - ( - udf_str_ptr, - str_ptr, - flags_tbl_ptr, - cases_tbl_ptr, - special_tbl_ptr, - ), - ) - - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - return result._getvalue() - - return id_func_impl - - return deco - - -@create_upper_or_lower("StringView.upper") -def upper_impl(result, st, flags, cases, special): - return _string_view_upper(result, st, flags, cases, special) - - -@create_upper_or_lower("StringView.lower") -def lower_impl(result, st, flags, cases, special): - return _string_view_lower(result, st, flags, cases, special) - - -@create_unary_identifier_func("StringView.isdigit") -def isdigit_impl(st, tbl): - return _string_view_isdigit(st, tbl) - - -@create_unary_identifier_func("StringView.isalnum") -def isalnum_impl(st, tbl): - return _string_view_isalnum(st, tbl) - - -@create_unary_identifier_func("StringView.isalpha") -def isalpha_impl(st, tbl): - return _string_view_isalpha(st, tbl) - - -@create_unary_identifier_func("StringView.isnumeric") -def isnumeric_impl(st, tbl): - return _string_view_isnumeric(st, tbl) - - -@create_unary_identifier_func("StringView.isdecimal") -def isdecimal_impl(st, tbl): - return _string_view_isdecimal(st, tbl) - - -@create_unary_identifier_func("StringView.isspace") -def isspace_impl(st, tbl): - return _string_view_isspace(st, tbl) - - -@create_unary_identifier_func("StringView.isupper") -def isupper_impl(st, tbl): - return _string_view_isupper(st, tbl) - - -@create_unary_identifier_func("StringView.islower") -def islower_impl(st, tbl): - return _string_view_islower(st, tbl) - - -@create_unary_identifier_func("StringView.istitle") -def istitle_impl(st, tbl): - return _string_view_istitle(st, tbl) diff --git a/python/strings_udf/versioneer.py b/python/strings_udf/versioneer.py deleted file mode 100644 index 6194b6a5698..00000000000 --- a/python/strings_udf/versioneer.py +++ /dev/null @@ -1,2245 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# Version: 0.23 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/python-versioneer/python-versioneer -* Brian Warner -* License: Public Domain (CC0-1.0) -* Compatible with: Python 3.7, 3.8, 3.9, 3.10 and pypy3 -* [![Latest Version][pypi-image]][pypi-url] -* [![Build Status][travis-image]][travis-url] - -This is a tool for managing a recorded version number in -distutils/setuptools-based python projects. The goal is to -remove the tedious and error-prone "update the embedded version string" -step from your release process. Making a new release should be as easy -as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere in your $PATH -* add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md)) -* run `versioneer install` in your source tree, commit the results -* Verify version information with `python setup.py version` - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes). - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/python-versioneer/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other languages) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) -is tracking this issue. The discussion in -[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) -describes the issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) -describes this one, but upgrading to a newer version of setuptools should -probably resolve it. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - -## Similar projects - -* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored - build-time dependency -* [minver](https://github.com/jbweston/miniver) - a lightweight - reimplementation of versioneer -* [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based - setuptools plugin - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg -[pypi-url]: https://pypi.python.org/pypi/versioneer/ -[travis-image]: -https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg -[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer - -""" -# pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring -# pylint:disable=missing-class-docstring,too-many-branches,too-many-statements -# pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error -# pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with -# pylint:disable=attribute-defined-outside-init,too-many-arguments - -import configparser -import errno -import functools -import json -import os -import re -import subprocess -import sys -from typing import Callable, Dict - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - my_path = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(my_path)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise OSError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.ConfigParser() - with open(setup_cfg, "r") as cfg_file: - parser.read_file(cfg_file) - VCS = parser.get("versioneer", "VCS") # mandatory - - # Dict-like interface for non-mandatory entries - section = parser["versioneer"] - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = section.get("style", "") - cfg.versionfile_source = section.get("versionfile_source") - cfg.versionfile_build = section.get("versionfile_build") - cfg.tag_prefix = section.get("tag_prefix") - if cfg.tag_prefix in ("''", '""', None): - cfg.tag_prefix = "" - cfg.parentdir_prefix = section.get("parentdir_prefix") - cfg.verbose = section.get("verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - HANDLERS.setdefault(vcs, {})[method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen( - [command] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - **popen_kwargs, - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -LONG_VERSION_PY[ - "git" -] = r''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.23 (https://github.com/python-versioneer/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys -from typing import Callable, Dict -import functools - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post( - pieces["closest-tag"] - ) - rendered = tag_version - if post_version is not None: - rendered += ".post%%d.dev%%d" %% ( - post_version + 1, pieces["distance"] - ) - else: - rendered += ".post0.dev%%d" %% (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r"\d", r): - continue - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - f"{tag_prefix}[[:digit:]]*", - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner( - GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root - ) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [versionfile_source] - if ipy: - files.append(ipy) - try: - my_path = __file__ - if my_path.endswith(".pyc") or my_path.endswith(".pyo"): - my_path = os.path.splitext(my_path)[0] + ".py" - versioneer_file = os.path.relpath(my_path) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - with open(".gitattributes", "r") as fobj: - for line in fobj: - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - break - except OSError: - pass - if not present: - with open(".gitattributes", "a+") as fobj: - fobj.write(f"{versionfile_source} export-subst\n") - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.23) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except OSError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps( - versions, sort_keys=True, indent=1, separators=(",", ": ") - ) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post( - pieces["closest-tag"] - ) - rendered = tag_version - if post_version is not None: - rendered += ".post%d.dev%d" % ( - post_version + 1, - pieces["distance"], - ) - else: - rendered += ".post0.dev%d" % (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(cmdclass=None): - """Get the custom setuptools subclasses used by Versioneer. - - If the package uses a different cmdclass (e.g. one from numpy), it - should be provide as an argument. - """ - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see - # https://github.com/python-versioneer/python-versioneer/issues/52 - - cmds = {} if cmdclass is None else cmdclass.copy() - - # we add "version" to setuptools - from setuptools import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # pip install -e . and setuptool/editable_wheel will invoke build_py - # but the build_py command is not expected to copy any files. - - # we override different "build_py" commands for both environments - if "build_py" in cmds: - _build_py = cmds["build_py"] - else: - from setuptools.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - if getattr(self, "editable_mode", False): - # During editable installs `.py` and data files are - # not copied to build_lib - return - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "build_ext" in cmds: - _build_ext = cmds["build_ext"] - else: - from setuptools.command.build_ext import build_ext as _build_ext - - class cmd_build_ext(_build_ext): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_ext.run(self) - if self.inplace: - # build_ext --inplace will only build extensions in - # build/lib<..> dir with no _version.py to write to. - # As in place builds will already have a _version.py - # in the module dir, we do not need to write one. - return - # now locate _version.py in the new build/ directory and replace - # it with an updated value - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - if not os.path.exists(target_versionfile): - print( - f"Warning: {target_versionfile} does not exist, skipping " - "version update. This can happen if you are running " - "build_ext without first running build_py." - ) - return - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_ext"] = cmd_build_ext - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - from py2exe.distutils_buildexe import py2exe as _py2exe - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # sdist farms its file list building out to egg_info - if "egg_info" in cmds: - _sdist = cmds["egg_info"] - else: - from setuptools.command.egg_info import egg_info as _egg_info - - class cmd_egg_info(_egg_info): - def find_sources(self): - # egg_info.find_sources builds the manifest list and writes it - # in one shot - super().find_sources() - - # Modify the filelist and normalize it - root = get_root() - cfg = get_config_from_root(root) - self.filelist.append("versioneer.py") - if cfg.versionfile_source: - # There are rare cases where versionfile_source might not be - # included by default, so we must be explicit - self.filelist.append(cfg.versionfile_source) - self.filelist.sort() - self.filelist.remove_duplicates() - - # The write method is hidden in the manifest_maker instance that - # generated the filelist and was thrown away - # We will instead replicate their final normalization (to unicode, - # and POSIX-style paths) - from setuptools import unicode_utils - - normalized = [ - unicode_utils.filesys_decode(f).replace(os.sep, "/") - for f in self.filelist.files - ] - - manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") - with open(manifest_filename, "w") as fobj: - fobj.write("\n".join(normalized)) - - cmds["egg_info"] = cmd_egg_info - - # we override different "sdist" commands for both environments - if "sdist" in cmds: - _sdist = cmds["sdist"] - else: - from setuptools.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -OLD_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - -INIT_PY_SNIPPET = """ -from . import {0} -__version__ = {0}.get_versions()['version'] -""" - - -def do_setup(): - """Do main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - OSError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (OSError, configparser.NoSectionError)): - print( - "Adding sample versioneer config to setup.cfg", file=sys.stderr - ) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except OSError: - old = "" - module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] - snippet = INIT_PY_SNIPPET.format(module) - if OLD_SNIPPET in old: - print(" replacing boilerplate in %s" % ipy) - with open(ipy, "w") as f: - f.write(old.replace(OLD_SNIPPET, snippet)) - elif snippet not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(snippet) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1)