Merge remote-tracking branch 'origin/branch-22.12' into wence/fix/iss…

…ue-12072
rapidsai · Nov 15, 2022 · 56cd889 · 56cd889
2 parents e55c9f0 + b2e5069
commit 56cd889
Show file tree

Hide file tree

Showing 213 changed files with 7,468 additions and 4,403 deletions.
diff --git a/.github/workflows/dependency-files.yml b/.github/workflows/dependency-files.yml
@@ -0,0 +1,12 @@
+name: pr
+
+on:
+  pull_request:
+
+jobs:
+  checks:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main
+    with:
+      enable_check_size: false
+      enable_check_style: false
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -99,13 +99,13 @@ cd $CUDF_HOME
 **Note:** Using a conda environment is the easiest way to satisfy the library's dependencies.
 Instructions for a minimal build environment without conda are included below.
 
-- Create the conda development environment `cudf_dev`:
+- Create the conda development environment:
 
 ```bash
 # create the conda environment (assuming in base `cudf` directory)
 # note: RAPIDS currently doesn't support `channel_priority: strict`;
 # use `channel_priority: flexible` instead
-conda env create --name cudf_dev --file conda/environments/cudf_dev_cuda11.5.yml
+conda env create --name cudf_dev --file conda/environments/all_cuda-115_arch-x86_64.yaml
 # activate the environment
 conda activate cudf_dev
 ```
@@ -114,9 +114,6 @@ conda activate cudf_dev
   development environment may also need to be updated if dependency versions or
   pinnings are changed.
 
-- For other CUDA versions, check the corresponding `cudf_dev_cuda*.yml` file in
-  `conda/environments/`.
-
 #### Building without a conda environment
 
 - libcudf has the following minimal dependencies (in addition to those listed in the [General
@@ -382,7 +379,7 @@ You can skip these checks with `git commit --no-verify` or with the short versio
 
 ## Developer Guidelines
 
-The [C++ Developer Guide](cpp/docs/DEVELOPER_GUIDE.md) includes details on contributing to libcudf C++ code.
+The [C++ Developer Guide](cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md) includes details on contributing to libcudf C++ code.
 
 The [Python Developer Guide](https://docs.rapids.ai/api/cudf/stable/developer_guide/index.html) includes details on contributing to cuDF Python code.
 

diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh
@@ -82,8 +82,8 @@ conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \
 
 # Install the conda-forge or nightly version of dask and distributed
 if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
-    gpuci_logger "gpuci_mamba_retry update dask"
-    gpuci_mamba_retry update dask
+    gpuci_logger "gpuci_mamba_retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'"
+    gpuci_mamba_retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed"
 else
     gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall"
     gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall

diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh
@@ -96,8 +96,8 @@ function install_dask {
     gpuci_logger "Install the conda-forge or nightly version of dask and distributed"
     set -x
     if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
-        gpuci_logger "gpuci_mamba_retry update dask"
-        gpuci_mamba_retry update dask
+        gpuci_logger "gpuci_mamba_retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'"
+        gpuci_mamba_retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed"
         conda list
     else
         gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall"
@@ -111,6 +111,8 @@ function install_dask {
     set +x
 }
 
+install_dask
+
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
 
     gpuci_logger "Install dependencies"
@@ -126,8 +128,6 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     # gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env
     # gpuci_mamba_retry install -y "your-pkg=1.0.0"
 
-    install_dask
-
     ################################################################################
     # BUILD - Build libcudf, cuDF, libcudf_kafka, dask_cudf, and strings_udf from source
     ################################################################################

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -63,9 +63,10 @@ sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/cudf/source
 sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/cudf/source/conf.py
 
 # bump rmm & dask-cuda
-for FILE in conda/environments/*.yml; do
-  sed_runner "s/rmm=${CURRENT_SHORT_TAG}/rmm=${NEXT_SHORT_TAG}/g" ${FILE};
+for FILE in conda/environments/*.yaml dependencies.yaml; do
   sed_runner "s/dask-cuda=${CURRENT_SHORT_TAG}/dask-cuda=${NEXT_SHORT_TAG}/g" ${FILE};
+  sed_runner "s/rmm=${CURRENT_SHORT_TAG}/rmm=${NEXT_SHORT_TAG}/g" ${FILE};
+  sed_runner "s/rmm-cu11=${CURRENT_SHORT_TAG}/rmm-cu11=${NEXT_SHORT_TAG}/g" ${FILE};
 done
 
 # Doxyfile update

diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml
@@ -0,0 +1,78 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- rapidsai
+- rapidsai-nightly
+- dask/label/dev
+- conda-forge
+- nvidia
+dependencies:
+- aiobotocore>=2.2.0
+- arrow-cpp=9
+- boto3>=1.21.21
+- botocore>=1.24.21
+- c-compiler
+- cachetools
+- cmake>=3.23.1
+- cubinlinker
+- cuda-python>=11.7.1,<12.0
+- cudatoolkit=11.5
+- cupy>=9.5.0,<12.0.0a0
+- cxx-compiler
+- cython>=0.29,<0.30
+- dask-cuda=22.12.*
+- dask>=2022.9.2
+- distributed>=2022.9.2
+- dlpack>=0.5,<0.6.0a0
+- doxygen=1.8.20
+- fastavro>=0.22.9
+- fsspec>=0.6.0
+- gcc_linux-64=9.*
+- hypothesis
+- ipython
+- librdkafka=1.7.0
+- mimesis>=4.1.0
+- moto>=4.0.8
+- myst-nb
+- nbsphinx
+- notebook>=0.5.0
+- numba>=0.56.2
+- numpy
+- numpydoc
+- nvcc_linux-64=11.5
+- nvtx>=0.2.1
+- packaging
+- pandas>=1.0,<1.6.0dev0
+- pandoc<=2.0.0
+- pip
+- pre-commit
+- protobuf>=3.20.1,<3.21.0a0
+- ptxcompiler
+- pyarrow=9.0.0
+- pydata-sphinx-theme
+- pytest
+- pytest-benchmark
+- pytest-cases
+- pytest-cov
+- pytest-xdist
+- python-confluent-kafka=1.7.0
+- python-snappy>=0.6.0
+- python>=3.8,<3.10
+- pytorch<1.12.0
+- rmm=22.12.*
+- s3fs>=2022.3.0
+- scikit-build>=0.13.1
+- scipy
+- sphinx
+- sphinx-autobuild
+- sphinx-copybutton
+- sphinx-markdown-tables
+- sphinxcontrib-websupport
+- streamz
+- sysroot_linux-64==2.17
+- transformers
+- typing_extensions
+- pip:
+  - git+https://github.com/python-streamz/streamz.git@master
+  - pyorc
+name: all_cuda-115_arch-x86_64
diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml
diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml
@@ -236,6 +236,7 @@ outputs:
         - test -f $PREFIX/include/cudf/strings/json.hpp
         - test -f $PREFIX/include/cudf/strings/padding.hpp
         - test -f $PREFIX/include/cudf/strings/regex/flags.hpp
+        - test -f $PREFIX/include/cudf/strings/regex/regex_program.hpp
         - test -f $PREFIX/include/cudf/strings/repeat_strings.hpp
         - test -f $PREFIX/include/cudf/strings/replace.hpp
         - test -f $PREFIX/include/cudf/strings/replace_re.hpp
@@ -254,7 +255,7 @@ outputs:
         - test -f $PREFIX/include/cudf/structs/structs_column_view.hpp
         - test -f $PREFIX/include/cudf/table/table.hpp
         - test -f $PREFIX/include/cudf/table/table_view.hpp
-        - test -f $PREFIX/include/cudf/tdigest/tdigest_column_view.cuh
+        - test -f $PREFIX/include/cudf/tdigest/tdigest_column_view.hpp
         - test -f $PREFIX/include/cudf/transform.hpp
         - test -f $PREFIX/include/cudf/transpose.hpp
         - test -f $PREFIX/include/cudf/types.hpp

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -69,6 +69,8 @@ option(CUDA_ENABLE_LINEINFO
 option(CUDA_WARNINGS_AS_ERRORS "Enable -Werror=all-warnings for all CUDA compilation" ON)
 # cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic linking
 option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF)
+option(USE_LIBARROW_FROM_PYARROW "Use the libarrow contained within pyarrow." OFF)
+mark_as_advanced(USE_LIBARROW_FROM_PYARROW)
 
 message(VERBOSE "CUDF: Build with NVTX support: ${USE_NVTX}")
 message(VERBOSE "CUDF: Configure CMake to build tests: ${BUILD_TESTS}")
@@ -344,7 +346,10 @@ add_library(
   src/io/parquet/chunk_dict.cu
   src/io/parquet/page_enc.cu
   src/io/parquet/page_hdr.cu
-  src/io/parquet/reader_impl.cu
+  src/io/parquet/reader.cpp
+  src/io/parquet/reader_impl.cpp
+  src/io/parquet/reader_impl_helpers.cpp
+  src/io/parquet/reader_impl_preprocess.cu
   src/io/parquet/writer_impl.cu
   src/io/statistics/orc_column_statistics.cu
   src/io/statistics/parquet_column_statistics.cu
@@ -499,7 +504,8 @@ add_library(
   src/strings/padding.cu
   src/strings/json/json_path.cu
   src/strings/regex/regcomp.cpp
-  src/strings/regex/regexec.cu
+  src/strings/regex/regexec.cpp
+  src/strings/regex/regex_program.cpp
   src/strings/repeat_strings.cu
   src/strings/replace/backref_re.cu
   src/strings/replace/multi_re.cu
@@ -690,10 +696,10 @@ add_library(cudf::cudf ALIAS cudf)
 add_library(
   cudftestutil STATIC
   tests/io/metadata_utilities.cpp
-  tests/quantiles/tdigest_utilities.cu
   tests/utilities/base_fixture.cpp
   tests/utilities/column_utilities.cu
   tests/utilities/table_utilities.cu
+  tests/utilities/tdigest_utilities.cu
 )
 
 set_target_properties(

diff --git a/cpp/benchmarks/join/conditional_join.cu b/cpp/benchmarks/join/conditional_join.cu
@@ -70,7 +70,7 @@ CONDITIONAL_LEFT_JOIN_BENCHMARK_DEFINE(conditional_left_join_64bit_nulls, int64_
                    cudf::table_view const& right,                                      \
                    cudf::ast::operation binary_pred,                                   \
                    cudf::null_equality compare_nulls) {                                \
-      return cudf::conditional_inner_join(left, right, binary_pred);                   \
+      return cudf::conditional_full_join(left, right, binary_pred);                    \
     };                                                                                 \
     constexpr bool is_conditional = true;                                              \
     BM_join<key_type, payload_type, nullable, is_conditional>(st, join);               \

diff --git a/cpp/benchmarks/string/json.cu b/cpp/benchmarks/string/json.cu
@@ -177,8 +177,8 @@ auto build_json_string_column(int desired_bytes, int num_rows)
   auto d_store_order = cudf::column_device_view::create(float_2bool_columns->get_column(2));
   json_benchmark_row_builder jb{
     desired_bytes, num_rows, {*d_books, *d_bicycles}, *d_book_pct, *d_misc_order, *d_store_order};
-  auto children =
-    cudf::strings::detail::make_strings_children(jb, num_rows, cudf::get_default_stream());
+  auto children = cudf::strings::detail::make_strings_children(
+    jb, num_rows, cudf::get_default_stream(), rmm::mr::get_current_device_resource());
   return cudf::make_strings_column(
     num_rows, std::move(children.first), std::move(children.second), 0, {});
 }