Merge pull request #11801 from davidwendt/branch-22.12-merge-22.10

Merge branch-22.10 into branch-22.12

CONTRIBUTING.md

@@ -73,7 +73,7 @@ Compilers:
 
 * `gcc` version 9.3+
 * `nvcc` version 11.5+
-* `cmake` version 3.20.1+
+* `cmake` version 3.23.1+
 
 CUDA/GPU:
 

ci/benchmark/build.sh

@@ -39,6 +39,9 @@ export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 # Dask & Distributed option to install main(nightly) or `conda-forge` packages.
 export INSTALL_DASK_MAIN=1
 
+# Dask version to install when `INSTALL_DASK_MAIN=0`
+export DASK_STABLE_VERSION="2022.7.1"
+
 function remove_libcudf_kernel_cache_dir {
     EXITCODE=$?
     logger "removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH"
@@ -82,8 +85,8 @@ if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
     gpuci_logger "gpuci_mamba_retry update dask"
     gpuci_mamba_retry update dask
 else
-    gpuci_logger "gpuci_mamba_retry install conda-forge::dask>=2022.7.1 conda-forge::distributed>=2022.7.1 conda-forge::dask-core>=2022.7.1 --force-reinstall"
-    gpuci_mamba_retry install conda-forge::dask>=2022.7.1 conda-forge::distributed>=2022.7.1 conda-forge::dask-core>=2022.7.1 --force-reinstall
+    gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall"
+    gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall
 fi
 
 # Install the master version of streamz

ci/cpu/build.sh

@@ -83,10 +83,17 @@ if [ "$BUILD_LIBCUDF" == '1' ]; then
 
   # BUILD_LIBCUDF == 1 means this job is being run on the cpu_build jobs
   # that is where we must also build the strings_udf package
+  mkdir -p ${CONDA_BLD_DIR}/strings_udf/work
+  STRINGS_UDF_BUILD_DIR=${CONDA_BLD_DIR}/strings_udf/work
+  gpuci_logger "Build conda pkg for cudf (python 3.8), for strings_udf"
+  gpuci_conda_retry mambabuild --no-build-id --croot ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/cudf ${CONDA_BUILD_ARGS} --python=3.8
+  gpuci_logger "Build conda pkg for cudf (python 3.9), for strings_udf"
+  gpuci_conda_retry mambabuild --no-build-id --croot ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/cudf ${CONDA_BUILD_ARGS} --python=3.9
+
   gpuci_logger "Build conda pkg for strings_udf (python 3.8)"
-  gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/strings_udf $CONDA_BUILD_ARGS --python=3.8
+  gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} -c ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/strings_udf $CONDA_BUILD_ARGS --python=3.8
   gpuci_logger "Build conda pkg for strings_udf (python 3.9)"
-  gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/strings_udf $CONDA_BUILD_ARGS --python=3.9
+  gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} -c ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/strings_udf $CONDA_BUILD_ARGS --python=3.9
 
   mkdir -p ${CONDA_BLD_DIR}/libcudf/work
   cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf/work

ci/gpu/build.sh

@@ -34,6 +34,9 @@ unset GIT_DESCRIBE_TAG
 # Dask & Distributed option to install main(nightly) or `conda-forge` packages.
 export INSTALL_DASK_MAIN=1
 
+# Dask version to install when `INSTALL_DASK_MAIN=0`
+export DASK_STABLE_VERSION="2022.7.1"
+
 # ucx-py version
 export UCX_PY_VERSION='0.29.*'
 
@@ -91,8 +94,8 @@ function install_dask {
         gpuci_mamba_retry update dask
         conda list
     else
-        gpuci_logger "gpuci_mamba_retry install conda-forge::dask>=2022.7.1 conda-forge::distributed>=2022.7.1 conda-forge::dask-core>=2022.7.1 --force-reinstall"
-        gpuci_mamba_retry install conda-forge::dask>=2022.7.1 conda-forge::distributed>=2022.7.1 conda-forge::dask-core>=2022.7.1 --force-reinstall
+        gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall"
+        gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall
     fi
     # Install the main version of streamz
     gpuci_logger "Install the main version of streamz"
@@ -188,6 +191,9 @@ else
     # copied by CI from the upstream 11.5 jobs into $CONDA_ARTIFACT_PATH
     gpuci_logger "Installing cudf, dask-cudf, cudf_kafka, and custreamz"
     gpuci_mamba_retry install cudf dask-cudf cudf_kafka custreamz -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}"
+
+    gpuci_logger "Check current conda environment"
+    conda list --show-channel-urls
 
     gpuci_logger "GoogleTests"
     # Run libcudf and libcudf_kafka gtests from libcudf-tests package

conda/environments/cudf_dev_cuda11.5.yml

@@ -14,7 +14,7 @@ dependencies:
   - clang-tools=11.1.0
   - cupy>=9.5.0,<12.0.0a0
   - rmm=22.12.*
-  - cmake>=3.20.1,!=3.23.0
+  - cmake>=3.23.1
   - cmake_setuptools>=0.1.3
   - scikit-build>=0.13.1
   - python>=3.8,<3.10

conda/recipes/cudf/conda_build_config.yaml

@@ -8,7 +8,7 @@ sysroot_version:
   - "2.17"
 
 cmake_version:
-  - ">=3.20.1,!=3.23.0"
+  - ">=3.23.1"
 
 cuda_compiler:
   - nvcc

conda/recipes/cudf_kafka/meta.yaml

@@ -22,7 +22,7 @@ build:
 
 requirements:
   build:
-    - cmake >=3.20.1,!=3.23.0
+    - cmake >=3.23.1
     - {{ compiler('c') }}
     - {{ compiler('cxx') }}
     - sysroot_{{ target_platform }} {{ sysroot_version }}

conda/recipes/libcudf/conda_build_config.yaml

@@ -11,7 +11,7 @@ sysroot_version:
   - "2.17"
 
 cmake_version:
-  - ">=3.20.1,!=3.23.0"
+  - ">=3.23.1"
 
 gtest_version:
   - "=1.10.0"

conda/recipes/strings_udf/conda_build_config.yaml

@@ -8,7 +8,7 @@ sysroot_version:
   - "2.17"
 
 cmake_version:
-  - ">=3.20.1,!=3.23.0"
+  - ">=3.23.1"
 
 cuda_compiler:
   - nvcc

cpp/CMakeLists.txt

@@ -12,7 +12,7 @@
 # the License.
 # =============================================================================
 
-cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.23.1 FATAL_ERROR)
 
 include(../fetch_rapids.cmake)
 include(rapids-cmake)

cpp/libcudf_kafka/CMakeLists.txt

@@ -11,7 +11,7 @@
 # or implied. See the License for the specific language governing permissions and limitations under
 # the License.
 # =============================================================================
-cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.23.1 FATAL_ERROR)
 
 include(../../fetch_rapids.cmake)
 include(rapids-cmake)

cpp/src/transform/row_bit_count.cu

@@ -409,7 +409,8 @@ __global__ void compute_row_sizes(device_span<column_device_view const> cols,
   auto const num_rows = output.size();
   if (tid >= num_rows) { return; }
 
-  // branch stack. points to the last list prior to branching.
+  // my_branch_stack points to the last span prior to branching. a branch occurs only
+  // when we are inside of a list contained within a struct column.
   row_span* my_branch_stack = thread_branch_stacks + (threadIdx.x * max_branch_depth);
   size_type branch_depth{0};
 
@@ -424,11 +425,12 @@ __global__ void compute_row_sizes(device_span<column_device_view const> cols,
   for (size_type idx = 0; idx < cols.size(); idx++) {
     column_device_view const& col = cols[idx];
 
-    // if we've returned from a branch
+    // if we've returned from a branch, pop to the proper span
     if (info[idx].branch_depth_start < last_branch_depth) {
-      cur_span = my_branch_stack[--branch_depth];
+      branch_depth = info[idx].branch_depth_start;
+      cur_span     = my_branch_stack[branch_depth];
     }
-    // if we're entering a new branch.
+    // if we're entering a new branch, push the current span
     // NOTE: this case can happen (a pop and a push by the same column)
     // when we have a struct<list, list>
     if (info[idx].branch_depth_end > info[idx].branch_depth_start) {

cpp/tests/transform/row_bit_count_test.cu

@@ -618,9 +618,47 @@ TEST_F(RowBitCount, Table)
     thrust::make_counting_iterator(0) + t.num_rows(),
     mcv.begin<size_type>(),
     sum_functor{cv0.data<size_type>(), cv1.data<size_type>(), cv2.data<size_type>()});
+
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *result);
 }
 
+TEST_F(RowBitCount, DepthJump)
+{
+  // jump more than 1 branch depth.
+
+  using T = int;
+
+  // struct<list<struct<list<int>>, int>
+  // the jump occurs from depth 2 (the leafmost int column)
+  // to depth 0 (the topmost int column)
+  cudf::test::fixed_width_column_wrapper<T> ____c0{1, 2, 3, 5, 5, 6, 7, 8};
+  cudf::test::fixed_width_column_wrapper<cudf::offset_type> ___offsets{0, 2, 4, 6, 8};
+  auto ___c0 = cudf::make_lists_column(4, ___offsets.release(), ____c0.release(), 0, {});
+  std::vector<std::unique_ptr<cudf::column>> __children;
+  __children.push_back(std::move(___c0));
+  cudf::test::structs_column_wrapper __c0(std::move(__children));
+  cudf::test::fixed_width_column_wrapper<cudf::offset_type> _offsets{0, 3, 4};
+  auto _c0 = cudf::make_lists_column(2, _offsets.release(), __c0.release(), 0, {});
+  cudf::test::fixed_width_column_wrapper<int> _c1{3, 4};
+  std::vector<std::unique_ptr<cudf::column>> children;
+  children.push_back(std::move(_c0));
+  children.push_back(_c1.release());
+  cudf::test::structs_column_wrapper c0(std::move(children));
+
+  table_view t({c0});
+  auto result = cudf::row_bit_count(t);
+
+  // expected size = (num rows at level 1 + num_rows at level 2) + (# values the leaf int column) +
+  // 1 (value in topmost int column)
+  constexpr size_type offset_size = sizeof(cudf::offset_type) * CHAR_BIT;
+  constexpr size_type type_size   = sizeof(T) * CHAR_BIT;
+  cudf::test::fixed_width_column_wrapper<size_type> expected{
+    ((1 + 3) * offset_size) + (6 * type_size) + (1 * type_size),
+    ((1 + 1) * offset_size) + (2 * type_size) + (1 * type_size)};
+
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, *result);
+}
+
 TEST_F(RowBitCount, SlicedColumnsFixedWidth)
 {
   auto const slice_size = 7;