Generate pyproject dependencies using dfg (#12906)

This PR updates cudf to generate all build, run and test dependencies in pyproject.toml using `rapids-dependency-file-generator`. In the process, a few additional relevant changes were made:
- dependencies.yaml was refactored to more precisely capture the requirements of the different packages in the repo as well as more accurately separate the build/run/test dependencies. This should facilitate generation of meta.yaml dependencies with `rapids-dependency-file-generator` if we should choose to do so.
- cudf_kafka's setup.py had as much logic as possible moved to pyproject.toml without having to change the way that extension modules are built. This package is somewhat unique within RAPIDS in having a C++ component that we have not converted to scikit-build because we have no plans for wheels there at this time. As a result, it was not previously converted to a pyproject.toml-driven build (classic Cython builds are not compatible with pyproject.toml-driven builds).
- A tiny unrelated fix to the Java build that removed an outdated CMake flag.

Authors:
  - Vyas Ramasubramani (https://github.com/vyasr)

Approvers:
  - Lawrence Mitchell (https://github.com/wence-)
  - Mark Sadang (https://github.com/msadang)
  - Robert Maynard (https://github.com/robertmaynard)

URL: #12906

.github/workflows/pr.yaml

@@ -133,5 +133,6 @@ jobs:
     with:
       build_type: pull-request
       package-name: dask_cudf
-      test-before: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf-dep && python -m pip install --no-deps ./local-cudf-dep/cudf*.whl"
+      # Install the cudf we just built, and also test against latest dask/distributed/dask-cuda.
+      test-before: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf-dep && python -m pip install --no-deps ./local-cudf-dep/cudf*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]"
       test-unittest: "python -m pytest -v -n 8 ./python/dask_cudf/dask_cudf/tests"

.github/workflows/test.yaml

@@ -97,4 +97,6 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       package-name: dask_cudf
+      # Test against latest dask/distributed/dask-cuda.
+      test-before: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]"
       test-unittest: "python -m pytest -v -n 8 ./python/dask_cudf/dask_cudf/tests"

.pre-commit-config.yaml

@@ -161,7 +161,7 @@ repos:
                     ^CHANGELOG.md$
                   )
       - repo: https://github.com/rapidsai/dependency-file-generator
-        rev: v1.4.0
+        rev: v1.5.1
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]

ci/release/update-version.sh

@@ -24,6 +24,11 @@ NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
 NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
 NEXT_UCX_PY_VERSION="$(curl -sL https://version.gpuci.io/rapids/${NEXT_SHORT_TAG}).*"
 
+# Need to distutils-normalize the versions for some use cases
+CURRENT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${CURRENT_SHORT_TAG}'))")
+NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")
+echo "current is ${CURRENT_SHORT_TAG_PEP440}, next is ${NEXT_SHORT_TAG_PEP440}"
+
 echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG"
 
 # Inplace sed replace; workaround for Linux and Mac
@@ -70,9 +75,10 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/cudf/source/
 
 # bump rmm & dask-cuda
 for FILE in conda/environments/*.yaml dependencies.yaml; do
-  sed_runner "s/dask-cuda=${CURRENT_SHORT_TAG}/dask-cuda=${NEXT_SHORT_TAG}/g" ${FILE};
-  sed_runner "s/rmm=${CURRENT_SHORT_TAG}/rmm=${NEXT_SHORT_TAG}/g" ${FILE};
-  sed_runner "s/rmm-cu11=${CURRENT_SHORT_TAG}/rmm-cu11=${NEXT_SHORT_TAG}/g" ${FILE};
+  sed_runner "s/dask-cuda==${CURRENT_SHORT_TAG_PEP440}/dask-cuda==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/rmm==${CURRENT_SHORT_TAG_PEP440}/rmm==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/cudf==${CURRENT_SHORT_TAG_PEP440}/cudf==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
+  sed_runner "s/cudf_kafka==${CURRENT_SHORT_TAG_PEP440}/cudf_kafka==${NEXT_SHORT_TAG_PEP440}/g" ${FILE};
 done
 
 # Doxyfile update
@@ -86,13 +92,11 @@ sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" README.md
 sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/basic/CMakeLists.txt
 sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/strings/CMakeLists.txt
 
-# Need to distutils-normalize the original version
-NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")
-
 # Dependency versions in pyproject.toml
 sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/pyproject.toml
 sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/dask_cudf/pyproject.toml
 
 for FILE in .github/workflows/*.yaml; do
   sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"
+  sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE};
 done

ci/test_java.sh

@@ -38,7 +38,7 @@ set +e
 
 rapids-logger "Run Java tests"
 pushd java
-mvn test -B -DCUDF_JNI_ARROW_STATIC=OFF -DCUDF_JNI_ENABLE_PROFILING=OFF
+mvn test -B -DCUDF_JNI_ENABLE_PROFILING=OFF
 popd
 
 rapids-logger "Test script exiting with value: $EXITCODE"

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -21,7 +21,7 @@ dependencies:
 - cupy>=9.5.0,<12.0.0a0
 - cxx-compiler
 - cython>=0.29,<0.30
-- dask-cuda=23.04.*
+- dask-cuda==23.4.*
 - dask>=2023.1.1
 - distributed>=2023.1.1
 - dlpack>=0.5,<0.6.0a0
@@ -32,11 +32,12 @@ dependencies:
 - gcc_linux-64=11.*
 - hypothesis
 - ipython
-- libarrow=10
+- libarrow==10.0.1.*
 - librdkafka=1.7.0
-- librmm=23.04.*
+- librmm==23.4.*
 - mimesis>=4.1.0
 - moto>=4.0.8
+- msgpack-python
 - myst-nb
 - nbsphinx
 - ninja
@@ -53,19 +54,19 @@ dependencies:
 - pre-commit
 - protobuf>=4.21.6,<4.22
 - ptxcompiler
-- pyarrow=10
+- pyarrow==10.0.1.*
 - pydata-sphinx-theme
 - pyorc
 - pytest
 - pytest-benchmark
 - pytest-cases
 - pytest-cov
 - pytest-xdist
-- python-confluent-kafka=1.7.0
+- python-confluent-kafka==1.7.0
 - python-snappy>=0.6.0
 - python>=3.8,<3.11
 - pytorch<1.12.0
-- rmm=23.04.*
+- rmm==23.4.*
 - s3fs>=2022.3.0
 - scikit-build>=0.13.1
 - scipy

cpp/cmake/thirdparty/get_arrow.cmake

@@ -379,6 +379,8 @@ endfunction()
 
 if(NOT DEFINED CUDF_VERSION_Arrow)
   set(CUDF_VERSION_Arrow
+      # This version must be kept in sync with the libarrow version pinned for builds in
+      # dependencies.yaml.
       10.0.1
       CACHE STRING "The version of Arrow to find (or build)"
   )