dependencies.yaml

# Dependency list for https://github.com/rapidsai/dependency-file-generator
files:
  all:
    output: conda
    matrix:
      cuda: ["11.8", "12.0"]
      arch: [x86_64]
    includes:
      - build_all
      - build_cpp
      - build_wheels
      - build_python
      - build_python_common
      - cudatoolkit
      - develop
      - docs
      - notebooks
      - py_version
      - run_common
      - run_cudf
      - run_dask_cudf
      - run_cudf_kafka
      - run_custreamz
      - test_cpp
      - test_python_common
      - test_python_cudf
      - test_python_dask_cudf
  test_cpp:
    output: none
    includes:
      - cudatoolkit
      - test_cpp
      - libarrow_run
  test_python:
    output: none
    includes:
      - cudatoolkit
      - py_version
      - test_python_common
      - test_python_cudf
      - test_python_dask_cudf
      - pyarrow_run
  test_java:
    output: none
    includes:
      - build_all
      - libarrow_run
      - cudatoolkit
      - test_java
  test_notebooks:
    output: none
    includes:
      - notebooks
      - py_version
  checks:
    output: none
    includes:
      - develop
      - py_version
  docs:
    output: none
    includes:
      - cudatoolkit
      - docs
      - py_version
  py_build_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: build-system
    includes:
      - build_all
      - build_python
      - build_python_common
      - build_wheels
  py_run_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: project
    includes:
      - run_common
      - run_cudf
      - pyarrow_run
  py_test_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
      - test_python_cudf
  py_build_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: build-system
    includes:
      - build_wheels
  py_run_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: project
    includes:
      - run_common
      - run_dask_cudf
  py_test_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
      - test_python_dask_cudf
  py_build_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: build-system
    includes:
      - build_wheels
      - build_python_common
  py_run_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: project
    includes:
      - run_cudf_kafka
  py_test_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
  py_build_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: build-system
    includes:
      - build_wheels
  py_run_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: project
    includes:
      - run_custreamz
  py_test_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
channels:
  - rapidsai
  - rapidsai-nightly
  - dask/label/dev
  - pytorch
  - conda-forge
  - nvidia
dependencies:
  build_all:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - &cmake_ver cmake>=3.26.4
          - ninja
      - output_types: conda
        packages:
          - c-compiler
          - cxx-compiler
          - dlpack>=0.5,<0.6.0a0
          - zlib>=1.2.13
    specific:
      - output_types: conda
        matrices:
          - matrix:
              arch: x86_64
            packages:
              - gcc_linux-64=11.*
              - sysroot_linux-64==2.17
          - matrix:
              arch: aarch64
            packages:
              - gcc_linux-aarch64=11.*
              - sysroot_linux-aarch64==2.17
      - output_types: conda
        matrices:
          - matrix:
              cuda: "12.0"
            packages:
              - cuda-version=12.0
              - cuda-nvcc
          - matrix:
              arch: x86_64
              cuda: "11.8"
            packages:
              - nvcc_linux-64=11.8
          - matrix:
              arch: aarch64
              cuda: "11.8"
            packages:
              - nvcc_linux-aarch64=11.8
  build_cpp:
    common:
      - output_types: [conda, requirements]
        packages:
          - librmm==23.10.*
          - libkvikio==23.10.*
      - output_types: conda
        packages:
          - fmt>=9.1.0,<10
          - &gbench benchmark==1.8.0
          - &gtest gtest>=1.13.0
          - &gmock gmock>=1.13.0
          # Hard pin the patch version used during the build. This must be kept
          # in sync with the version pinned in get_arrow.cmake.
          - libarrow==12.0.1.*
          - librdkafka>=1.9.0,<1.10.0a0
          - spdlog>=1.11.0,<1.12
    specific:
      - output_types: conda
        matrices:
          - matrix:
              arch: x86_64
            packages:
              # Align nvcomp version with rapids-cmake
              # TODO: not yet available for aarch64 CUDA 12
              - &nvcomp nvcomp==2.6.1
          - matrix:
              arch: aarch64
              cuda: "11.8"
            packages:
              - *nvcomp
          # TODO: Fallback matrix for aarch64 CUDA 12. After migrating to nvcomp 3,
          # all CUDA/arch combinations should be supported by existing packages.
          - matrix:
            packages:
  build_wheels:
    common:
      - output_types: pyproject
        packages:
          - wheel
          - setuptools
  build_python_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - cython>=3.0.0
          # Hard pin the patch version used during the build. This must be kept
          # in sync with the version pinned in get_arrow.cmake.
          - pyarrow==12.0.1.*
          - numpy>=1.21
  build_python:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - scikit-build>=0.13.1
          - rmm==23.10.*
      - output_types: conda
        packages:
          - &protobuf protobuf>=4.21,<5
      - output_types: pyproject
        packages:
          - protoc-wheel
  libarrow_run:
    common:
      - output_types: [conda, requirements]
        packages:
          # Allow runtime version to float up to minor version
          - libarrow==12.*
  pyarrow_run:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          # Allow runtime version to float up to minor version
          - pyarrow==12.*
  cudatoolkit:
    specific:
      - output_types: conda
        matrices:
          - matrix:
              cuda: "12.0"
            packages:
              - cuda-version=12.0
              - cuda-cudart-dev
              - cuda-nvrtc-dev
              - cuda-nvtx-dev
              - libcurand-dev
          - matrix:
              cuda: "11.8"
            packages:
              - cuda-version=11.8
              - cudatoolkit
              - cuda-nvtx=11.8
              - libcurand-dev=10.3.0.86
              - libcurand=10.3.0.86
          - matrix:
              cuda: "11.5"
            packages:
              - cuda-version=11.5
              - cudatoolkit
              - cuda-nvtx=11.5
                # Can't hard pin the version since 11.x is missing many
                # packages for specific versions
              - libcurand-dev>=10.2.6.48,<=10.2.7.107
              - libcurand>=10.2.6.48,<=10.2.7.107
          - matrix:
              cuda: "11.4"
            packages:
              - cuda-version=11.4
              - cudatoolkit
              - &cudanvtx114 cuda-nvtx=11.4
              - &libcurand_dev114 libcurand-dev>=10.2.5.43,<=10.2.5.120
              - &libcurand114 libcurand>=10.2.5.43,<=10.2.5.120
          - matrix:
              cuda: "11.2"
            packages:
              - cuda-version=11.2
              - cudatoolkit
                # The NVIDIA channel doesn't publish pkgs older than 11.4 for
                # these libs, so 11.2 uses 11.4 packages (the oldest
                # available).
              - *cudanvtx114
              - *libcurand_dev114
              - *libcurand114
      - output_types: conda
        matrices:
          - matrix:
              cuda: "12.0"
              arch: x86_64
            packages:
              - libcufile-dev
          - matrix:
              cuda: "11.8"
              arch: x86_64
            packages:
              - libcufile=1.4.0.31
              - libcufile-dev=1.4.0.31
          - matrix:
              cuda: "11.5"
              arch: x86_64
            packages:
              - libcufile>=1.1.0.37,<=1.1.1.25
              - libcufile-dev>=1.1.0.37,<=1.1.1.25
          - matrix:
              cuda: "11.4"
              arch: x86_64
            packages:
              - &libcufile_114 libcufile>=1.0.0.82,<=1.0.2.10
              - &libcufile_dev114 libcufile-dev>=1.0.0.82,<=1.0.2.10
          - matrix:
              cuda: "11.2"
              arch: x86_64
            packages:
              # The NVIDIA channel doesn't publish pkgs older than 11.4 for these libs,
              # so 11.2 uses 11.4 packages (the oldest available).
              - *libcufile_114
              - *libcufile_dev114
          # Fallback matrix for aarch64, which doesn't support libcufile.
          - matrix:
            packages:
  develop:
    common:
      - output_types: [conda, requirements]
        packages:
          - pre-commit
          # pre-commit requires identify minimum version 1.0, but clang-format requires textproto support and that was
          # added in 2.5.20, so we need to call out the minimum version needed for our plugins
          - identify>=2.5.20
      - output_types: conda
        packages:
          - &doxygen doxygen=1.9.1 # pre-commit hook needs a specific version.
  docs:
    common:
      - output_types: [conda]
        packages:
          - dask-cuda==23.10.*
          - *doxygen
          - make
          - myst-nb
          - nbsphinx
          - numpydoc
          - pandoc
          - pydata-sphinx-theme
          - scipy
          - sphinx
          - sphinx-autobuild
          - sphinx-copybutton
          - sphinx-markdown-tables
          - sphinxcontrib-websupport
  notebooks:
    common:
      - output_types: [conda, requirements]
        packages:
          - ipython
          - notebook
          - scipy
  py_version:
    specific:
      - output_types: conda
        matrices:
          - matrix:
              py: "3.9"
            packages:
              - python=3.9
          - matrix:
              py: "3.10"
            packages:
              - python=3.10
          - matrix:
            packages:
              - python>=3.9,<3.11
  run_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - fsspec>=0.6.0
          - numpy>=1.21
          - pandas>=1.3,<1.6.0dev0
  run_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - cachetools
          - &numba numba>=0.57
          - nvtx>=0.2.1
          - packaging
          - rmm==23.10.*
          - typing_extensions>=4.0.0
          - *protobuf
      - output_types: conda
        packages:
          - cupy>=12.0.0
          - pip
          - pip:
              - git+https://github.com/python-streamz/streamz.git@master
      - output_types: requirements
        packages:
          # pip recognizes the index as a global option for the requirements.txt file
          # This index is needed for rmm, cubinlinker, ptxcompiler.
          - --extra-index-url=https://pypi.ngc.nvidia.com
          - git+https://github.com/python-streamz/streamz.git@master
          - &cupy_pip cupy-cuda11x>=12.0.0
      - output_types: pyproject
        packages:
          - cubinlinker
          - *cupy_pip
          - ptxcompiler
    specific:
      - output_types: [conda, requirements, pyproject]
        matrices:
          - matrix:
              cuda: "12.0"
            packages:
              - cuda-python>=12.0,<13.0a0
          - matrix: # All CUDA 11 versions
            packages:
              - cuda-python>=11.7.1,<12.0a0
      - output_types: [conda, pyproject]
        matrices:
          - matrix:
              cuda: "12.0"
            packages:
          - matrix: # All CUDA 11 versions
            packages:
              - cubinlinker
              - ptxcompiler
      - output_types: requirements
        matrices:
          - matrix:
              cuda: "12.0"
            packages:
          - matrix: # All CUDA 11 versions
            packages:
              - cubinlinker-cu11
              - ptxcompiler-cu11
  run_dask_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - dask>=2023.7.1
          - distributed>=2023.7.1
      - output_types: conda
        packages:
          - cupy>=12.0.0
          - dask-core>=2023.7.1  # dask-core in conda is the actual package & dask is the meta package
      - output_types: pyproject
        packages:
          - &cudf cudf==23.10.*
          - *cupy_pip
  run_cudf_kafka:
    common:
      - output_types: [requirements, pyproject]
        packages:
          - *cudf
  run_custreamz:
    common:
      - output_types: conda
        packages:
          - python-confluent-kafka>=1.9.0,<1.10.0a0
      - output_types: [conda, requirements, pyproject]
        packages:
          - streamz
      - output_types: [requirements, pyproject]
        packages:
          - confluent-kafka>=1.9.0,<1.10.0a0
          - *cudf
          - cudf_kafka==23.10.*
  test_cpp:
    common:
      - output_types: conda
        packages:
          - *cmake_ver
          - *gbench
          - *gtest
          - *gmock
    specific:
      - output_types: conda
        matrices:
          - matrix:
              cuda: "12.0"
            packages:
              - cuda-version=12.0
              - cuda-sanitizer-api
          - matrix:
              cuda: "11.8"
            packages:
              - cuda-sanitizer-api=11.8.86
          - matrix:
            packages:
  test_java:
    common:
      - output_types: conda
        packages:
          - *cmake_ver
          - maven
          - openjdk=8.*
  test_python_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - pytest
          - pytest-cov
          - pytest-xdist
  test_python_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - fastavro>=0.22.9
          - hypothesis
          - mimesis>=4.1.0
          - pyorc
          - pytest-benchmark
          - pytest-cases
          - python-snappy>=0.6.0
          - scipy
      - output_types: conda
        packages:
          - aiobotocore>=2.2.0
          - boto3>=1.21.21
          - botocore>=1.24.21
          - msgpack-python
          - moto>=4.0.8
          - s3fs>=2022.3.0
      - output_types: pyproject
        packages:
          - msgpack
          - &tokenizers tokenizers==0.13.1
          - &transformers transformers==4.24.0
          - tzdata
    specific:
      - output_types: conda
        matrices:
          - matrix:
              arch: x86_64
            packages:
              # Currently, CUDA builds of pytorch do not exist for aarch64. We require
              # version <1.12.0 because newer versions use nvidia::cuda-toolkit.
              - pytorch<1.12.0
              # We only install these on x86_64 to avoid pulling pytorch as a
              # dependency of transformers.
              - *tokenizers
              - *transformers
          - matrix:
            packages:
  test_python_dask_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - dask-cuda==23.10.*
          - *numba