diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000000..c19bb68986f --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,35 @@ +# syntax=docker/dockerfile:1.5 + +ARG BASE +ARG PYTHON_PACKAGE_MANAGER=conda + +FROM ${BASE} as pip-base + +RUN apt update -y \ + && DEBIAN_FRONTEND=noninteractive apt install -y \ + librdkafka-dev \ + && rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*; + +ENV DEFAULT_VIRTUAL_ENV=rapids + +FROM ${BASE} as conda-base + +ENV DEFAULT_CONDA_ENV=rapids + +FROM ${PYTHON_PACKAGE_MANAGER}-base + +ARG CUDA +ENV CUDAARCHS="RAPIDS" +ENV CUDA_VERSION="${CUDA_VERSION:-${CUDA}}" + +ARG PYTHON_PACKAGE_MANAGER +ENV PYTHON_PACKAGE_MANAGER="${PYTHON_PACKAGE_MANAGER}" + +ENV PYTHONSAFEPATH="1" +ENV PYTHONUNBUFFERED="1" +ENV PYTHONDONTWRITEBYTECODE="1" + +ENV SCCACHE_REGION="us-east-2" +ENV SCCACHE_BUCKET="rapids-sccache-devs" +ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai" +ENV HISTFILE="/home/coder/.cache/._bash_history" diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000000..91ee7ef85f7 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,35 @@ +# cuDF Development Containers + +This directory contains [devcontainer configurations](https://containers.dev/implementors/json_reference/) for using VSCode to [develop in a container](https://code.visualstudio.com/docs/devcontainers/containers) via the `Remote Containers` [extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) or [GitHub Codespaces](https://github.com/codespaces). + +This container is a turnkey development environment for building and testing the cuDF C++ and Python libraries. + +## Table of Contents + +* [Prerequisites](#prerequisites) +* [Host bind mounts](#host-bind-mounts) +* [Launch a Dev Container](#launch-a-dev-container) + +## Prerequisites + +* [VSCode](https://code.visualstudio.com/download) +* [VSCode Remote Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +## Host bind mounts + +By default, the following directories are bind-mounted into the devcontainer: + +* `${repo}:/home/coder/cudf` +* `${repo}/../.aws:/home/coder/.aws` +* `${repo}/../.local:/home/coder/.local` +* `${repo}/../.cache:/home/coder/.cache` +* `${repo}/../.conda:/home/coder/.conda` +* `${repo}/../.config:/home/coder/.config` + +This ensures caches, configurations, dependencies, and your commits are persisted on the host across container runs. + +## Launch a Dev Container + +To launch a devcontainer from VSCode, open the cuDF repo and select the "Reopen in Container" button in the bottom right:
+ +Alternatively, open the VSCode command palette (typically `cmd/ctrl + shift + P`) and run the "Rebuild and Reopen in Container" command. diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json new file mode 100644 index 00000000000..6c3322dfc61 --- /dev/null +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -0,0 +1,37 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "11.8", + "PYTHON_PACKAGE_MANAGER": "conda", + "BASE": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.2": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda11.8-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json new file mode 100644 index 00000000000..d2afd3a497e --- /dev/null +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -0,0 +1,36 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "11.8", + "PYTHON_PACKAGE_MANAGER": "pip", + "BASE": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda11.8-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.2": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json new file mode 100644 index 00000000000..42ed334fe03 --- /dev/null +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -0,0 +1,37 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "12.0", + "PYTHON_PACKAGE_MANAGER": "conda", + "BASE": "rapidsai/devcontainers:24.02-cpp-mambaforge-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.2": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.0-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json new file mode 100644 index 00000000000..306a2065ef0 --- /dev/null +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -0,0 +1,36 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "12.0", + "PYTHON_PACKAGE_MANAGER": "pip", + "BASE": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda12.0-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.2": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cudf,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 462fad6e938..5a77c6749fe 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,7 @@ jobs: - wheel-tests-cudf - wheel-build-dask-cudf - wheel-tests-dask-cudf + - devcontainer - unit-tests-cudf-pandas - pandas-tests #- pandas-tests-diff @@ -130,6 +131,14 @@ jobs: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) build_type: pull-request script: ci/test_wheel_dask_cudf.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.02 + with: + build_command: | + sccache -z; + build-all -DBUILD_BENCHMARKS=ON -DNVBench_ENABLE_CUPTI=OFF --verbose; + sccache -s; unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit diff --git a/.gitignore b/.gitignore index 4a4a6a98efb..243ba73e5b0 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,7 @@ jupyter_execute # cibuildwheel /wheelhouse + +# clang tooling +compile_commands.json +.clangd/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a62104722e8..7db8d9ab52f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -145,7 +145,7 @@ repos: ^CHANGELOG.md$ ) - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.5.1 + rev: v1.7.1 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 16742465c32..f9b1436495a 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -115,3 +115,9 @@ NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}-SNAPSHOT" sed_runner "s|.*-SNAPSHOT|${NEXT_FULL_JAVA_TAG}|g" java/pom.xml sed_runner "s/branch-.*/branch-${NEXT_SHORT_TAG}/g" java/ci/README.md sed_runner "s/cudf-.*-SNAPSHOT/cudf-${NEXT_FULL_JAVA_TAG}/g" java/ci/README.md + +# .devcontainer files +find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r -d '' filename; do + sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}" + sed_runner "s@rapidsai/devcontainers/features/rapids-build-utils:[0-9.]*@rapidsai/devcontainers/features/rapids-build-utils:${NEXT_SHORT_TAG_PEP440}@" "${filename}" +done diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 1a944fbdb21..0bafdd13af5 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -14,6 +14,8 @@ dependencies: - botocore>=1.24.21 - c-compiler - cachetools +- clang-tools=16.0.6 +- clang==16.0.6 - cmake>=3.26.4 - cramjam - cubinlinker diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 21837b652f4..27dede9b519 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -14,6 +14,8 @@ dependencies: - botocore>=1.24.21 - c-compiler - cachetools +- clang-tools=16.0.6 +- clang==16.0.6 - cmake>=3.26.4 - cramjam - cuda-cudart-dev diff --git a/cpp/.clangd b/cpp/.clangd new file mode 100644 index 00000000000..7c4fe036ddf --- /dev/null +++ b/cpp/.clangd @@ -0,0 +1,65 @@ +# https://clangd.llvm.org/config + +# Apply a config conditionally to all C files +If: + PathMatch: .*\.(c|h)$ + +--- + +# Apply a config conditionally to all C++ files +If: + PathMatch: .*\.(c|h)pp + +--- + +# Apply a config conditionally to all CUDA files +If: + PathMatch: .*\.cuh? +CompileFlags: + Add: + - "-x" + - "cuda" + # No error on unknown CUDA versions + - "-Wno-unknown-cuda-version" + # Allow variadic CUDA functions + - "-Xclang=-fcuda-allow-variadic-functions" +Diagnostics: + Suppress: + - "variadic_device_fn" + - "attributes_not_allowed" + +--- + +# Tweak the clangd parse settings for all files +CompileFlags: + Add: + # report all errors + - "-ferror-limit=0" + - "-fmacro-backtrace-limit=0" + - "-ftemplate-backtrace-limit=0" + # Skip the CUDA version check + - "--no-cuda-version-check" + Remove: + # remove gcc's -fcoroutines + - -fcoroutines + # remove nvc++ flags unknown to clang + - "-gpu=*" + - "-stdpar*" + # remove nvcc flags unknown to clang + - "-arch*" + - "-gencode*" + - "--generate-code*" + - "-ccbin*" + - "-t=*" + - "--threads*" + - "-Xptxas*" + - "-Xcudafe*" + - "-Xfatbin*" + - "-Xcompiler*" + - "--diag-suppress*" + - "--diag_suppress*" + - "--compiler-options*" + - "--expt-extended-lambda" + - "--expt-relaxed-constexpr" + - "-forward-unknown-to-host-compiler" + - "-Werror=cross-execution-space-call" diff --git a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake index df285bdea55..baabffceeac 100644 --- a/cpp/cmake/Modules/JitifyPreprocessKernels.cmake +++ b/cpp/cmake/Modules/JitifyPreprocessKernels.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -54,6 +54,10 @@ function(jit_preprocess_files) ) endfunction() +if(NOT (EXISTS "${CUDF_GENERATED_INCLUDE_DIR}/include")) + make_directory("${CUDF_GENERATED_INCLUDE_DIR}/include") +endif() + jit_preprocess_files( SOURCE_DIRECTORY ${CUDF_SOURCE_DIR}/src FILES binaryop/jit/kernel.cu transform/jit/kernel.cu rolling/jit/kernel.cu diff --git a/dependencies.yaml b/dependencies.yaml index f00273a5db1..a84a8fa2bf2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -19,12 +19,12 @@ files: - run_common - run_cudf - run_dask_cudf - - run_cudf_kafka - run_custreamz - test_cpp - test_python_common - test_python_cudf - test_python_dask_cudf + - depends_on_cupy test_cpp: output: none includes: @@ -83,6 +83,7 @@ files: - run_common - run_cudf - pyarrow_run + - depends_on_cupy py_test_cudf: output: pyproject pyproject_dir: python/cudf @@ -123,6 +124,8 @@ files: includes: - run_common - run_dask_cudf + - depends_on_cudf + - depends_on_cupy py_test_dask_cudf: output: pyproject pyproject_dir: python/dask_cudf @@ -146,7 +149,7 @@ files: extras: table: project includes: - - run_cudf_kafka + - depends_on_cudf py_test_cudf_kafka: output: pyproject pyproject_dir: python/cudf_kafka @@ -169,6 +172,8 @@ files: table: project includes: - run_custreamz + - depends_on_cudf + - depends_on_cudf_kafka py_test_custreamz: output: pyproject pyproject_dir: python/custreamz @@ -229,16 +234,14 @@ dependencies: - nvcc_linux-aarch64=11.8 build_cpp: common: - - output_types: [conda, requirements] - packages: - - librmm==24.2.* - - libkvikio==24.2.* - output_types: conda packages: - fmt>=9.1.0,<10 - &gbench benchmark==1.8.0 - >est gtest>=1.13.0 - &gmock gmock>=1.13.0 + - librmm==24.2.* + - libkvikio==24.2.* # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - libarrow-all==14.0.1.* @@ -248,7 +251,7 @@ dependencies: - spdlog>=1.11.0,<1.12 build_wheels: common: - - output_types: pyproject + - output_types: [requirements, pyproject] packages: - wheel - setuptools @@ -267,15 +270,40 @@ dependencies: - pyarrow==14.0.1.* build_python_cudf: common: - - output_types: [conda, requirements, pyproject] - packages: - - rmm==24.2.* - output_types: conda packages: + - &rmm_conda rmm==24.2.* - &protobuf protobuf>=4.21,<5 - - output_types: pyproject + - pip + - pip: + - git+https://github.com/python-streamz/streamz.git@master + - output_types: [requirements, pyproject] packages: - protoc-wheel + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm-cu{11,12}. + - --extra-index-url=https://pypi.nvidia.com + - git+https://github.com/python-streamz/streamz.git@master + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &build_python_packages_cu12 + - &rmm_cu12 rmm-cu12==24.2.* + - {matrix: {cuda: "12.1"}, packages: *build_python_packages_cu12} + - {matrix: {cuda: "12.0"}, packages: *build_python_packages_cu12} + - matrix: {cuda: "11.8"} + packages: &build_python_packages_cu11 + - &rmm_cu11 rmm-cu11==24.2.* + - {matrix: {cuda: "11.5"}, packages: *build_python_packages_cu11} + - {matrix: {cuda: "11.4"}, packages: *build_python_packages_cu11} + - {matrix: {cuda: "11.2"}, packages: *build_python_packages_cu11} + - {matrix: null, packages: null } + - output_types: pyproject + matrices: + - {matrix: null, packages: [*rmm_conda] } libarrow_run: common: - output_types: conda @@ -385,6 +413,8 @@ dependencies: - identify>=2.5.20 - output_types: conda packages: + - clang==16.0.6 + - clang-tools=16.0.6 - &doxygen doxygen=1.9.1 # pre-commit hook needs a specific version. docs: common: @@ -444,72 +474,65 @@ dependencies: - nvtx>=0.2.1 - packaging - rich - - rmm==24.2.* - typing_extensions>=4.0.0 - *protobuf - output_types: conda packages: - - cupy>=12.0.0 - - pip - - pip: - - git+https://github.com/python-streamz/streamz.git@master + - *rmm_conda - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file # This index is needed for rmm, cubinlinker, ptxcompiler. - - --extra-index-url=https://pypi.ngc.nvidia.com - - git+https://github.com/python-streamz/streamz.git@master - - &cupy_pip cupy-cuda11x>=12.0.0 - - output_types: pyproject - packages: - - cubinlinker - - *cupy_pip - - ptxcompiler + - --extra-index-url=https://pypi.nvidia.com specific: - output_types: [conda, requirements, pyproject] matrices: - - matrix: - cuda: "12.0" - packages: + - matrix: {cuda: "12.2"} + packages: &run_cudf_packages_all_cu12 - cuda-python>=12.0,<13.0a0 - - matrix: # All CUDA 11 versions - packages: + - {matrix: {cuda: "12.1"}, packages: *run_cudf_packages_all_cu12} + - {matrix: {cuda: "12.0"}, packages: *run_cudf_packages_all_cu12} + - matrix: {cuda: "11.8"} + packages: &run_cudf_packages_all_cu11 - cuda-python>=11.7.1,<12.0a0 - - output_types: [conda, pyproject] + - {matrix: {cuda: "11.5"}, packages: *run_cudf_packages_all_cu11} + - {matrix: {cuda: "11.4"}, packages: *run_cudf_packages_all_cu11} + - {matrix: {cuda: "11.2"}, packages: *run_cudf_packages_all_cu11} + - {matrix: null, packages: *run_cudf_packages_all_cu11} + - output_types: conda matrices: - - matrix: - cuda: "12.0" - packages: - - matrix: # All CUDA 11 versions - packages: + - matrix: {cuda: "11.8"} + packages: &run_cudf_packages_conda_cu11 - cubinlinker - ptxcompiler - - output_types: requirements + - {matrix: {cuda: "11.5"}, packages: *run_cudf_packages_conda_cu11} + - {matrix: {cuda: "11.4"}, packages: *run_cudf_packages_conda_cu11} + - {matrix: {cuda: "11.2"}, packages: *run_cudf_packages_conda_cu11} + - {matrix: null, packages: null} + - output_types: [requirements, pyproject] matrices: - - matrix: - cuda: "12.0" - packages: - - matrix: # All CUDA 11 versions - packages: + - matrix: {cuda: "12.2"} + packages: &run_cudf_packages_pip_cu12 + - rmm-cu12==24.2.* + - {matrix: {cuda: "12.1"}, packages: *run_cudf_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *run_cudf_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &run_cudf_packages_pip_cu11 + - rmm-cu11==24.2.* - cubinlinker-cu11 - ptxcompiler-cu11 + - {matrix: {cuda: "11.5"}, packages: *run_cudf_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *run_cudf_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *run_cudf_packages_pip_cu11} + - {matrix: null, packages: null} + - output_types: pyproject + matrices: + - {matrix: null, packages: [cubinlinker, ptxcompiler, *rmm_conda] } run_dask_cudf: common: - output_types: [conda, requirements, pyproject] packages: - rapids-dask-dependency==24.2.* - - output_types: conda - packages: - - cupy>=12.0.0 - - output_types: pyproject - packages: - - &cudf cudf==24.2.* - - *cupy_pip - run_cudf_kafka: - common: - - output_types: [requirements, pyproject] - packages: - - *cudf run_custreamz: common: - output_types: conda @@ -521,8 +544,6 @@ dependencies: - output_types: [requirements, pyproject] packages: - confluent-kafka>=1.9.0,<1.10.0a0 - - *cudf - - cudf_kafka==24.2.* test_cpp: common: - output_types: conda @@ -606,6 +627,78 @@ dependencies: packages: - dask-cuda==24.2.* - *numba + depends_on_cudf: + common: + - output_types: conda + packages: + - &cudf_conda cudf==24.2.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm, cubinlinker, ptxcompiler. + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &cudf_packages_pip_cu12 + - cudf-cu12==24.2.* + - {matrix: {cuda: "12.1"}, packages: *cudf_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *cudf_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &cudf_packages_pip_cu11 + - cudf-cu11==24.2.* + - {matrix: {cuda: "11.5"}, packages: *cudf_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *cudf_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *cudf_packages_pip_cu11} + - {matrix: null, packages: [*cudf_conda]} + depends_on_cudf_kafka: + common: + - output_types: conda + packages: + - &cudf_kafka_conda cudf_kafka==24.2.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm, cubinlinker, ptxcompiler. + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &cudf_kafka_packages_pip_cu12 + - cudf_kafka-cu12==24.2.* + - {matrix: {cuda: "12.1"}, packages: *cudf_kafka_packages_pip_cu12} + - {matrix: {cuda: "12.0"}, packages: *cudf_kafka_packages_pip_cu12} + - matrix: {cuda: "11.8"} + packages: &cudf_kafka_packages_pip_cu11 + - cudf_kafka-cu11==24.2.* + - {matrix: {cuda: "11.5"}, packages: *cudf_kafka_packages_pip_cu11} + - {matrix: {cuda: "11.4"}, packages: *cudf_kafka_packages_pip_cu11} + - {matrix: {cuda: "11.2"}, packages: *cudf_kafka_packages_pip_cu11} + - {matrix: null, packages: [*cudf_kafka_conda]} + depends_on_cupy: + common: + - output_types: conda + packages: + - cupy>=12.0.0 + specific: + - output_types: [requirements, pyproject] + matrices: + # All CUDA 12 versions + - matrix: {cuda: "12.2"} + packages: &cupy_packages_cu12 + - cupy-cuda12x>=12.0.0 + - {matrix: {cuda: "12.1"}, packages: *cupy_packages_cu12} + - {matrix: {cuda: "12.0"}, packages: *cupy_packages_cu12} + # All CUDA 11 versions + - matrix: {cuda: "11.8"} + packages: &cupy_packages_cu11 + - cupy-cuda11x>=12.0.0 + - {matrix: {cuda: "11.5"}, packages: *cupy_packages_cu11} + - {matrix: {cuda: "11.4"}, packages: *cupy_packages_cu11} + - {matrix: {cuda: "11.2"}, packages: *cupy_packages_cu11} + - {matrix: null, packages: *cupy_packages_cu11} test_python_pandas_cudf: common: - output_types: pyproject