From d17376aaa964c5256aa0cdcbac5fa15e0be4aca7 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Wed, 27 Sep 2023 06:25:34 -0700 Subject: [PATCH 1/9] Add RAFT devcontainers (#1791) This PR adds some [devcontainers](https://containers.dev/) to help simplify building the RAFT C++ and Python libraries. It also adds an optional job to the `pr.yaml` to [build the RAFT libs in each devcontainer](https://github.com/trxcllnt/raft/blob/fea/devcontainers/.github/workflows/pr.yaml#L96-L101), so the build caches are populated for devs by CI. A devcontainer can be launched by clicking the "Reopen in Container" button that VSCode shows when opening the repo (or by using the "Rebuild and Reopen in Container" command from the command palette): ![image](https://user-images.githubusercontent.com/178183/221771999-97ab29d5-e718-4e5f-b32f-2cdd51bba25c.png) Clicking this button will cause VSCode to prompt the user to select one of these devcontainer variants: ![image](https://github.com/rapidsai/rmm/assets/178183/68d4b264-4fc2-4008-92b6-cb4bdd19b29f) On startup, the devcontainer creates or updates the conda/pip environment using `raft/dependencies.yaml`. The envs/package caches are cached on the host via volume mounts, which are described in more detail in [`.devcontainer/README.md`](https://github.com/trxcllnt/raft/blob/fea/devcontainers/.devcontainer/README.md). The container includes convenience functions to clean, configure, and build the various RAFT components: ```shell $ clean-raft-cpp # only cleans the C++ build dir $ clean-pylibraft-python # only cleans the Python build dir $ clean-raft # cleans both C++ and Python build dirs $ configure-raft-cpp # only configures raft C++ lib $ build-raft-cpp # only builds raft C++ lib $ build-pylibraft-python # only builds raft Python lib $ build-raft # builds both C++ and Python libs ``` * The C++ build script is a small wrapper around `cmake -S ~/raft/cpp -B ~/raft/cpp/build` and `cmake --build ~/raft/cpp/build` * The Python build script is a small wrapper around `pip install --editable ~/raft/cpp` Unlike `build.sh`, these convenience scripts *don't* install the libraries after building them. Instead, they automatically inject the correct arguments to build the C++ libraries from source and use their build dirs as package roots: ```shell $ cmake -S ~/raft/cpp -B ~/raft/cpp/build $ CMAKE_ARGS="-Draft_ROOT=~/raft/cpp/build" \ # <-- this argument is automatic pip install -e ~/raft/cpp ``` Authors: - Paul Taylor (https://github.com/trxcllnt) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/raft/pull/1791 --- .devcontainer/Dockerfile | 30 ++++ .devcontainer/README.md | 64 ++++++++ .../cuda11.8-conda/devcontainer.json | 37 +++++ .devcontainer/cuda11.8-pip/devcontainer.json | 38 +++++ .../cuda12.0-conda/devcontainer.json | 37 +++++ .devcontainer/cuda12.0-pip/devcontainer.json | 38 +++++ .github/workflows/pr.yaml | 9 ++ .gitignore | 4 + ci/release/update-version.sh | 7 + .../all_cuda-118_arch-x86_64.yaml | 4 +- .../all_cuda-120_arch-x86_64.yaml | 4 +- .../bench_ann_cuda-118_arch-x86_64.yaml | 3 +- cpp/.clangd | 65 ++++++++ cpp/test/core/math_device.cu | 6 +- dependencies.yaml | 152 ++++++++++++++++-- python/raft-dask/CMakeLists.txt | 12 +- 16 files changed, 480 insertions(+), 30 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/README.md create mode 100644 .devcontainer/cuda11.8-conda/devcontainer.json create mode 100644 .devcontainer/cuda11.8-pip/devcontainer.json create mode 100644 .devcontainer/cuda12.0-conda/devcontainer.json create mode 100644 .devcontainer/cuda12.0-pip/devcontainer.json create mode 100644 cpp/.clangd diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..9d35e3f97f --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1.5 + +ARG BASE +ARG PYTHON_PACKAGE_MANAGER=conda + +FROM ${BASE} as pip-base + +ENV DEFAULT_VIRTUAL_ENV=rapids + +FROM ${BASE} as conda-base + +ENV DEFAULT_CONDA_ENV=rapids + +FROM ${PYTHON_PACKAGE_MANAGER}-base + +ARG CUDA +ENV CUDAARCHS="RAPIDS" +ENV CUDA_VERSION="${CUDA_VERSION:-${CUDA}}" + +ARG PYTHON_PACKAGE_MANAGER +ENV PYTHON_PACKAGE_MANAGER="${PYTHON_PACKAGE_MANAGER}" + +ENV PYTHONSAFEPATH="1" +ENV PYTHONUNBUFFERED="1" +ENV PYTHONDONTWRITEBYTECODE="1" + +ENV SCCACHE_REGION="us-east-2" +ENV SCCACHE_BUCKET="rapids-sccache-devs" +ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai" +ENV HISTFILE="/home/coder/.cache/._bash_history" diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 0000000000..3c76b8963d --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,64 @@ +# RAFT Development Containers + +This directory contains [devcontainer configurations](https://containers.dev/implementors/json_reference/) for using VSCode to [develop in a container](https://code.visualstudio.com/docs/devcontainers/containers) via the `Remote Containers` [extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) or [GitHub Codespaces](https://github.com/codespaces). + +This container is a turnkey development environment for building and testing the RAFT C++ and Python libraries. + +## Table of Contents + +* [Prerequisites](#prerequisites) +* [Host bind mounts](#host-bind-mounts) +* [Launch a Dev Container](#launch-a-dev-container) + +## Prerequisites + +* [VSCode](https://code.visualstudio.com/download) +* [VSCode Remote Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +## Host bind mounts + +By default, the following directories are bind-mounted into the devcontainer: + +* `${repo}:/home/coder/raft` +* `${repo}/../.aws:/home/coder/.aws` +* `${repo}/../.local:/home/coder/.local` +* `${repo}/../.cache:/home/coder/.cache` +* `${repo}/../.conda:/home/coder/.conda` +* `${repo}/../.config:/home/coder/.config` + +This ensures caches, configurations, dependencies, and your commits are persisted on the host across container runs. + +## Launch a Dev Container + +To launch a devcontainer from VSCode, open the RAFT repo and select the "Reopen in Container" button in the bottom right:
+ +Alternatively, open the VSCode command palette (typically `cmd/ctrl + shift + P`) and run the "Rebuild and Reopen in Container" command. + +## Using the devcontainer + +On startup, the devcontainer creates or updates the conda/pip environment using `raft/dependencies.yaml`. + +The container includes convenience functions to clean, configure, and build the various RAFT components: + +```shell +$ clean-raft-cpp # only cleans the C++ build dir +$ clean-pylibraft-python # only cleans the Python build dir +$ clean-raft # cleans both C++ and Python build dirs + +$ configure-raft-cpp # only configures raft C++ lib + +$ build-raft-cpp # only builds raft C++ lib +$ build-pylibraft-python # only builds raft Python lib +$ build-raft # builds both C++ and Python libs +``` + +* The C++ build script is a small wrapper around `cmake -S ~/raft/cpp -B ~/raft/cpp/build` and `cmake --build ~/raft/cpp/build` +* The Python build script is a small wrapper around `pip install --editable ~/raft/cpp` + +Unlike `build.sh`, these convenience scripts *don't* install the libraries after building them. Instead, they automatically inject the correct arguments to build the C++ libraries from source and use their build dirs as package roots: + +```shell +$ cmake -S ~/raft/cpp -B ~/raft/cpp/build +$ CMAKE_ARGS="-Draft_ROOT=~/raft/cpp/build" \ # <-- this argument is automatic + pip install -e ~/raft/cpp +``` diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json new file mode 100644 index 0000000000..8da9b5428a --- /dev/null +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -0,0 +1,37 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "11.8", + "PYTHON_PACKAGE_MANAGER": "conda", + "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/raft,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda11.8-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json new file mode 100644 index 0000000000..0b3ec79e37 --- /dev/null +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -0,0 +1,38 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "11.8", + "PYTHON_PACKAGE_MANAGER": "pip", + "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/ucx:23.10": {"version": "1.14.1"}, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/ucx", + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/raft,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json new file mode 100644 index 0000000000..f5af166b46 --- /dev/null +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -0,0 +1,37 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "12.0", + "PYTHON_PACKAGE_MANAGER": "conda", + "BASE": "rapidsai/devcontainers:23.10-cpp-mambaforge-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/raft,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.0-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json new file mode 100644 index 0000000000..9f28002d38 --- /dev/null +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -0,0 +1,38 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "12.0", + "PYTHON_PACKAGE_MANAGER": "pip", + "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda12.0-ubuntu22.04" + } + }, + "hostRequirements": {"gpu": "optional"}, + "features": { + "ghcr.io/rapidsai/devcontainers/features/ucx:23.10": {"version": "1.14.1"}, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/ucx", + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" + ], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs}"], + "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/raft,type=bind,consistency=consistent", + "mounts": [ + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "ms-python.flake8", + "nvidia.nsight-vscode-edition" + ] + } + } +} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4fa3c5df86..e539877851 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,6 +22,7 @@ jobs: - wheel-tests-pylibraft - wheel-build-raft-dask - wheel-tests-raft-dask + - devcontainer secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.10 checks: @@ -92,3 +93,11 @@ jobs: with: build_type: pull-request script: ci/test_wheel_raft_dask.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-23.10 + with: + build_command: | + sccache -z; + build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; + sccache -s; diff --git a/.gitignore b/.gitignore index 7939fc1622..11b7bc3eba 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,7 @@ _xml # sphinx _html _text + +# clang tooling +compile_commands.json +.clangd/ diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 7a69b95da1..a867a71f68 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -91,3 +91,10 @@ sed_runner "/^PROJECT_NUMBER/ s|\".*\"|\"${NEXT_SHORT_TAG}\"|g" cpp/doxygen/Doxy sed_runner "/^set(RAFT_VERSION/ s|\".*\"|\"${NEXT_SHORT_TAG}\"|g" docs/source/build.md sed_runner "/GIT_TAG.*branch-/ s|branch-.*|branch-${NEXT_SHORT_TAG}|g" docs/source/build.md sed_runner "/rapidsai\/raft/ s|branch-[0-9][0-9].[0-9][0-9]|branch-${NEXT_SHORT_TAG}|g" docs/source/developer_guide.md + +# .devcontainer files +find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r -d '' filename; do + sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}" + sed_runner "s@rapidsai/devcontainers/features/ucx:[0-9.]*@rapidsai/devcontainers/features/ucx:${NEXT_SHORT_TAG_PEP440}@" "${filename}" + sed_runner "s@rapidsai/devcontainers/features/rapids-build-utils:[0-9.]*@rapidsai/devcontainers/features/rapids-build-utils:${NEXT_SHORT_TAG_PEP440}@" "${filename}" +done diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 65b4232d83..93e8821575 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -10,7 +10,7 @@ dependencies: - breathe - c-compiler - clang-tools=16.0.6 -- clang=16.0.6 +- clang==16.0.6 - cmake>=3.26.4 - cuda-profiler-api=11.8.86 - cuda-python>=11.7.1,<12.0a0 @@ -43,6 +43,8 @@ dependencies: - numba>=0.57 - numpy>=1.21 - numpydoc +- nvcc_linux-64=11.8 +- pre-commit - pydata-sphinx-theme - pytest - pytest-cov diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 9db38ed1de..01d9bca5c2 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -10,9 +10,10 @@ dependencies: - breathe - c-compiler - clang-tools=16.0.6 -- clang=16.0.6 +- clang==16.0.6 - cmake>=3.26.4 - cuda-cudart-dev +- cuda-nvcc - cuda-profiler-api - cuda-python>=12.0,<13.0a0 - cuda-version=12.0 @@ -39,6 +40,7 @@ dependencies: - numba>=0.57 - numpy>=1.21 - numpydoc +- pre-commit - pydata-sphinx-theme - pytest - pytest-cov diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 5a9ef5bd32..4f1df12dfa 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -10,7 +10,7 @@ dependencies: - benchmark>=1.8.2 - c-compiler - clang-tools=16.0.6 -- clang=16.0.6 +- clang==16.0.6 - cmake>=3.26.4 - cuda-profiler-api=11.8.86 - cuda-version=11.8 @@ -34,6 +34,7 @@ dependencies: - nccl>=2.9.9 - ninja - nlohmann_json>=3.11.2 +- nvcc_linux-64=11.8 - scikit-build>=0.13.1 - sysroot_linux-64==2.17 name: bench_ann_cuda-118_arch-x86_64 diff --git a/cpp/.clangd b/cpp/.clangd new file mode 100644 index 0000000000..7c4fe036dd --- /dev/null +++ b/cpp/.clangd @@ -0,0 +1,65 @@ +# https://clangd.llvm.org/config + +# Apply a config conditionally to all C files +If: + PathMatch: .*\.(c|h)$ + +--- + +# Apply a config conditionally to all C++ files +If: + PathMatch: .*\.(c|h)pp + +--- + +# Apply a config conditionally to all CUDA files +If: + PathMatch: .*\.cuh? +CompileFlags: + Add: + - "-x" + - "cuda" + # No error on unknown CUDA versions + - "-Wno-unknown-cuda-version" + # Allow variadic CUDA functions + - "-Xclang=-fcuda-allow-variadic-functions" +Diagnostics: + Suppress: + - "variadic_device_fn" + - "attributes_not_allowed" + +--- + +# Tweak the clangd parse settings for all files +CompileFlags: + Add: + # report all errors + - "-ferror-limit=0" + - "-fmacro-backtrace-limit=0" + - "-ftemplate-backtrace-limit=0" + # Skip the CUDA version check + - "--no-cuda-version-check" + Remove: + # remove gcc's -fcoroutines + - -fcoroutines + # remove nvc++ flags unknown to clang + - "-gpu=*" + - "-stdpar*" + # remove nvcc flags unknown to clang + - "-arch*" + - "-gencode*" + - "--generate-code*" + - "-ccbin*" + - "-t=*" + - "--threads*" + - "-Xptxas*" + - "-Xcudafe*" + - "-Xfatbin*" + - "-Xcompiler*" + - "--diag-suppress*" + - "--diag_suppress*" + - "--compiler-options*" + - "--expt-extended-lambda" + - "--expt-relaxed-constexpr" + - "-forward-unknown-to-host-compiler" + - "-Werror=cross-execution-space-call" diff --git a/cpp/test/core/math_device.cu b/cpp/test/core/math_device.cu index 15c7b2b33a..8e3a9df01b 100644 --- a/cpp/test/core/math_device.cu +++ b/cpp/test/core/math_device.cu @@ -21,7 +21,9 @@ #include #include -#if _RAFT_HAS_CUDA +#include + +#ifdef _RAFT_HAS_CUDA #include #include #endif @@ -35,7 +37,7 @@ __global__ void math_eval_kernel(OutT* out, OpT op, Args... args) template auto math_eval(OpT op, Args&&... args) { - typedef decltype(op(args...)) OutT; + using OutT = cuda::std::invoke_result_t; auto stream = rmm::cuda_stream_default; rmm::device_scalar result(stream); math_eval_kernel<<<1, 1, 0, stream>>>(result.data(), op, std::forward(args)...); diff --git a/dependencies.yaml b/dependencies.yaml index 700a6db1bf..b827f11228 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -10,12 +10,15 @@ files: - build_pylibraft - cudatoolkit - develop + - checks + - build_wheels - test_libraft - docs - run_raft_dask - run_pylibraft - test_python_common - test_pylibraft + - cupy bench_ann: output: conda matrix: @@ -38,6 +41,7 @@ files: - py_version - test_python_common - test_pylibraft + - cupy checks: output: none includes: @@ -47,6 +51,7 @@ files: output: none includes: - test_pylibraft + - cupy - cudatoolkit - docs - py_version @@ -75,6 +80,7 @@ files: includes: - test_python_common - test_pylibraft + - cupy py_build_raft_dask: output: pyproject pyproject_dir: python/raft-dask @@ -145,11 +151,37 @@ dependencies: packages: - gcc_linux-aarch64=11.* - sysroot_linux-aarch64==2.17 + - output_types: conda + matrices: + - matrix: {cuda: "12.0"} + packages: [cuda-version=12.0, cuda-nvcc] + - matrix: {cuda: "11.8", arch: x86_64} + packages: [nvcc_linux-64=11.8] + - matrix: {cuda: "11.8", arch: aarch64} + packages: [nvcc_linux-aarch64=11.8] + - matrix: {cuda: "11.5", arch: x86_64} + packages: [nvcc_linux-64=11.5] + - matrix: {cuda: "11.5", arch: aarch64} + packages: [nvcc_linux-aarch64=11.5] + - matrix: {cuda: "11.4", arch: x86_64} + packages: [nvcc_linux-64=11.4] + - matrix: {cuda: "11.4", arch: aarch64} + packages: [nvcc_linux-aarch64=11.4] + - matrix: {cuda: "11.2", arch: x86_64} + packages: [nvcc_linux-64=11.2] + - matrix: {cuda: "11.2", arch: aarch64} + packages: [nvcc_linux-aarch64=11.2] + build_pylibraft: common: - - output_types: [conda, requirements, pyproject] + - output_types: [conda] packages: - - &rmm rmm==23.10.* + - &rmm_conda rmm==23.10.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm-cu{11,12}. + - --extra-index-url=https://pypi.nvidia.com specific: - output_types: [conda, requirements, pyproject] matrices: @@ -160,6 +192,20 @@ dependencies: - matrix: # All CUDA 11 versions packages: - &cuda_python11 cuda-python>=11.7.1,<12.0a0 + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &build_pylibraft_packages_cu12 + - &rmm_cu12 rmm-cu12==23.10.* + - {matrix: {cuda: "12.1"}, packages: *build_pylibraft_packages_cu12} + - {matrix: {cuda: "12.0"}, packages: *build_pylibraft_packages_cu12} + - matrix: {cuda: "11.8"} + packages: &build_pylibraft_packages_cu11 + - &rmm_cu11 rmm-cu11==23.10.* + - {matrix: {cuda: "11.5"}, packages: *build_pylibraft_packages_cu11} + - {matrix: {cuda: "11.4"}, packages: *build_pylibraft_packages_cu11} + - {matrix: {cuda: "11.2"}, packages: *build_pylibraft_packages_cu11} + - {matrix: null, packages: [*rmm_conda] } checks: common: - output_types: [conda, requirements] @@ -167,11 +213,9 @@ dependencies: - pre-commit develop: common: - - output_types: [conda, requirements] - packages: - - clang=16.0.6 - - output_types: [conda] + - output_types: conda packages: + - clang==16.0.6 - clang-tools=16.0.6 nn_bench: common: @@ -265,6 +309,45 @@ dependencies: - *libcusolver114 - *libcusparse_dev114 - *libcusparse114 + + cupy: + common: + - output_types: conda + packages: + - cupy>=12.0.0 + specific: + - output_types: [requirements, pyproject] + matrices: + # All CUDA 12 + x86_64 versions + - matrix: {cuda: "12.2", arch: x86_64} + packages: &cupy_packages_cu12_x86_64 + - &cupy_cu12_x86_64 cupy-cuda12x>=12.0.0 + - {matrix: {cuda: "12.1", arch: x86_64}, packages: *cupy_packages_cu12_x86_64} + - {matrix: {cuda: "12.0", arch: x86_64}, packages: *cupy_packages_cu12_x86_64} + # All CUDA 12 + aarch64 versions + - matrix: {cuda: "12.2", arch: aarch64} + packages: &cupy_packages_cu12_aarch64 + - &cupy_cu12_aarch64 cupy-cuda12x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. + - {matrix: {cuda: "12.1", arch: aarch64}, packages: *cupy_packages_cu12_aarch64} + - {matrix: {cuda: "12.0", arch: aarch64}, packages: *cupy_packages_cu12_aarch64} + + # All CUDA 11 + x86_64 versions + - matrix: {cuda: "11.8", arch: x86_64} + packages: &cupy_packages_cu11_x86_64 + - cupy-cuda11x>=12.0.0 + - {matrix: {cuda: "11.5", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} + - {matrix: {cuda: "11.4", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} + - {matrix: {cuda: "11.2", arch: x86_64}, packages: *cupy_packages_cu11_x86_64} + + # All CUDA 11 + aarch64 versions + - matrix: {cuda: "11.8", arch: aarch64} + packages: &cupy_packages_cu11_aarch64 + - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. + - {matrix: {cuda: "11.5", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} + - {matrix: {cuda: "11.4", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} + - {matrix: {cuda: "11.2", arch: aarch64}, packages: *cupy_packages_cu11_aarch64} + - {matrix: null, packages: [cupy-cuda11x>=12.0.0]} + test_libraft: common: - output_types: [conda] @@ -287,7 +370,7 @@ dependencies: - sphinx-markdown-tables build_wheels: common: - - output_types: pyproject + - output_types: [requirements, pyproject] packages: - wheel - setuptools @@ -311,7 +394,14 @@ dependencies: - output_types: [conda, pyproject] packages: - &numpy numpy>=1.21 - - *rmm + - output_types: [conda] + packages: + - *rmm_conda + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for cudf and rmm. + - --extra-index-url=https://pypi.nvidia.com specific: - output_types: [conda, requirements, pyproject] matrices: @@ -322,6 +412,20 @@ dependencies: - matrix: # All CUDA 11 versions packages: - *cuda_python11 + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &run_pylibraft_packages_cu12 + - *rmm_cu12 + - {matrix: {cuda: "12.1"}, packages: *run_pylibraft_packages_cu12} + - {matrix: {cuda: "12.0"}, packages: *run_pylibraft_packages_cu12} + - matrix: {cuda: "11.8"} + packages: &run_pylibraft_packages_cu11 + - *rmm_cu11 + - {matrix: {cuda: "11.5"}, packages: *run_pylibraft_packages_cu11} + - {matrix: {cuda: "11.4"}, packages: *run_pylibraft_packages_cu11} + - {matrix: {cuda: "11.2"}, packages: *run_pylibraft_packages_cu11} + - {matrix: null, packages: [*rmm_conda]} run_raft_dask: common: - output_types: [conda, pyproject] @@ -332,15 +436,37 @@ dependencies: - joblib>=0.11 - numba>=0.57 - *numpy - - ucx-py==0.34.* - output_types: conda packages: - dask-core>=2023.7.1 - ucx>=1.13.0 - ucx-proc=*=gpu + - &ucx_py_conda ucx-py==0.34.* - output_types: pyproject packages: - - pylibraft==23.10.* + - &pylibraft_conda pylibraft==23.10.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for cudf and rmm. + - --extra-index-url=https://pypi.nvidia.com + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.2"} + packages: &run_raft_dask_packages_cu12 + - &pylibraft_cu12 pylibraft-cu12==23.10.* + - &ucx_py_cu12 ucx-py-cu12==0.34.* + - {matrix: {cuda: "12.1"}, packages: *run_raft_dask_packages_cu12} + - {matrix: {cuda: "12.0"}, packages: *run_raft_dask_packages_cu12} + - matrix: {cuda: "11.8"} + packages: &run_raft_dask_packages_cu11 + - &pylibraft_cu11 pylibraft-cu11==23.10.* + - &ucx_py_cu11 ucx-py-cu11==0.34.* + - {matrix: {cuda: "11.5"}, packages: *run_raft_dask_packages_cu11} + - {matrix: {cuda: "11.4"}, packages: *run_raft_dask_packages_cu11} + - {matrix: {cuda: "11.2"}, packages: *run_raft_dask_packages_cu11} + - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} test_python_common: common: - output_types: [conda, requirements, pyproject] @@ -353,9 +479,3 @@ dependencies: packages: - scikit-learn - scipy - - output_types: conda - packages: - - cupy>=12.0.0 - - output_types: pyproject - packages: - - cupy-cuda11x>=12.0.0 diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index f9a20b46bb..af98f31857 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -17,6 +17,8 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) set(raft_dask_version 23.10.00) include(../../fetch_rapids.cmake) +include(rapids-cuda) +rapids_cuda_init_architectures(raft-dask-python) project( raft-dask-python @@ -25,7 +27,7 @@ project( # language to be enabled here. The test project that is built in scikit-build to verify # various linking options for the python library is hardcoded to build with C, so until # that is fixed we need to keep C. - C CXX + C CXX CUDA ) option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" @@ -42,14 +44,6 @@ else() endif() if(NOT raft_FOUND) - # TODO: This will not be necessary once we upgrade to CMake 3.22, which will pull in the required - # languages for the C++ project even if this project does not require those languages. - include(rapids-cuda) - rapids_cuda_init_architectures(raft-dask) - enable_language(CUDA) - # Since raft-dask only enables CUDA optionally we need to manually include the file that - # rapids_cuda_init_architectures relies on `project` including. - include("${CMAKE_PROJECT_raft-dask_INCLUDE}") find_package(ucx REQUIRED) # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all From e618fb07b49bf750357d4fc3c619385ad86e0a87 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 29 Sep 2023 09:20:59 -0500 Subject: [PATCH 2/9] Pin dask and distributed for 23.10 release (#1864) This PR pins `dask` and `distributed` to `2023.9.2` for `23.10` release. xref: https://github.com/rapidsai/cudf/pull/14225 Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Ray Douglass (https://github.com/raydouglass) - Peter Andreas Entschev (https://github.com/pentschev) - Ben Frederickson (https://github.com/benfred) --- ci/test_wheel_raft_dask.sh | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 6 +++--- conda/environments/all_cuda-120_arch-x86_64.yaml | 6 +++--- conda/recipes/raft-dask/meta.yaml | 6 +++--- dependencies.yaml | 6 +++--- python/raft-dask/pyproject.toml | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index 676d642de9..fd9668e968 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -12,7 +12,7 @@ RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl # Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.10 +python -m pip install git+https://github.com/dask/dask.git@2023.9.2 git+https://github.com/dask/distributed.git@2023.9.2 git+https://github.com/rapidsai/dask-cuda.git@branch-23.10 # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/raft_dask*.whl)[test] diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 93e8821575..739e1e9785 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -19,10 +19,10 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.7.1 +- dask-core==2023.9.2 - dask-cuda==23.10.* -- dask>=2023.7.1 -- distributed>=2023.7.1 +- dask==2023.9.2 +- distributed==2023.9.2 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 01d9bca5c2..321c17bf4f 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -20,10 +20,10 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.7.1 +- dask-core==2023.9.2 - dask-cuda==23.10.* -- dask>=2023.7.1 -- distributed>=2023.7.1 +- dask==2023.9.2 +- distributed==2023.9.2 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index c9caa4dd9b..04dfef5063 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -60,10 +60,10 @@ requirements: - cudatoolkit {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - - dask >=2023.7.1 - - dask-core >=2023.7.1 + - dask ==2023.9.2 + - dask-core ==2023.9.2 - dask-cuda ={{ minor_version }} - - distributed >=2023.7.1 + - distributed ==2023.9.2 - joblib >=0.11 - nccl >=2.9.9 - pylibraft {{ version }} diff --git a/dependencies.yaml b/dependencies.yaml index b827f11228..3ad51a6377 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -430,15 +430,15 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask>=2023.7.1 + - dask==2023.9.2 - dask-cuda==23.10.* - - distributed>=2023.7.1 + - distributed==2023.9.2 - joblib>=0.11 - numba>=0.57 - *numpy - output_types: conda packages: - - dask-core>=2023.7.1 + - dask-core==2023.9.2 - ucx>=1.13.0 - ucx-proc=*=gpu - &ucx_py_conda ucx-py==0.34.* diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index bdbcf61e0f..3e0ffc2848 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -35,8 +35,8 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "dask-cuda==23.10.*", - "dask>=2023.7.1", - "distributed>=2023.7.1", + "dask==2023.9.2", + "distributed==2023.9.2", "joblib>=0.11", "numba>=0.57", "numpy>=1.21", From 3521733835c27c45ec982dc9b1405295735e1485 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 4 Oct 2023 04:18:04 +0200 Subject: [PATCH 3/9] Update pyproject.toml Co-authored-by: GALI PREM SAGAR --- python/raft-dask/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 6a73b55410..659b75ee2b 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -35,8 +35,8 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "dask-cuda==23.12.*", - "dask>=2023.9.2", - "distributed>=2023.9.2", + "dask==2023.9.2", + "distributed==2023.9.2", "joblib>=0.11", "numba>=0.57", "numpy>=1.21", From 1ff634a95eb5e46039eb44471e3bfff269886d6e Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 3 Oct 2023 22:45:35 -0400 Subject: [PATCH 4/9] Update --- python/raft-dask/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 659b75ee2b..6a73b55410 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -35,8 +35,8 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "dask-cuda==23.12.*", - "dask==2023.9.2", - "distributed==2023.9.2", + "dask>=2023.9.2", + "distributed>=2023.9.2", "joblib>=0.11", "numba>=0.57", "numpy>=1.21", From 84fe8818e8d6faa629ef98474a45ecf42959da82 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 3 Oct 2023 23:42:15 -0400 Subject: [PATCH 5/9] Fixing wheel test script --- ci/test_wheel_raft_dask.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index a20e950313..a9ae5dcabb 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -12,7 +12,7 @@ RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl # Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.12 +python -m pip install git+https://github.com/dask/dask.git@2023.9.2 git+https://github.com/dask/distributed.git@2023.9.2 git+https://github.com/rapidsai/dask-cuda.git@branch-23.12 # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/raft_dask*.whl)[test] From 8c8774609712080efd0a0a40c69bb3faf5f54bbf Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 4 Oct 2023 00:31:07 -0400 Subject: [PATCH 6/9] Trying again --- conda/environments/all_cuda-118_arch-x86_64.yaml | 4 ++-- conda/environments/all_cuda-120_arch-x86_64.yaml | 4 ++-- dependencies.yaml | 4 ++-- python/raft-dask/pyproject.toml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8eab21ada2..bba3b79b8c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -21,8 +21,8 @@ dependencies: - cython>=3.0.0 - dask-core==2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 +- dask==2023.9.2 +- distributed==2023.9.2 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index cae6f623dd..764f8e8e6f 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -22,8 +22,8 @@ dependencies: - cython>=3.0.0 - dask-core==2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 +- dask==2023.9.2 +- distributed==2023.9.2 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 diff --git a/dependencies.yaml b/dependencies.yaml index de837ed7b5..b3a6d47248 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -430,9 +430,9 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask>=2023.9.2 + - dask==2023.9.2 - dask-cuda==23.12.* - - distributed>=2023.9.2 + - distributed==2023.9.2 - joblib>=0.11 - numba>=0.57 - *numpy diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 6a73b55410..659b75ee2b 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -35,8 +35,8 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "dask-cuda==23.12.*", - "dask>=2023.9.2", - "distributed>=2023.9.2", + "dask==2023.9.2", + "distributed==2023.9.2", "joblib>=0.11", "numba>=0.57", "numpy>=1.21", From 25bac962a274056c57f9ac7f924b032bd7f72794 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 4 Oct 2023 16:56:18 -0400 Subject: [PATCH 7/9] Fixing remaining 23.10 refs --- .github/workflows/pr.yaml | 2 +- python/raft-dask/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5b9e9e9c43..1b7fb8e1a5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -95,7 +95,7 @@ jobs: script: ci/test_wheel_raft_dask.sh devcontainer: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-23.10 + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-23.12 with: build_command: | sccache -z; diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 659b75ee2b..c7b8501390 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "joblib>=0.11", "numba>=0.57", "numpy>=1.21", - "pylibraft==23.10.*", + "pylibraft==23.12.*", "ucx-py==0.34.*", "ucx-py==0.35.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 1b7f8036131e3cb0fa0e6a747cb67188c4892d14 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 4 Oct 2023 16:56:53 -0400 Subject: [PATCH 8/9] Fixing more --- .devcontainer/cuda11.8-conda/devcontainer.json | 4 ++-- .devcontainer/cuda11.8-pip/devcontainer.json | 6 +++--- .devcontainer/cuda12.0-conda/devcontainer.json | 4 ++-- .devcontainer/cuda12.0-pip/devcontainer.json | 6 +++--- dependencies.yaml | 10 +++++----- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 8da9b5428a..203f52f1a2 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -5,12 +5,12 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:23.12-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 0b3ec79e37..080ece996e 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -5,13 +5,13 @@ "args": { "CUDA": "11.8", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:23.12-cpp-llvm16-cuda11.8-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:23.10": {"version": "1.14.1"}, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + "ghcr.io/rapidsai/devcontainers/features/ucx:23.12": {"version": "1.14.1"}, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json index f5af166b46..da8bfb4db9 100644 --- a/.devcontainer/cuda12.0-conda/devcontainer.json +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -5,12 +5,12 @@ "args": { "CUDA": "12.0", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:23.10-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:23.12-cpp-mambaforge-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json index 9f28002d38..e2bee94f8a 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -5,13 +5,13 @@ "args": { "CUDA": "12.0", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda12.0-ubuntu22.04" + "BASE": "rapidsai/devcontainers:23.12-cpp-llvm16-cuda12.0-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:23.10": {"version": "1.14.1"}, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.10": {} + "ghcr.io/rapidsai/devcontainers/features/ucx:23.12": {"version": "1.14.1"}, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:23.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", diff --git a/dependencies.yaml b/dependencies.yaml index b3a6d47248..1070ba30e5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -196,12 +196,12 @@ dependencies: matrices: - matrix: {cuda: "12.2"} packages: &build_pylibraft_packages_cu12 - - &rmm_cu12 rmm-cu12==23.10.* + - &rmm_cu12 rmm-cu12==23.12.* - {matrix: {cuda: "12.1"}, packages: *build_pylibraft_packages_cu12} - {matrix: {cuda: "12.0"}, packages: *build_pylibraft_packages_cu12} - matrix: {cuda: "11.8"} packages: &build_pylibraft_packages_cu11 - - &rmm_cu11 rmm-cu11==23.10.* + - &rmm_cu11 rmm-cu11==23.12.* - {matrix: {cuda: "11.5"}, packages: *build_pylibraft_packages_cu11} - {matrix: {cuda: "11.4"}, packages: *build_pylibraft_packages_cu11} - {matrix: {cuda: "11.2"}, packages: *build_pylibraft_packages_cu11} @@ -445,7 +445,7 @@ dependencies: - &ucx_py_conda ucx-py==0.34.* - output_types: pyproject packages: - - &pylibraft_conda pylibraft==23.10.* + - &pylibraft_conda pylibraft==23.12.* - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -456,13 +456,13 @@ dependencies: matrices: - matrix: {cuda: "12.2"} packages: &run_raft_dask_packages_cu12 - - &pylibraft_cu12 pylibraft-cu12==23.10.* + - &pylibraft_cu12 pylibraft-cu12==23.12.* - &ucx_py_cu12 ucx-py-cu12==0.34.* - {matrix: {cuda: "12.1"}, packages: *run_raft_dask_packages_cu12} - {matrix: {cuda: "12.0"}, packages: *run_raft_dask_packages_cu12} - matrix: {cuda: "11.8"} packages: &run_raft_dask_packages_cu11 - - &pylibraft_cu11 pylibraft-cu11==23.10.* + - &pylibraft_cu11 pylibraft-cu11==23.12.* - &ucx_py_cu11 ucx-py-cu11==0.34.* - {matrix: {cuda: "11.5"}, packages: *run_raft_dask_packages_cu11} - {matrix: {cuda: "11.4"}, packages: *run_raft_dask_packages_cu11} From dec06d871cb723e8078c8352e992b9f245d8b9d5 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 4 Oct 2023 17:01:08 -0400 Subject: [PATCH 9/9] Setting ucx-py version --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - conda/environments/all_cuda-120_arch-x86_64.yaml | 1 - dependencies.yaml | 6 +++--- python/raft-dask/pyproject.toml | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index bba3b79b8c..fcbf1451a8 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -57,7 +57,6 @@ dependencies: - sphinx-markdown-tables - sysroot_linux-64==2.17 - ucx-proc=*=gpu -- ucx-py==0.34.* - ucx-py==0.35.* - ucx>=1.13.0 name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 764f8e8e6f..a1e22f50a0 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -53,7 +53,6 @@ dependencies: - sphinx-markdown-tables - sysroot_linux-64==2.17 - ucx-proc=*=gpu -- ucx-py==0.34.* - ucx-py==0.35.* - ucx>=1.13.0 name: all_cuda-120_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 1070ba30e5..fe4a4620e0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -442,7 +442,7 @@ dependencies: - dask-core==2023.9.2 - ucx>=1.13.0 - ucx-proc=*=gpu - - &ucx_py_conda ucx-py==0.34.* + - &ucx_py_conda ucx-py==0.35.* - output_types: pyproject packages: - &pylibraft_conda pylibraft==23.12.* @@ -457,13 +457,13 @@ dependencies: - matrix: {cuda: "12.2"} packages: &run_raft_dask_packages_cu12 - &pylibraft_cu12 pylibraft-cu12==23.12.* - - &ucx_py_cu12 ucx-py-cu12==0.34.* + - &ucx_py_cu12 ucx-py-cu12==0.35.* - {matrix: {cuda: "12.1"}, packages: *run_raft_dask_packages_cu12} - {matrix: {cuda: "12.0"}, packages: *run_raft_dask_packages_cu12} - matrix: {cuda: "11.8"} packages: &run_raft_dask_packages_cu11 - &pylibraft_cu11 pylibraft-cu11==23.12.* - - &ucx_py_cu11 ucx-py-cu11==0.34.* + - &ucx_py_cu11 ucx-py-cu11==0.35.* - {matrix: {cuda: "11.5"}, packages: *run_raft_dask_packages_cu11} - {matrix: {cuda: "11.4"}, packages: *run_raft_dask_packages_cu11} - {matrix: {cuda: "11.2"}, packages: *run_raft_dask_packages_cu11} diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index c7b8501390..1619edbbbf 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -41,7 +41,6 @@ dependencies = [ "numba>=0.57", "numpy>=1.21", "pylibraft==23.12.*", - "ucx-py==0.34.*", "ucx-py==0.35.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [