From 107f6e34c35430af36987b3315dbd9e4349d9525 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 20 Dec 2022 19:21:57 -0800 Subject: [PATCH] Add GitHub Actions Workflows. (#1076) This PR adds GitHub Actions workflows to `raft`. **Wait until after the 22.12 release is complete (and potentially until other repos are ready to migrate to GHA) to merge.** ### Task list Coverage required for this PR: - [x] C++ tests - [x] Two test failures on ARM? `Result/18: SWoRTests/SWoRTestD; /opt/conda/conda-bld/work/cpp/test/random/sample_without_replacement.cu:269` - [x] Python tests - [x] Codecov - [x] Style checks Future work: - Build Metrics Report (copy from cudf when it is added) - JUnit output annotations? https://github.com/marketplace/actions/junit-report-action - Reduce exit code handling verbosity (use global variable with `trap` function) - Set up `rapids-dependency-file-generator` pre-commit hook - Docs builds Authors: - Bradley Dice (https://github.com/bdice) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/raft/pull/1076 --- .github/CODEOWNERS | 1 + .github/labeler.yml | 2 +- .github/workflows/build.yaml | 57 ++++++++ .github/workflows/dependency-files.yml | 12 -- .github/workflows/pr.yaml | 49 +++++++ .github/workflows/test.yaml | 34 +++++ ci/build_cpp.sh | 16 +++ ci/build_python.sh | 29 ++++ ci/check_style.sh | 18 +++ ci/cpu/build.sh | 12 +- ci/gpu/build.sh | 7 + ci/release/update-version.sh | 1 + ci/test_cpp.sh | 54 ++++++++ ci/test_python.sh | 78 +++++++++++ .../all_cuda-115_arch-x86_64.yaml | 16 ++- conda/recipes/libraft/conda_build_config.yaml | 44 +++++- conda/recipes/libraft/meta.yaml | 129 +++++++++++------- conda/recipes/pylibraft/meta.yaml | 38 +++--- .../recipes/raft-dask/conda_build_config.yaml | 3 + conda/recipes/raft-dask/meta.yaml | 53 +++---- dependencies.yaml | 69 ++++++++-- python/pylibraft/.coveragerc | 3 + python/raft-dask/.coveragerc | 3 + 23 files changed, 596 insertions(+), 132 deletions(-) create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/dependency-files.yml create mode 100644 .github/workflows/pr.yaml create mode 100644 .github/workflows/test.yaml create mode 100755 ci/build_cpp.sh create mode 100755 ci/build_python.sh create mode 100755 ci/check_style.sh create mode 100755 ci/test_cpp.sh create mode 100755 ci/test_python.sh create mode 100644 python/pylibraft/.coveragerc create mode 100644 python/raft-dask/.coveragerc diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a0528e4011..fc4fcd458b 100755 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,3 +18,4 @@ conda/ @rapidsai/ops-codeowners **/Dockerfile @rapidsai/ops-codeowners **/.dockerignore @rapidsai/ops-codeowners docker/ @rapidsai/ops-codeowners +dependencies.yaml @rapidsai/ops-codeowners diff --git a/.github/labeler.yml b/.github/labeler.yml index 9809e2cc2e..56f77e69c0 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -12,5 +12,5 @@ CMake: - '**/CMakeLists.txt' - '**/cmake/**' -gpuCI: +ci: - 'ci/**' diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000000..4e7d76d8eb --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,57 @@ +name: build + +on: + push: + branches: + - "branch-*" + tags: + - v[0-9][0-9].[0-9][0-9].[0-9][0-9] + workflow_call: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + build_type: + type: string + default: nightly + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + cpp-build: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + python-build: + needs: [cpp-build] + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + upload-conda: + needs: [cpp-build, python-build] + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} diff --git a/.github/workflows/dependency-files.yml b/.github/workflows/dependency-files.yml deleted file mode 100644 index 2c5d41ff1f..0000000000 --- a/.github/workflows/dependency-files.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: dependency-files - -on: - pull_request: - -jobs: - checks: - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main - with: - enable_check_size: false - enable_check_style: false diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000000..ca2e2356c0 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,49 @@ +name: pr + +on: + push: + branches: + - "pull-request/[0-9]+" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pr-builder: + needs: + - checks + - conda-cpp-build + - conda-cpp-tests + - conda-python-build + - conda-python-tests + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@main + checks: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main + with: + build_type: pull-request + node_type: cpu16 + conda-cpp-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main + with: + build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main + with: + build_type: pull-request + conda-python-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + with: + build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..34a104082b --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,34 @@ +name: test + +on: + workflow_call: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + +jobs: + conda-cpp-tests: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main + with: + build_type: nightly + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + conda-python-tests: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + with: + build_type: nightly + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh new file mode 100755 index 0000000000..853ae095d3 --- /dev/null +++ b/ci/build_cpp.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-env-update + +export CMAKE_GENERATOR=Ninja + +rapids-print-env + +rapids-logger "Begin cpp build" + +rapids-mamba-retry mambabuild conda/recipes/libraft + +rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh new file mode 100755 index 0000000000..b20fd51bca --- /dev/null +++ b/ci/build_python.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-env-update + +export CMAKE_GENERATOR=Ninja + +rapids-print-env + +rapids-logger "Begin py build" + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) + +# TODO: Remove `--no-test` flags once importing on a CPU +# node works correctly +rapids-mamba-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + conda/recipes/pylibraft + +rapids-mamba-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + conda/recipes/raft-dask + +rapids-upload-conda-to-s3 python diff --git a/ci/check_style.sh b/ci/check_style.sh new file mode 100755 index 0000000000..be3ac3f4b8 --- /dev/null +++ b/ci/check_style.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright (c) 2020-2022, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-logger "Create checks conda environment" +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file_key checks \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n checks +conda activate checks + +# Run pre-commit checks +pre-commit run --hook-stage manual --all-files --show-diff-on-failure diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 04f9fd008a..2f0e2b94ca 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -31,6 +31,13 @@ fi export GPUCI_CONDA_RETRY_MAX=1 export GPUCI_CONDA_RETRY_SLEEP=30 +# Workaround to keep Jenkins builds working +# until we migrate fully to GitHub Actions +export RAPIDS_CUDA_VERSION="${CUDA}" +export SCCACHE_BUCKET=rapids-sccache +export SCCACHE_REGION=us-west-2 +export SCCACHE_IDLE_TIMEOUT=32768 + # Use Ninja to build export CMAKE_GENERATOR="Ninja" export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld" @@ -123,5 +130,6 @@ fi # UPLOAD - Conda packages ################################################################################ -gpuci_logger "Upload conda packages" -source ci/cpu/upload.sh +# Uploads disabled due to new GH Actions implementation +# gpuci_logger "Upload conda packages" +# source ci/cpu/upload.sh diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 8102f0664b..1808480d37 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -21,6 +21,13 @@ export PARALLEL_LEVEL=${PARALLEL_LEVEL:-8} export CUDA_REL=${CUDA_VERSION%.*} CONDA_ARTIFACT_PATH=${WORKSPACE}/ci/artifacts/raft/cpu/.conda-bld/ # notice there is no `linux-64` here +# Workaround to keep Jenkins builds working +# until we migrate fully to GitHub Actions +export RAPIDS_CUDA_VERSION="${CUDA}" +export SCCACHE_BUCKET=rapids-sccache +export SCCACHE_REGION=us-west-2 +export SCCACHE_IDLE_TIMEOUT=32768 + # Set home to the job's workspace export HOME=$WORKSPACE diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 415ab34e34..0b6410f9c9 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -52,3 +52,4 @@ done sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/gpu/build.sh sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/cpu/build.sh +sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/raft-dask/conda_build_config.yaml diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh new file mode 100755 index 0000000000..d8538bdf47 --- /dev/null +++ b/ci/test_cpp.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +set -euo pipefail + +. /opt/conda/etc/profile.d/conda.sh + +rapids-logger "Generate C++ testing dependencies" +rapids-dependency-file-generator \ + --output conda \ + --file_key test_cpp \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n test + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate test +set -u + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ +mkdir -p "${RAPIDS_TESTS_DIR}" +SUITEERROR=0 + +rapids-print-env + +rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + libraft-headers libraft-distance libraft-nn libraft-tests + +rapids-logger "Check GPU usage" +nvidia-smi + +set +e + +# Run libraft gtests from libraft-tests package +rapids-logger "Run gtests" + +# TODO: exit code handling is too verbose. Find a cleaner solution. + +for gt in "$CONDA_PREFIX"/bin/gtests/libraft/* ; do + test_name=$(basename ${gt}) + echo "Running gtest $test_name" + ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} + + exitcode=$? + if (( ${exitcode} != 0 )); then + SUITEERROR=${exitcode} + echo "FAILED: GTest ${gt}" + fi +done + +exit ${SUITEERROR} diff --git a/ci/test_python.sh b/ci/test_python.sh new file mode 100755 index 0000000000..eb458d2a5a --- /dev/null +++ b/ci/test_python.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +set -euo pipefail + +. /opt/conda/etc/profile.d/conda.sh + +rapids-logger "Generate Python testing dependencies" +rapids-dependency-file-generator \ + --output conda \ + --file_key test_python \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n test + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate test +set -u + +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) + +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} +RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} +mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" +SUITEERROR=0 + +rapids-print-env + +rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + libraft-distance libraft-headers pylibraft raft-dask + +rapids-logger "Check GPU usage" +nvidia-smi + +set +e + +rapids-logger "pytest pylibraft" +pushd python/pylibraft/pylibraft +pytest \ + --cache-clear \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-pylibraft.xml" \ + --cov-config=../.coveragerc \ + --cov=pylibraft \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/pylibraft-coverage.xml" \ + --cov-report=term \ + test +exitcode=$? + +if (( ${exitcode} != 0 )); then + SUITEERROR=${exitcode} + echo "FAILED: 1 or more tests in pylibraft" +fi +popd + +rapids-logger "pytest raft-dask" +pushd python/raft-dask/raft_dask +pytest \ + --cache-clear \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-raft-dask.xml" \ + --cov-config=../.coveragerc \ + --cov=raft_dask \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/raft-dask-coverage.xml" \ + --cov-report=term \ + test +exitcode=$? + +if (( ${exitcode} != 0 )); then + SUITEERROR=${exitcode} + echo "FAILED: 1 or more tests in raft-dask" +fi +popd + +exit ${SUITEERROR} diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 1a12390a85..18e0a8187f 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -12,8 +12,10 @@ dependencies: - clang-tools=11.1.0 - clang=11.1.0 - cmake>=3.23.1,!=3.25.0 +- cuda-profiler-api>=11.4.240,<=11.8.86 - cuda-python >=11.7.1,<12.0 - cudatoolkit=11.5 +- cupy - cxx-compiler - cython>=0.29,<0.30 - dask-cuda=23.02.* @@ -22,14 +24,22 @@ dependencies: - doxygen>=1.8.20 - faiss-proc=*=cuda - gcc_linux-64=9.* +- libcublas-dev>=11.7.3.1,<=11.7.4.6 +- libcublas>=11.7.3.1,<=11.7.4.6 +- libcurand-dev>=10.2.6.48,<=10.2.7.107 +- libcurand>=10.2.6.48,<=10.2.7.107 +- libcusolver-dev>=11.2.1.48,<=11.3.2.107 +- libcusolver>=11.2.1.48,<=11.3.2.107 +- libcusparse-dev>=11.7.0.31,<=11.7.0.107 +- libcusparse>=11.7.0.31,<=11.7.0.107 - libfaiss>=1.7.0=cuda* - ninja - pytest -- rapids-build-env=23.02.* -- rapids-doc-env=23.02.* -- rapids-notebook-env=23.02.* +- pytest-cov - rmm=23.02.* - scikit-build>=0.13.1 +- scikit-learn +- scipy - sphinx-markdown-tables - sysroot_linux-64==2.17 - ucx-proc=*=gpu diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index facb478562..399dd198eb 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -19,11 +19,43 @@ nccl_version: gtest_version: - "=1.10.0" -libcusolver_version: - - ">=11.2.1,<=11.4.1.48" - -libcusparse_version: - - ">=11.5.0,<12.0" - libfaiss_version: - "1.7.0 *_cuda" + +# The CTK libraries below are missing from the conda-forge::cudatoolkit +# package. The "*_host_*" version specifiers correspond to `11.5` packages and the +# "*_run_*" version specifiers correspond to `11.x` packages. + +libcublas_host_version: + - ">=11.7.3.1,<=11.7.4.6" + +libcublas_run_version: + - ">=11.5.2.43,<=11.11.3.6" + +libcurand_host_version: + - ">=10.2.6.48,<=10.2.7.107" + +libcurand_run_version: + - ">=10.2.5.43,<=10.3.0.86" + +libcusolver_host_version: + - ">=11.2.1.48,<=11.3.2.107" + +libcusolver_run_version: + - ">=11.2.0.43,<=11.4.1.48" + +libcusparse_host_version: + - ">=11.7.0.31,<=11.7.0.107" + +libcusparse_run_version: + - ">=11.6.0.43,<=11.7.5.86" + +# `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all +# architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the +# "*_run_*" version specifiers correspond to `11.x` packages. + +cuda_profiler_api_host_version: + - ">=11.8.86,<12" + +cuda_profiler_api_run_version: + - ">=11.4.240,<12" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index c32fd1264f..b0d6c47ee9 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -4,9 +4,8 @@ # conda build . -c conda-forge -c nvidia -c rapidsai {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version = '.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} +{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} -{% set ucx_py_version = environ.get('UCX_PY_VERSION') %} {% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 package: @@ -21,18 +20,18 @@ outputs: script: build_libraft_headers.sh build: script_env: &script_env - - PARALLEL_LEVEL - - VERSION_SUFFIX - - PROJECT_FLASH - - CMAKE_GENERATOR + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_GENERATOR + - PARALLEL_LEVEL + - SCCACHE_BUCKET + - SCCACHE_IDLE_TIMEOUT + - SCCACHE_REGION - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_BUCKET=rapids-sccache - - SCCACHE_REGION=us-west-2 - - SCCACHE_IDLE_TIMEOUT=32768 number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: @@ -42,23 +41,35 @@ outputs: - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} - - sysroot_{{ target_platform }} {{ sysroot_version }} - cmake {{ cmake_version }} + - ninja + - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit {{ cuda_version }}.* - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} - - librmm {{ minor_version }} - - nccl {{ nccl_version }} - - ucx-proc=*=gpu - - ucx-py {{ ucx_py_version }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} + - cudatoolkit ={{ cuda_version }} + - libcublas {{ libcublas_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcurand {{ libcurand_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} + - librmm ={{ minor_version }} run: - - cudatoolkit {{ cuda_spec }} - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} - - librmm {{ minor_version }} + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} + - libcurand {{ libcurand_run_version }} + - libcurand-dev {{ libcurand_run_version }} + - libcusolver {{ libcusolver_run_version }} + - libcusolver-dev {{ libcusolver_run_version }} + - libcusparse {{ libcusparse_run_version }} + - libcusparse-dev {{ libcusparse_run_version }} + - librmm ={{ minor_version }} about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 summary: libraft-headers library - name: libraft-distance @@ -72,23 +83,27 @@ outputs: - {{ compiler('cuda') }} requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cxx') }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit {{ cuda_version }}.* - - librmm {{ minor_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} + - libcublas {{ libcublas_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcurand {{ libcurand_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} run: - - cudatoolkit {{ cuda_spec }} - - librmm {{ minor_version }} - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 summary: libraft-distance library - name: libraft-nn @@ -102,28 +117,32 @@ outputs: - {{ compiler('cuda') }} requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cxx') }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit {{ cuda_version }}.* + - {{ pin_subpackage('libraft-headers', exact=True) }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} - faiss-proc=*=cuda - lapack + - libcublas {{ libcublas_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcurand {{ libcurand_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} - libfaiss {{ libfaiss_version }} - - librmm {{ minor_version }} - - {{ pin_subpackage('libraft-headers', exact=True) }} run: - - cudatoolkit {{ cuda_spec }} - faiss-proc=*=cuda - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} - libfaiss {{ libfaiss_version }} - - librmm {{ minor_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 summary: libraft-nn library - name: libraft-tests @@ -137,28 +156,34 @@ outputs: - {{ compiler('cuda') }} requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cxx') }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit {{ cuda_version }}.* - - gmock {{ gtest_version }} - - gtest {{ gtest_version }} - {{ pin_subpackage('libraft-distance', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - {{ pin_subpackage('libraft-nn', exact=True) }} - run: - - cudatoolkit {{ cuda_spec }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} + - libcublas {{ libcublas_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcurand {{ libcurand_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} + run: - {{ pin_subpackage('libraft-distance', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - {{ pin_subpackage('libraft-nn', exact=True) }} + - gmock {{ gtest_version }} + - gtest {{ gtest_version }} about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 summary: libraft tests diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 68e2d5952d..6bc091a219 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -3,10 +3,10 @@ # Usage: # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %} -{% set cuda_major=cuda_version.split('.')[0] %} -{% set py_version=environ.get('CONDA_PY', 36) %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set py_version = environ['CONDA_PY'] %} +{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} package: name: pylibraft @@ -23,36 +23,38 @@ build: requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: + - cuda-python >=11.7.1,<12.0 + - cudatoolkit ={{ cuda_version }} + - cython >=0.29,<0.30 + - libraft-distance {{ version }} + - libraft-headers {{ version }} - python x.x + - rmm ={{ minor_version }} + - scikit-build >=0.13.1 - setuptools - - cython>=0.29,<0.30 - - scikit-build>=0.13.1 - - rmm {{ minor_version }} - - libraft-headers {{ version }} - - libraft-distance {{ version }} - - cudatoolkit {{ cuda_version }}.* - - cuda-python >=11.7.1,<12.0 run: - - python x.x - - libraft-headers {{ version }} - - libraft-distance {{ version }} - - cuda-python >=11.7.1,<12.0 - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-python >=11.7.1,<12.0 + - libraft-distance {{ version }} + - libraft-headers {{ version }} + - python x.x +# TODO: Remove the linux64 tags on tests after disabling gpuCI / Jenkins tests: # [linux64] requirements: # [linux64] - - cudatoolkit {{ cuda_version }}.* # [linux64] + - cudatoolkit ={{ cuda_version }} # [linux64] imports: # [linux64] - pylibraft # [linux64] about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 # license_file: LICENSE summary: pylibraft library diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index 3b42dab182..42d7e3a900 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -13,5 +13,8 @@ sysroot_version: ucx_version: - "1.13.0" +ucx_py_version: + - "0.30.*" + cmake_version: - ">=3.23.1,!=3.25.0" diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 41815db975..3e41e0df17 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -3,11 +3,10 @@ # Usage: # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %} -{% set cuda_major=cuda_version.split('.')[0] %} -{% set py_version=environ.get('CONDA_PY', 36) %} -{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set py_version = environ['CONDA_PY'] %} +{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} package: name: raft-dask @@ -24,47 +23,49 @@ build: requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: + - cuda-python >=11.7.1,<12.0 + - cudatoolkit ={{ cuda_version }} + - cython >=0.29,<0.30 + - nccl >=2.9.9 + - pylibraft {{ version }} - python x.x + - rmm ={{ minor_version }} + - scikit-build >=0.13.1 - setuptools - - cython>=0.29,<0.30 - - scikit-build>=0.13.1 - - rmm {{ minor_version }} - - pylibraft {{ version }} - - cudatoolkit {{ cuda_version }}.* - - cuda-python >=11.7.1,<12.0 - - nccl>=2.9.9 - ucx {{ ucx_version }} - - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu + - ucx-py {{ ucx_py_version }} run: - - python x.x - - dask-cuda {{ minor_version }} + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-python >=11.7.1,<12.0 + - dask >=2022.12.0 + - dask-cuda ={{ minor_version }} + - distributed >=2022.12.0 + - joblib >=0.11 + - nccl >=2.9.9 - pylibraft {{ version }} - - nccl>=2.9.9 - - rmm {{ minor_version }} + - python x.x + - rmm ={{ minor_version }} - ucx >={{ ucx_version }} - - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - - dask>=2022.12.0 - - distributed>=2022.12.0 - - cuda-python >=11.7.1,<12.0 - - joblib >=0.11 - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - ucx-py {{ ucx_py_version }} +# TODO: Remove the linux64 tags on tests after disabling gpuCI / Jenkins tests: # [linux64] requirements: # [linux64] - - cudatoolkit {{ cuda_version }}.* # [linux64] + - cudatoolkit ={{ cuda_version }} # [linux64] imports: # [linux64] - raft_dask # [linux64] about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 # license_file: LICENSE summary: raft-dask library diff --git a/dependencies.yaml b/dependencies.yaml index 7d12d5b28c..52054d9c7d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -10,9 +10,23 @@ files: - cudatoolkit - develop - doc - - notebook - run - test_python + test_cpp: + output: none + includes: + - cudatoolkit + test_python: + output: none + includes: + - cudatoolkit + - py_version + - test_python + checks: + output: none + includes: + - checks + - py_version channels: - rapidsai - rapidsai-nightly @@ -33,7 +47,6 @@ dependencies: packages: - c-compiler - cxx-compiler - - rapids-build-env=23.02.* specific: - output_types: conda matrices: @@ -47,6 +60,11 @@ dependencies: packages: - gcc_linux-aarch64=9.* - sysroot_linux-aarch64==2.17 + checks: + common: + - output_types: [conda, requirements] + packages: + - pre-commit develop: common: - output_types: [conda, requirements] @@ -60,17 +78,46 @@ dependencies: - output_types: conda matrices: - matrix: - cuda: "11.2" + cuda: "11.5" packages: - - cudatoolkit=11.2 + - cudatoolkit=11.5 + - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages + - libcublas-dev>=11.7.3.1,<=11.7.4.6 + - libcublas>=11.7.3.1,<=11.7.4.6 + - libcurand-dev>=10.2.6.48,<=10.2.7.107 + - libcurand>=10.2.6.48,<=10.2.7.107 + - libcusolver-dev>=11.2.1.48,<=11.3.2.107 + - libcusolver>=11.2.1.48,<=11.3.2.107 + - libcusparse-dev>=11.7.0.31,<=11.7.0.107 + - libcusparse>=11.7.0.31,<=11.7.0.107 - matrix: cuda: "11.4" packages: - cudatoolkit=11.4 + - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages + - &libcublas_dev114 libcublas-dev>=11.5.2.43,<=11.6.5.2 + - &libcublas114 libcublas>=11.5.2.43,<=11.6.5.2 + - &libcurand_dev114 libcurand-dev>=10.2.5.43,<=10.2.5.120 + - &libcurand114 libcurand>=10.2.5.43,<=10.2.5.120 + - &libcusolver_dev114 libcusolver-dev>=11.2.0.43,<=11.2.0.120 + - &libcusolver114 libcusolver>=11.2.0.43,<=11.2.0.120 + - &libcusparse_dev114 libcusparse-dev>=11.6.0.43,<=11.6.0.120 + - &libcusparse114 libcusparse>=11.6.0.43,<=11.6.0.120 - matrix: - cuda: "11.5" + cuda: "11.2" packages: - - cudatoolkit=11.5 + - cudatoolkit=11.2 + - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages + # The NVIDIA channel doesn't publish pkgs older than 11.4 for these libs, + # so 11.2 uses 11.4 packages (the oldest available). + - *libcublas_dev114 + - *libcublas114 + - *libcurand_dev114 + - *libcurand114 + - *libcusolver_dev114 + - *libcusolver114 + - *libcusparse_dev114 + - *libcusparse114 doc: common: - output_types: [conda, requirements] @@ -82,13 +129,7 @@ dependencies: - output_types: [conda] packages: - doxygen>=1.8.20 - - rapids-doc-env=23.02.* - sphinx-markdown-tables - notebook: - common: - - output_types: [conda] - packages: - - rapids-notebook-env=23.02.* py_version: specific: - output_types: conda @@ -121,4 +162,8 @@ dependencies: common: - output_types: [conda, requirements] packages: + - cupy - pytest + - pytest-cov + - scikit-learn + - scipy diff --git a/python/pylibraft/.coveragerc b/python/pylibraft/.coveragerc new file mode 100644 index 0000000000..fc087fb9c5 --- /dev/null +++ b/python/pylibraft/.coveragerc @@ -0,0 +1,3 @@ +# Configuration file for Python coverage tests +[run] +source = pylibraft \ No newline at end of file diff --git a/python/raft-dask/.coveragerc b/python/raft-dask/.coveragerc new file mode 100644 index 0000000000..968c4b898a --- /dev/null +++ b/python/raft-dask/.coveragerc @@ -0,0 +1,3 @@ +# Configuration file for Python coverage tests +[run] +source = raft_dask \ No newline at end of file