From 1224ffdfa62b203fa563f499c92172e8ecb5302e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 7 Dec 2022 14:52:45 -0600 Subject: [PATCH 01/48] Add GitHub Actions Workflows. --- .github/workflows/pr.yaml | 48 ++++++++++++++++++++ .github/workflows/test.yaml | 34 ++++++++++++++ ci/build_cpp.sh | 16 +++++++ ci/build_python.sh | 30 +++++++++++++ ci/check_style.sh | 25 +++++++++++ ci/cpu/build.sh | 7 +++ ci/test_cpp.sh | 54 +++++++++++++++++++++++ ci/test_python.sh | 78 +++++++++++++++++++++++++++++++++ conda/recipes/libraft/meta.yaml | 6 +-- 9 files changed, 295 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/pr.yaml create mode 100644 .github/workflows/test.yaml create mode 100755 ci/build_cpp.sh create mode 100755 ci/build_python.sh create mode 100755 ci/check_style.sh create mode 100755 ci/test_cpp.sh create mode 100755 ci/test_python.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000000..3ddb08dd79 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,48 @@ +name: pr + +on: + push: + branches: + - "pull-request/[0-9]+" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pr-builder: + needs: + - checks + - conda-cpp-build + - conda-cpp-tests + - conda-python-build + - conda-python-tests + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@main + checks: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main + with: + build_type: pull-request + conda-cpp-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main + with: + build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main + with: + build_type: pull-request + conda-python-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + with: + build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..34a104082b --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,34 @@ +name: test + +on: + workflow_call: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + +jobs: + conda-cpp-tests: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main + with: + build_type: nightly + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + conda-python-tests: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + with: + build_type: nightly + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh new file mode 100755 index 0000000000..853ae095d3 --- /dev/null +++ b/ci/build_cpp.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-env-update + +export CMAKE_GENERATOR=Ninja + +rapids-print-env + +rapids-logger "Begin cpp build" + +rapids-mamba-retry mambabuild conda/recipes/libraft + +rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh new file mode 100755 index 0000000000..17ec157837 --- /dev/null +++ b/ci/build_python.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-env-update + +export CMAKE_GENERATOR=Ninja + +rapids-print-env + +rapids-logger "Begin py build" + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) + +# TODO: Remove `--no-test` flag once importing on a CPU +# node works correctly +rapids-mamba-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + conda/recipes/pylibraft + +# TODO: Remove `--no-test` flag once importing on a CPU +# node works correctly +rapids-mamba-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + conda/recipes/raft-dask + +rapids-upload-conda-to-s3 python diff --git a/ci/check_style.sh b/ci/check_style.sh new file mode 100755 index 0000000000..a8351ea06b --- /dev/null +++ b/ci/check_style.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright (c) 2020-2022, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-logger "Create checks conda environment" +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file_key checks \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n checks +conda activate checks + +set +e + +FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.02/cmake-format-rapids-cmake.json +export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json +mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) +wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} + +# Run pre-commit checks +pre-commit run --hook-stage manual --all-files --show-diff-on-failure diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 04f9fd008a..92dc663f33 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -31,6 +31,13 @@ fi export GPUCI_CONDA_RETRY_MAX=1 export GPUCI_CONDA_RETRY_SLEEP=30 +# Workaround to keep Jenkins builds working +# until we migrate fully to GitHub Actions +export RAPIDS_CUDA_VERSION="${CUDA}" +export SCCACHE_BUCKET=rapids-sccache +export SCCACHE_REGION=us-west-2 +export SCCACHE_IDLE_TIMEOUT=32768 + # Use Ninja to build export CMAKE_GENERATOR="Ninja" export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld" diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh new file mode 100755 index 0000000000..3f0fbbf13a --- /dev/null +++ b/ci/test_cpp.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +set -euo pipefail + +. /opt/conda/etc/profile.d/conda.sh + +rapids-logger "Generate C++ testing dependencies" +rapids-dependency-file-generator \ + --output conda \ + --file_key test_cpp \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n test + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate test +set -u + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ +mkdir -p "${RAPIDS_TESTS_DIR}" +SUITEERROR=0 + +rapids-print-env + +rapids-mamba-retry install \ + -c "${CPP_CHANNEL}" \ + libraft-headers libraft-distance libraft-nn libraft-tests + +rapids-logger "Check GPU usage" +nvidia-smi + +set +e + +# Run libraft gtests from libraft-tests package +rapids-logger "Run gtests" + +# TODO: exit code handling is too verbose. Find a cleaner solution. + +for gt in "$CONDA_PREFIX"/bin/gtests/libraft/* ; do + test_name=$(basename ${gt}) + echo "Running gtest $test_name" + ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} + + exitcode=$? + if (( ${exitcode} != 0 )); then + SUITEERROR=${exitcode} + echo "FAILED: GTest ${gt}" + fi +done + +exit ${SUITEERROR} diff --git a/ci/test_python.sh b/ci/test_python.sh new file mode 100755 index 0000000000..cae8a4e497 --- /dev/null +++ b/ci/test_python.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +set -euo pipefail + +. /opt/conda/etc/profile.d/conda.sh + +rapids-logger "Generate Python testing dependencies" +rapids-dependency-file-generator \ + --output conda \ + --file_key test_python \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n test + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate test +set -u + +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) + +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} +RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} +mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" +SUITEERROR=0 + +rapids-print-env + +rapids-mamba-retry install \ + -c "${CPP_CHANNEL}" \ + -c "${PYTHON_CHANNEL}" \ + pylibraft raft-dask + +rapids-logger "Check GPU usage" +nvidia-smi + +set +e + +rapids-logger "pytest pylibraft" +pushd python/pylibraft/pylibraft +pytest \ + --cache-clear \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-pylibraft.xml" \ + --cov-config=../.coveragerc \ + --cov=pylibraft \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/pylibraft-coverage.xml" \ + --cov-report=term \ + test +exitcode=$? + +if (( ${exitcode} != 0 )); then + SUITEERROR=${exitcode} + echo "FAILED: 1 or more tests in pylibraft" +fi +popd + +rapids-logger "pytest raft-dask" +pushd python/raft-dask/raft_dask +pytest \ + --cache-clear \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-raft-dask.xml" \ + --cov-config=../.coveragerc \ + --cov=raft_dask \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/raft-dask-coverage.xml" \ + --cov-report=term \ + test +exitcode=$? + +if (( ${exitcode} != 0 )); then + SUITEERROR=${exitcode} + echo "FAILED: 1 or more tests in raft-dask" +fi +popd + +exit ${SUITEERROR} diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 339fa76065..c4c2a2b515 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -30,9 +30,9 @@ outputs: - CMAKE_CUDA_COMPILER_LAUNCHER - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_BUCKET=rapids-sccache - - SCCACHE_REGION=us-west-2 - - SCCACHE_IDLE_TIMEOUT=32768 + - SCCACHE_BUCKET + - SCCACHE_REGION + - SCCACHE_IDLE_TIMEOUT number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: From 7e2924ab4e14a28fe2aa583664df04cbb2138421 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 7 Dec 2022 17:22:37 -0600 Subject: [PATCH 02/48] More GitHub Actions. --- .github/labeler.yml | 2 +- .github/workflows/build.yaml | 57 +++++++++++++++++++++++++++++++ ci/gpu/build.sh | 7 ++++ conda/recipes/libraft/meta.yaml | 2 +- conda/recipes/pylibraft/meta.yaml | 4 +-- dependencies.yaml | 10 ++++++ 6 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build.yaml diff --git a/.github/labeler.yml b/.github/labeler.yml index 9809e2cc2e..56f77e69c0 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -12,5 +12,5 @@ CMake: - '**/CMakeLists.txt' - '**/cmake/**' -gpuCI: +ci: - 'ci/**' diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000000..4e7d76d8eb --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,57 @@ +name: build + +on: + push: + branches: + - "branch-*" + tags: + - v[0-9][0-9].[0-9][0-9].[0-9][0-9] + workflow_call: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + build_type: + type: string + default: nightly + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + cpp-build: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + python-build: + needs: [cpp-build] + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + upload-conda: + needs: [cpp-build, python-build] + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@main + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/raft + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index a24784005b..7665575ea9 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -21,6 +21,13 @@ export PARALLEL_LEVEL=${PARALLEL_LEVEL:-8} export CUDA_REL=${CUDA_VERSION%.*} CONDA_ARTIFACT_PATH=${WORKSPACE}/ci/artifacts/raft/cpu/.conda-bld/ # notice there is no `linux-64` here +# Workaround to keep Jenkins builds working +# until we migrate fully to GitHub Actions +export RAPIDS_CUDA_VERSION="${CUDA}" +export SCCACHE_BUCKET=rapids-sccache +export SCCACHE_REGION=us-west-2 +export SCCACHE_IDLE_TIMEOUT=32768 + # Set home to the job's workspace export HOME=$WORKSPACE diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index c4c2a2b515..d010271253 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -4,7 +4,7 @@ # conda build . -c conda-forge -c nvidia -c rapidsai {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version = '.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} +{% set cuda_version = '.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} {% set ucx_py_version = environ.get('UCX_PY_VERSION') %} {% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 68e2d5952d..b9202077cd 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -4,9 +4,9 @@ # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %} +{% set cuda_version='.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} {% set cuda_major=cuda_version.split('.')[0] %} -{% set py_version=environ.get('CONDA_PY', 36) %} +{% set py_version=environ.get('CONDA_PY', 38) %} package: name: pylibraft diff --git a/dependencies.yaml b/dependencies.yaml index 5906a8a428..f1431c5722 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -13,6 +13,11 @@ files: - notebook - run - test_python + checks: + output: none + includes: + - checks + - py_version channels: - rapidsai - rapidsai-nightly @@ -47,6 +52,11 @@ dependencies: packages: - gcc_linux-aarch64=9.* - sysroot_linux-aarch64==2.17 + checks: + common: + - output_types: [conda, requirements] + packages: + - pre-commit develop: common: - output_types: [conda, requirements] From e1c7f382fd3c484ab740aef3540ee0ab10748307 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 7 Dec 2022 17:28:13 -0600 Subject: [PATCH 03/48] Update conda recipes, .coveragerc. --- conda/recipes/libraft/meta.yaml | 14 ++++++++------ conda/recipes/pylibraft/meta.yaml | 13 +++++++------ conda/recipes/raft-dask/meta.yaml | 15 ++++++++------- python/pylibraft/.coveragerc | 3 +++ python/raft-dask/.coveragerc | 3 +++ 5 files changed, 29 insertions(+), 19 deletions(-) create mode 100644 python/pylibraft/.coveragerc create mode 100644 python/raft-dask/.coveragerc diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index d010271253..7511d0a511 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -23,7 +23,6 @@ outputs: script_env: &script_env - PARALLEL_LEVEL - VERSION_SUFFIX - - PROJECT_FLASH - CMAKE_GENERATOR - CMAKE_C_COMPILER_LAUNCHER - CMAKE_CXX_COMPILER_LAUNCHER @@ -33,6 +32,8 @@ outputs: - SCCACHE_BUCKET - SCCACHE_REGION - SCCACHE_IDLE_TIMEOUT + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: @@ -42,8 +43,9 @@ outputs: - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} - - sysroot_{{ target_platform }} {{ sysroot_version }} - cmake {{ cmake_version }} + - ninja + - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* - libcusolver {{ libcusolver_version }} @@ -61,7 +63,7 @@ outputs: - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 summary: libraft-headers library - name: libraft-distance @@ -97,7 +99,7 @@ outputs: - libcusparse {{ libcusparse_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 summary: libraft-distance library - name: libraft-nn @@ -132,7 +134,7 @@ outputs: - librmm {{ minor_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 summary: libraft-nn library - name: libraft-tests @@ -168,6 +170,6 @@ outputs: - {{ pin_subpackage('libraft-headers', exact=True) }} - {{ pin_subpackage('libraft-nn', exact=True) }} about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 summary: libraft tests diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index b9202077cd..543cdce247 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -3,10 +3,10 @@ # Usage: # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version='.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} -{% set cuda_major=cuda_version.split('.')[0] %} -{% set py_version=environ.get('CONDA_PY', 38) %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set cuda_version = '.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} +{% set py_version = environ.get('CONDA_PY', 38) %} package: name: pylibraft @@ -23,10 +23,11 @@ build: requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - python x.x @@ -52,7 +53,7 @@ tests: # [linux64] - pylibraft # [linux64] about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 # license_file: LICENSE summary: pylibraft library diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index f9a7c58e24..550ff0d3da 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -3,11 +3,11 @@ # Usage: # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %} -{% set cuda_major=cuda_version.split('.')[0] %} -{% set py_version=environ.get('CONDA_PY', 36) %} -{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set cuda_version = '.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} +{% set py_version = environ.get('CONDA_PY', 38) %} +{% set ucx_py_version = environ.get('UCX_PY_VERSION') %} package: name: raft-dask @@ -24,10 +24,11 @@ build: requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - python x.x @@ -64,7 +65,7 @@ tests: # [linux64] - raft_dask # [linux64] about: - home: http://rapids.ai/ + home: https://rapids.ai/ license: Apache-2.0 # license_file: LICENSE summary: raft-dask library diff --git a/python/pylibraft/.coveragerc b/python/pylibraft/.coveragerc new file mode 100644 index 0000000000..fc087fb9c5 --- /dev/null +++ b/python/pylibraft/.coveragerc @@ -0,0 +1,3 @@ +# Configuration file for Python coverage tests +[run] +source = pylibraft \ No newline at end of file diff --git a/python/raft-dask/.coveragerc b/python/raft-dask/.coveragerc new file mode 100644 index 0000000000..968c4b898a --- /dev/null +++ b/python/raft-dask/.coveragerc @@ -0,0 +1,3 @@ +# Configuration file for Python coverage tests +[run] +source = raft_dask \ No newline at end of file From df2702b559399907901154f35ecd4ab6fffba73e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 7 Dec 2022 17:29:40 -0600 Subject: [PATCH 04/48] Disable separate workflow for checks. --- .github/workflows/dependency-files.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .github/workflows/dependency-files.yml diff --git a/.github/workflows/dependency-files.yml b/.github/workflows/dependency-files.yml deleted file mode 100644 index 2c5d41ff1f..0000000000 --- a/.github/workflows/dependency-files.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: dependency-files - -on: - pull_request: - -jobs: - checks: - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main - with: - enable_check_size: false - enable_check_style: false From 804d294e302040ce4606691589396b0daadb84c7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 7 Dec 2022 21:00:09 -0600 Subject: [PATCH 05/48] Add UCX_PY_VERSION. --- ci/build_cpp.sh | 3 +++ ci/build_python.sh | 3 +++ ci/release/update-version.sh | 2 ++ 3 files changed, 8 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 853ae095d3..c9fba5b751 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,6 +7,9 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja +# ucx-py version +export UCX_PY_VERSION='0.30.*' + rapids-print-env rapids-logger "Begin cpp build" diff --git a/ci/build_python.sh b/ci/build_python.sh index 17ec157837..aae6ee164e 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -7,6 +7,9 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja +# ucx-py version +export UCX_PY_VERSION='0.30.*' + rapids-print-env rapids-logger "Begin py build" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 415ab34e34..e0b00c969a 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -52,3 +52,5 @@ done sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/gpu/build.sh sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/cpu/build.sh +sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/build_cpp.sh +sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/build_python.sh From af7e4efc5f9511dfbf039734ffe867dbad299731 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 00:05:30 -0600 Subject: [PATCH 06/48] Add VERSION_SUFFIX. --- ci/build_cpp.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index c9fba5b751..eef9b0074b 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,6 +7,13 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja +# If nightly build, append current YYMMDD to version +if [[ "$RAPIDS_BUILD_TYPE" = "nightly" && "$RAPIDS_REF_NAME" = branch-* ]] ; then + export VERSION_SUFFIX=$(date +%y%m%d) +else + export VERSION_SUFFIX="" +fi + # ucx-py version export UCX_PY_VERSION='0.30.*' From e2f2ca635e8f728c8136734ea3613046a9441c7a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 00:05:52 -0600 Subject: [PATCH 07/48] Add VERSION_SUFFIX. --- ci/build_python.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/build_python.sh b/ci/build_python.sh index aae6ee164e..2580c62890 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -7,6 +7,13 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja +# If nightly build, append current YYMMDD to version +if [[ "$RAPIDS_BUILD_TYPE" = "nightly" && "$RAPIDS_REF_NAME" = branch-* ]] ; then + export VERSION_SUFFIX=$(date +%y%m%d) +else + export VERSION_SUFFIX="" +fi + # ucx-py version export UCX_PY_VERSION='0.30.*' From 841c6cb5c06c70a803a8b36d8b4f3542d4a510b7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 00:20:23 -0600 Subject: [PATCH 08/48] Require ninja for all packages. --- conda/recipes/libraft/meta.yaml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 7511d0a511..0c364833f7 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -50,7 +50,7 @@ outputs: - cudatoolkit {{ cuda_version }}.* - libcusolver {{ libcusolver_version }} - libcusparse {{ libcusparse_version }} - - librmm {{ minor_version }} + - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} @@ -58,7 +58,7 @@ outputs: - cudatoolkit {{ cuda_spec }} - libcusolver {{ libcusolver_version }} - libcusparse {{ libcusparse_version }} - - librmm {{ minor_version }} + - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} @@ -77,21 +77,22 @@ outputs: - {{ compiler('cuda') }} requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* - - librmm {{ minor_version }} + - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} run: - cudatoolkit {{ cuda_spec }} - - librmm {{ minor_version }} + - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} @@ -113,17 +114,18 @@ outputs: - {{ compiler('cuda') }} requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* - faiss-proc=*=cuda - lapack - libfaiss {{ libfaiss_version }} - - librmm {{ minor_version }} + - librmm {{ minor_version }}.* - {{ pin_subpackage('libraft-headers', exact=True) }} run: - cudatoolkit {{ cuda_spec }} @@ -131,7 +133,7 @@ outputs: - libcusolver {{ libcusolver_version }} - libcusparse {{ libcusparse_version }} - libfaiss {{ libfaiss_version }} - - librmm {{ minor_version }} + - librmm {{ minor_version }}.* - {{ pin_subpackage('libraft-headers', exact=True) }} about: home: https://rapids.ai/ @@ -148,10 +150,11 @@ outputs: - {{ compiler('cuda') }} requirements: build: - - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - cmake {{ cmake_version }} + - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* From 3ba607cfe35dfd095ee03a964ea7fb457ada008a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 14:48:30 -0600 Subject: [PATCH 09/48] Update CTK package info. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 3 +++ conda/recipes/libraft/conda_build_config.yaml | 7 +++++-- conda/recipes/libraft/meta.yaml | 5 +++-- dependencies.yaml | 9 +++++++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index a36747d38d..a61fc5852b 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -22,6 +22,9 @@ dependencies: - doxygen>=1.8.20 - faiss-proc=*=cuda - gcc_linux-64=9.* +- libcublas-dev=11.7.4.6 +- libcusolver-dev=11.3.2.107 +- libcusparse-dev=11.7.0.107 - libfaiss>=1.7.0 - ninja - pytest diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index fc77dfc89b..ad65f786f1 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -19,11 +19,14 @@ nccl_version: gtest_version: - "=1.10.0" +libcublas_version: + - ">=11.7.3.1,<12" + libcusolver_version: - - ">=11.2.1" + - ">=11.2.1,<12" libcusparse_version: - - ">=11.5.0" + - ">=11.5.0,<12" libfaiss_version: - "1.7.0 *_cuda" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 0c364833f7..7221340af7 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -48,8 +48,9 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} + - libcublas-dev {{ libcublas_version }} + - libcusolver-dev {{ libcusolver_version }} + - libcusparse-dev {{ libcusparse_version }} - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu diff --git a/dependencies.yaml b/dependencies.yaml index f1431c5722..c911aca889 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -73,14 +73,23 @@ dependencies: cuda: "11.2" packages: - cudatoolkit=11.2 + - libcublas-dev=11.5.2.43 # belongs to cuda-11.4.0 but it's the oldest available + - libcusolver-dev=11.2.0.43 # belongs to cuda-11.4.0 but it's the oldest available + - libcusparse-dev=11.6.0.43 # belongs to cuda-11.4.0 but it's the oldest available - matrix: cuda: "11.4" packages: - cudatoolkit=11.4 + - libcublas-dev=11.6.5.2 + - libcusolver-dev=11.2.0.120 + - libcusparse-dev=11.6.0.120 - matrix: cuda: "11.5" packages: - cudatoolkit=11.5 + - libcublas-dev=11.7.4.6 + - libcusolver-dev=11.3.2.107 + - libcusparse-dev=11.7.0.107 doc: common: - output_types: [conda, requirements] From 4d124d686b3c6c577686a90e763a024f30ae6022 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 16:34:57 -0600 Subject: [PATCH 10/48] Add libcublas-dev to libraft-distance. --- conda/recipes/libraft/meta.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 7221340af7..48599cff42 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -57,6 +57,7 @@ outputs: - ucx-py {{ ucx_py_version }} run: - cudatoolkit {{ cuda_spec }} + - libcublas {{ libcublas_version }} - libcusolver {{ libcusolver_version }} - libcusparse {{ libcusparse_version }} - librmm {{ minor_version }}.* @@ -86,6 +87,7 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* + - libcublas-dev {{ libcublas_version }} - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu @@ -93,12 +95,13 @@ outputs: - {{ pin_subpackage('libraft-headers', exact=True) }} run: - cudatoolkit {{ cuda_spec }} + - libcublas {{ libcublas_version }} + - libcusolver {{ libcusolver_version }} + - libcusparse {{ libcusparse_version }} - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} about: home: https://rapids.ai/ From 9ac1b2a9866f2c76f83380ae1ca23b0b742da48b Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 17:19:56 -0600 Subject: [PATCH 11/48] Change dependencies. --- conda/recipes/libraft/meta.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 48599cff42..7f17f7ba69 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -57,9 +57,9 @@ outputs: - ucx-py {{ ucx_py_version }} run: - cudatoolkit {{ cuda_spec }} - - libcublas {{ libcublas_version }} - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} + - libcublas-dev {{ libcublas_version }} + - libcusolver-dev {{ libcusolver_version }} + - libcusparse-dev {{ libcusparse_version }} - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu @@ -87,7 +87,6 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* - - libcublas-dev {{ libcublas_version }} - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu @@ -95,9 +94,6 @@ outputs: - {{ pin_subpackage('libraft-headers', exact=True) }} run: - cudatoolkit {{ cuda_spec }} - - libcublas {{ libcublas_version }} - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} - librmm {{ minor_version }}.* - nccl {{ nccl_version }} - ucx-proc=*=gpu @@ -171,8 +167,6 @@ outputs: - cudatoolkit {{ cuda_spec }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} - {{ pin_subpackage('libraft-distance', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - {{ pin_subpackage('libraft-nn', exact=True) }} From f7c8365ecf047810b4f7e21dd8dc900b3cbfac45 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 17:25:19 -0600 Subject: [PATCH 12/48] Adopt a radically minimal set of dependencies. --- conda/recipes/libraft/meta.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 7f17f7ba69..02e7555b31 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -86,18 +86,8 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit {{ cuda_version }}.* - - librmm {{ minor_version }}.* - - nccl {{ nccl_version }} - - ucx-proc=*=gpu - - ucx-py {{ ucx_py_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} run: - - cudatoolkit {{ cuda_spec }} - - librmm {{ minor_version }}.* - - nccl {{ nccl_version }} - - ucx-proc=*=gpu - - ucx-py {{ ucx_py_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} about: home: https://rapids.ai/ @@ -121,19 +111,13 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit {{ cuda_version }}.* - faiss-proc=*=cuda - lapack - libfaiss {{ libfaiss_version }} - - librmm {{ minor_version }}.* - {{ pin_subpackage('libraft-headers', exact=True) }} run: - - cudatoolkit {{ cuda_spec }} - faiss-proc=*=cuda - - libcusolver {{ libcusolver_version }} - - libcusparse {{ libcusparse_version }} - libfaiss {{ libfaiss_version }} - - librmm {{ minor_version }}.* - {{ pin_subpackage('libraft-headers', exact=True) }} about: home: https://rapids.ai/ @@ -157,14 +141,12 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit {{ cuda_version }}.* - gmock {{ gtest_version }} - gtest {{ gtest_version }} - {{ pin_subpackage('libraft-distance', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - {{ pin_subpackage('libraft-nn', exact=True) }} run: - - cudatoolkit {{ cuda_spec }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} - {{ pin_subpackage('libraft-distance', exact=True) }} From 96c583b99182ba48a40550a848070b1768f163da Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 17:32:48 -0600 Subject: [PATCH 13/48] More radically minimal dependencies. --- conda/recipes/libraft/meta.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 02e7555b31..687247a8f5 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -52,18 +52,12 @@ outputs: - libcusolver-dev {{ libcusolver_version }} - libcusparse-dev {{ libcusparse_version }} - librmm {{ minor_version }}.* - - nccl {{ nccl_version }} - - ucx-proc=*=gpu - - ucx-py {{ ucx_py_version }} run: - cudatoolkit {{ cuda_spec }} - libcublas-dev {{ libcublas_version }} - libcusolver-dev {{ libcusolver_version }} - libcusparse-dev {{ libcusparse_version }} - librmm {{ minor_version }}.* - - nccl {{ nccl_version }} - - ucx-proc=*=gpu - - ucx-py {{ ucx_py_version }} about: home: https://rapids.ai/ license: Apache-2.0 From 8d9dee61df31aaa86f81101c2d013193c6003a5b Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Dec 2022 17:43:06 -0600 Subject: [PATCH 14/48] Add curand. --- conda/recipes/libraft/conda_build_config.yaml | 5 ++++- conda/recipes/libraft/meta.yaml | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index ad65f786f1..ca20ad4f22 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -22,8 +22,11 @@ gtest_version: libcublas_version: - ">=11.7.3.1,<12" +libcurand_version: + - ">=10.2.7.107,<10.3.1.50" + libcusolver_version: - - ">=11.2.1,<12" + - ">=11.2.1,<11.4.2.57" libcusparse_version: - ">=11.5.0,<12" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 687247a8f5..da8e6d1ae2 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -49,12 +49,14 @@ outputs: host: - cudatoolkit {{ cuda_version }}.* - libcublas-dev {{ libcublas_version }} + - libcurand-dev {{ libcurand_version }} - libcusolver-dev {{ libcusolver_version }} - libcusparse-dev {{ libcusparse_version }} - librmm {{ minor_version }}.* run: - cudatoolkit {{ cuda_spec }} - libcublas-dev {{ libcublas_version }} + - libcurand-dev {{ libcurand_version }} - libcusolver-dev {{ libcusolver_version }} - libcusparse-dev {{ libcusparse_version }} - librmm {{ minor_version }}.* From f4004f7a97e9db8f5d2669a9b0708e16c215c267 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 9 Dec 2022 13:36:31 -0600 Subject: [PATCH 15/48] Add cuda-profiler-api. --- conda/recipes/libraft/conda_build_config.yaml | 3 +++ conda/recipes/libraft/meta.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index ca20ad4f22..e1419f803a 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -19,6 +19,9 @@ nccl_version: gtest_version: - "=1.10.0" +cuda_profiler_api_version: + - ">=11.4.240,<12" + libcublas_version: - ">=11.7.3.1,<12" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index da8e6d1ae2..360eddb00c 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -48,6 +48,7 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* + - cuda-profiler-api {{ cuda_profiler_api_version }} - libcublas-dev {{ libcublas_version }} - libcurand-dev {{ libcurand_version }} - libcusolver-dev {{ libcusolver_version }} @@ -55,6 +56,7 @@ outputs: - librmm {{ minor_version }}.* run: - cudatoolkit {{ cuda_spec }} + - cuda-profiler-api {{ cuda_profiler_api_version }} - libcublas-dev {{ libcublas_version }} - libcurand-dev {{ libcurand_version }} - libcusolver-dev {{ libcusolver_version }} From 9fc0b26140c4ea09da804d35c51a5b744b0f4009 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 12 Dec 2022 14:41:41 -0500 Subject: [PATCH 16/48] add `test_cpp` and `test_python` file entries to `dependencies.yaml` --- conda/environments/all_cuda-115_arch-x86_64.yaml | 1 + dependencies.yaml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 7d6bc7992a..831a608e08 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -28,6 +28,7 @@ dependencies: - libfaiss>=1.7.0=cuda* - ninja - pytest +- pytest-cov - rapids-build-env=23.02.* - rapids-doc-env=23.02.* - rapids-notebook-env=23.02.* diff --git a/dependencies.yaml b/dependencies.yaml index 941bd19713..c264fc1966 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -13,6 +13,16 @@ files: - notebook - run - test_python + test_cpp: + output: none + includes: + - cudatoolkit + test_python: + output: none + includes: + - cudatoolkit + - py_version + - test_python checks: output: none includes: @@ -73,7 +83,7 @@ dependencies: cuda: "11.2" packages: - cudatoolkit=11.2 - - libcublas-dev=11.5.2.43 # belongs to cuda-11.4.0 but it's the oldest available + - libcublas-dev=11.5.2.43 # belongs to cuda-11.4.0 but it's the oldest available - libcusolver-dev=11.2.0.43 # belongs to cuda-11.4.0 but it's the oldest available - libcusparse-dev=11.6.0.43 # belongs to cuda-11.4.0 but it's the oldest available - matrix: @@ -141,3 +151,4 @@ dependencies: - output_types: [conda, requirements] packages: - pytest + - pytest-cov From e8393b750e609bee565d8d65500c23c311e73750 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 12 Dec 2022 14:33:35 -0600 Subject: [PATCH 17/48] Use ucx_py_version in conda_build_config.yaml. --- ci/build_cpp.sh | 10 ---------- ci/build_python.sh | 10 ---------- ci/release/update-version.sh | 3 +-- conda/recipes/libraft/meta.yaml | 1 - conda/recipes/raft-dask/conda_build_config.yaml | 3 +++ conda/recipes/raft-dask/meta.yaml | 1 - 6 files changed, 4 insertions(+), 24 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index eef9b0074b..853ae095d3 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,16 +7,6 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja -# If nightly build, append current YYMMDD to version -if [[ "$RAPIDS_BUILD_TYPE" = "nightly" && "$RAPIDS_REF_NAME" = branch-* ]] ; then - export VERSION_SUFFIX=$(date +%y%m%d) -else - export VERSION_SUFFIX="" -fi - -# ucx-py version -export UCX_PY_VERSION='0.30.*' - rapids-print-env rapids-logger "Begin cpp build" diff --git a/ci/build_python.sh b/ci/build_python.sh index 2580c62890..17ec157837 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -7,16 +7,6 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja -# If nightly build, append current YYMMDD to version -if [[ "$RAPIDS_BUILD_TYPE" = "nightly" && "$RAPIDS_REF_NAME" = branch-* ]] ; then - export VERSION_SUFFIX=$(date +%y%m%d) -else - export VERSION_SUFFIX="" -fi - -# ucx-py version -export UCX_PY_VERSION='0.30.*' - rapids-print-env rapids-logger "Begin py build" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index e0b00c969a..0b6410f9c9 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -52,5 +52,4 @@ done sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/gpu/build.sh sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/cpu/build.sh -sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/build_cpp.sh -sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/build_python.sh +sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/raft-dask/conda_build_config.yaml diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 360eddb00c..1199db1cbf 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -6,7 +6,6 @@ {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set cuda_version = '.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} -{% set ucx_py_version = environ.get('UCX_PY_VERSION') %} {% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 package: diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index 3b42dab182..42d7e3a900 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -13,5 +13,8 @@ sysroot_version: ucx_version: - "1.13.0" +ucx_py_version: + - "0.30.*" + cmake_version: - ">=3.23.1,!=3.25.0" diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 550ff0d3da..0280b14046 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -7,7 +7,6 @@ {% set cuda_version = '.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} {% set py_version = environ.get('CONDA_PY', 38) %} -{% set ucx_py_version = environ.get('UCX_PY_VERSION') %} package: name: raft-dask From e45a13355f8a53a9ab3fb0b8a205dca666d5f2a7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 12 Dec 2022 14:49:07 -0600 Subject: [PATCH 18/48] Remove VERSION_SUFFIX from script_env. --- conda/recipes/libraft/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 1199db1cbf..0b30fe18ef 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -21,7 +21,6 @@ outputs: build: script_env: &script_env - PARALLEL_LEVEL - - VERSION_SUFFIX - CMAKE_GENERATOR - CMAKE_C_COMPILER_LAUNCHER - CMAKE_CXX_COMPILER_LAUNCHER From bb0e341bafa9391a54b5ef118d285d06a5e8fbd1 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 12 Dec 2022 23:55:10 -0600 Subject: [PATCH 19/48] Update all_cuda-115_arch-x86_64.yaml --- conda/environments/all_cuda-115_arch-x86_64.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 831a608e08..eafb3572b3 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -34,6 +34,8 @@ dependencies: - rapids-notebook-env=23.02.* - rmm=23.02.* - scikit-build>=0.13.1 +- scikit-learn +- scipy - sphinx-markdown-tables - sysroot_linux-64==2.17 - ucx-proc=*=gpu From 6899d5671daa574502c96376ab0aeb8550389008 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 12 Dec 2022 23:56:25 -0600 Subject: [PATCH 20/48] Add scipy, scikit-learn to Python test dependencies. --- dependencies.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index c264fc1966..ccf6796c49 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -152,3 +152,5 @@ dependencies: packages: - pytest - pytest-cov + - scikit-learn + - scipy From 80324737e423139e6d805a91b6b760dbc3f90fd1 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 11:15:43 -0600 Subject: [PATCH 21/48] Add cupy to Python test dependencies. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index eafb3572b3..7cf230e2c5 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -14,6 +14,7 @@ dependencies: - cmake>=3.23.1,!=3.25.0 - cuda-python >=11.7.1,<12.0 - cudatoolkit=11.5 +- cupy - cxx-compiler - cython>=0.29,<0.30 - dask-cuda=23.02.* diff --git a/dependencies.yaml b/dependencies.yaml index ccf6796c49..a59d634250 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -150,6 +150,7 @@ dependencies: common: - output_types: [conda, requirements] packages: + - cupy - pytest - pytest-cov - scikit-learn From fe21aa4037073eda2310b5577150c2694f6c03f4 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 11:28:43 -0600 Subject: [PATCH 22/48] Pin runtime libraries and not just the "-dev" package. This is needed to prevent the CUDA 11 "-dev" package from matching CUDA 12 runtime libraries due to its >= pinnings. --- conda/recipes/libraft/meta.yaml | 8 ++++++++ dependencies.yaml | 3 +++ 2 files changed, 11 insertions(+) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 0b30fe18ef..72b113a5e7 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -48,17 +48,25 @@ outputs: - cudatoolkit {{ cuda_version }}.* - cuda-profiler-api {{ cuda_profiler_api_version }} - libcublas-dev {{ libcublas_version }} + - libcublas {{ libcublas_version }} - libcurand-dev {{ libcurand_version }} + - libcurand {{ libcurand_version }} - libcusolver-dev {{ libcusolver_version }} + - libcusolver {{ libcusolver_version }} - libcusparse-dev {{ libcusparse_version }} + - libcusparse {{ libcusparse_version }} - librmm {{ minor_version }}.* run: - cudatoolkit {{ cuda_spec }} - cuda-profiler-api {{ cuda_profiler_api_version }} - libcublas-dev {{ libcublas_version }} + - libcublas {{ libcublas_version }} - libcurand-dev {{ libcurand_version }} + - libcurand {{ libcurand_version }} - libcusolver-dev {{ libcusolver_version }} + - libcusolver {{ libcusolver_version }} - libcusparse-dev {{ libcusparse_version }} + - libcusparse {{ libcusparse_version }} - librmm {{ minor_version }}.* about: home: https://rapids.ai/ diff --git a/dependencies.yaml b/dependencies.yaml index a59d634250..b61479d96a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -84,6 +84,7 @@ dependencies: packages: - cudatoolkit=11.2 - libcublas-dev=11.5.2.43 # belongs to cuda-11.4.0 but it's the oldest available + - libcurand-dev=10.2.5.43 # belongs to cuda-11.4.0 but it's the oldest available - libcusolver-dev=11.2.0.43 # belongs to cuda-11.4.0 but it's the oldest available - libcusparse-dev=11.6.0.43 # belongs to cuda-11.4.0 but it's the oldest available - matrix: @@ -91,6 +92,7 @@ dependencies: packages: - cudatoolkit=11.4 - libcublas-dev=11.6.5.2 + - libcurand-dev=10.2.5.120 - libcusolver-dev=11.2.0.120 - libcusparse-dev=11.6.0.120 - matrix: @@ -98,6 +100,7 @@ dependencies: packages: - cudatoolkit=11.5 - libcublas-dev=11.7.4.6 + - libcurand-dev=10.2.7.107 - libcusolver-dev=11.3.2.107 - libcusparse-dev=11.7.0.107 doc: From 6597c8a05eced9da271450ad725ea8efa87a89c2 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 11:36:43 -0600 Subject: [PATCH 23/48] Remove "set +e". --- ci/check_style.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/check_style.sh b/ci/check_style.sh index a8351ea06b..e120fe418a 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -14,8 +14,6 @@ rapids-dependency-file-generator \ rapids-mamba-retry env create --force -f env.yaml -n checks conda activate checks -set +e - FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.02/cmake-format-rapids-cmake.json export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) From 9ae9fe452fbedc2a212698284046ede0df251cdd Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 11:43:11 -0600 Subject: [PATCH 24/48] Add libcurand-dev. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 7cf230e2c5..1693f1b08d 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -24,6 +24,7 @@ dependencies: - faiss-proc=*=cuda - gcc_linux-64=9.* - libcublas-dev=11.7.4.6 +- libcurand-dev=10.2.7.107 - libcusolver-dev=11.3.2.107 - libcusparse-dev=11.7.0.107 - libfaiss>=1.7.0=cuda* From 44d211967251405c061a31f047406145bcd12cd6 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 11:44:09 -0600 Subject: [PATCH 25/48] Disable rapids-cmake format file fetching. --- ci/check_style.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ci/check_style.sh b/ci/check_style.sh index e120fe418a..be3ac3f4b8 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -14,10 +14,5 @@ rapids-dependency-file-generator \ rapids-mamba-retry env create --force -f env.yaml -n checks conda activate checks -FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.02/cmake-format-rapids-cmake.json -export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json -mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) -wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} - # Run pre-commit checks pre-commit run --hook-stage manual --all-files --show-diff-on-failure From b2fe582c9385691ed2848fd94b79253c2aa9a705 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 13:54:24 -0600 Subject: [PATCH 26/48] Update recipes. --- conda/recipes/libraft/meta.yaml | 4 ++-- conda/recipes/pylibraft/meta.yaml | 4 ++-- conda/recipes/raft-dask/meta.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 72b113a5e7..8cc6680707 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -4,7 +4,7 @@ # conda build . -c conda-forge -c nvidia -c rapidsai {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version = '.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} +{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} {% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 @@ -57,7 +57,7 @@ outputs: - libcusparse {{ libcusparse_version }} - librmm {{ minor_version }}.* run: - - cudatoolkit {{ cuda_spec }} + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda_profiler_api_version }} - libcublas-dev {{ libcublas_version }} - libcublas {{ libcublas_version }} diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 543cdce247..7576cb4126 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -4,9 +4,9 @@ # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version = '.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} {% set py_version = environ.get('CONDA_PY', 38) %} +{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} package: name: pylibraft diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 0280b14046..e8e0c646df 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -4,9 +4,9 @@ # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version = '.'.join(environ.get('RAPIDS_CUDA_VERSION', '11.5.2').split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} {% set py_version = environ.get('CONDA_PY', 38) %} +{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} package: name: raft-dask From f6d69b293e01d88c105a8bfcfec162e86b0b1fe5 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 17:22:01 -0600 Subject: [PATCH 27/48] Use .* pinnings on minor versions. --- conda/recipes/pylibraft/meta.yaml | 2 +- conda/recipes/raft-dask/meta.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 7576cb4126..f227b11857 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -34,7 +34,7 @@ requirements: - setuptools - cython>=0.29,<0.30 - scikit-build>=0.13.1 - - rmm {{ minor_version }} + - rmm {{ minor_version }}.* - libraft-headers {{ version }} - libraft-distance {{ version }} - cudatoolkit {{ cuda_version }}.* diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index d8f576db99..eb9f96ea56 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -34,7 +34,7 @@ requirements: - setuptools - cython>=0.29,<0.30 - scikit-build>=0.13.1 - - rmm {{ minor_version }} + - rmm {{ minor_version }}.* - pylibraft {{ version }} - cudatoolkit {{ cuda_version }}.* - cuda-python >=11.7.1,<12.0 @@ -44,10 +44,10 @@ requirements: - ucx-proc=*=gpu run: - python x.x - - dask-cuda {{ minor_version }} + - dask-cuda {{ minor_version }}.* - pylibraft {{ version }} - nccl>=2.9.9 - - rmm {{ minor_version }} + - rmm {{ minor_version }}.* - ucx >={{ ucx_version }} - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu From 33e788b316dcecf2512e7f41536c7a314a115f29 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 17:55:11 -0600 Subject: [PATCH 28/48] Set lower bound of libcusparse-dev to oldest available version. --- conda/recipes/libraft/conda_build_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index e1419f803a..71e486bd05 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -32,7 +32,7 @@ libcusolver_version: - ">=11.2.1,<11.4.2.57" libcusparse_version: - - ">=11.5.0,<12" + - ">=11.6.0.43,<12" libfaiss_version: - "1.7.0 *_cuda" From 870d046b377f8718bfb395a027f3e7e6bf48b79c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 13 Dec 2022 22:20:49 -0600 Subject: [PATCH 29/48] Use cpu16 nodes for raft C++ builds. --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3ddb08dd79..ca2e2356c0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -28,6 +28,7 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main with: build_type: pull-request + node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit From efd55565de99c5eeded4c7bd66b04bdb764e7128 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Wed, 14 Dec 2022 18:33:28 -0500 Subject: [PATCH 30/48] Update ci/build_python.sh --- ci/build_python.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_python.sh b/ci/build_python.sh index 17ec157837..9f5b117f5d 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -25,6 +25,7 @@ rapids-mamba-retry mambabuild \ rapids-mamba-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-dask rapids-upload-conda-to-s3 python From 406212d27816ca0e5048949d68ff78a38f9a3943 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sun, 18 Dec 2022 10:34:56 -0600 Subject: [PATCH 31/48] Add cuda-profiler-api to dependencies. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 1 + dependencies.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index ab34847d7d..aaefc344af 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -12,6 +12,7 @@ dependencies: - clang-tools=11.1.0 - clang=11.1.0 - cmake>=3.23.1,!=3.25.0 +- cuda-profiler-api>=11.4.240,<11.8.86 - cuda-python >=11.7.1,<12.0 - cudatoolkit=11.5 - cupy diff --git a/dependencies.yaml b/dependencies.yaml index d26ea79775..099c1e53fb 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -83,6 +83,7 @@ dependencies: cuda: "11.2" packages: - cudatoolkit=11.2 + - cuda-profiler-api>=11.4.240,<11.8.86 # only cuda-11.4 is available until cuda-11.8 - libcublas-dev=11.5.2.43 # belongs to cuda-11.4.0 but it's the oldest available - libcurand-dev=10.2.5.43 # belongs to cuda-11.4.0 but it's the oldest available - libcusolver-dev=11.2.0.43 # belongs to cuda-11.4.0 but it's the oldest available @@ -91,6 +92,7 @@ dependencies: cuda: "11.4" packages: - cudatoolkit=11.4 + - cuda-profiler-api>=11.4.240,<11.8.86 # only cuda-11.4 is available until cuda-11.8 - libcublas-dev=11.6.5.2 - libcurand-dev=10.2.5.120 - libcusolver-dev=11.2.0.120 @@ -99,6 +101,7 @@ dependencies: cuda: "11.5" packages: - cudatoolkit=11.5 + - cuda-profiler-api>=11.4.240,<11.8.86 # only cuda-11.4 is available until cuda-11.8 - libcublas-dev=11.7.4.6 - libcurand-dev=10.2.7.107 - libcusolver-dev=11.3.2.107 From 43f22713163253a59c4cdbffd9229121b214caef Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 10:21:46 -0500 Subject: [PATCH 32/48] rm `rapids-*-env` packages --- conda/environments/all_cuda-115_arch-x86_64.yaml | 3 --- dependencies.yaml | 3 --- 2 files changed, 6 deletions(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index aaefc344af..bb18350c58 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -32,9 +32,6 @@ dependencies: - ninja - pytest - pytest-cov -- rapids-build-env=23.02.* -- rapids-doc-env=23.02.* -- rapids-notebook-env=23.02.* - rmm=23.02.* - scikit-build>=0.13.1 - scikit-learn diff --git a/dependencies.yaml b/dependencies.yaml index 099c1e53fb..8431cfb357 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -48,7 +48,6 @@ dependencies: packages: - c-compiler - cxx-compiler - - rapids-build-env=23.02.* specific: - output_types: conda matrices: @@ -117,13 +116,11 @@ dependencies: - output_types: [conda] packages: - doxygen>=1.8.20 - - rapids-doc-env=23.02.* - sphinx-markdown-tables notebook: common: - output_types: [conda] packages: - - rapids-notebook-env=23.02.* py_version: specific: - output_types: conda From c747915a50fd8f3b47363665f51c6ce62f73b350 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 10:21:55 -0500 Subject: [PATCH 33/48] rm empty `notebook` dependency list `raft` doesn't have notebooks anyway --- dependencies.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 8431cfb357..af4cd54a96 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -10,7 +10,6 @@ files: - cudatoolkit - develop - doc - - notebook - run - test_python test_cpp: @@ -117,10 +116,6 @@ dependencies: packages: - doxygen>=1.8.20 - sphinx-markdown-tables - notebook: - common: - - output_types: [conda] - packages: py_version: specific: - output_types: conda From 9524b2abf7df43fc22e4902b9c78daa019ce9856 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 10:24:52 -0500 Subject: [PATCH 34/48] update `cuda-profiler-api` version spec the `11.4.x` `cuda-profiler-api` package is only available for `aarch64`. therefore, this commit increases the upper limit of the version specifier for this package to avoid conda solver issues. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 2 +- dependencies.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index bb18350c58..b969018b08 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -12,7 +12,7 @@ dependencies: - clang-tools=11.1.0 - clang=11.1.0 - cmake>=3.23.1,!=3.25.0 -- cuda-profiler-api>=11.4.240,<11.8.86 +- cuda-profiler-api>=11.4.240,<=11.8.86 - cuda-python >=11.7.1,<12.0 - cudatoolkit=11.5 - cupy diff --git a/dependencies.yaml b/dependencies.yaml index af4cd54a96..e8e81039b6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -81,7 +81,7 @@ dependencies: cuda: "11.2" packages: - cudatoolkit=11.2 - - cuda-profiler-api>=11.4.240,<11.8.86 # only cuda-11.4 is available until cuda-11.8 + - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - libcublas-dev=11.5.2.43 # belongs to cuda-11.4.0 but it's the oldest available - libcurand-dev=10.2.5.43 # belongs to cuda-11.4.0 but it's the oldest available - libcusolver-dev=11.2.0.43 # belongs to cuda-11.4.0 but it's the oldest available @@ -90,7 +90,7 @@ dependencies: cuda: "11.4" packages: - cudatoolkit=11.4 - - cuda-profiler-api>=11.4.240,<11.8.86 # only cuda-11.4 is available until cuda-11.8 + - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - libcublas-dev=11.6.5.2 - libcurand-dev=10.2.5.120 - libcusolver-dev=11.2.0.120 @@ -99,7 +99,7 @@ dependencies: cuda: "11.5" packages: - cudatoolkit=11.5 - - cuda-profiler-api>=11.4.240,<11.8.86 # only cuda-11.4 is available until cuda-11.8 + - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - libcublas-dev=11.7.4.6 - libcurand-dev=10.2.7.107 - libcusolver-dev=11.3.2.107 From d263d78c2ec08ddaad3ace64a099d0faae5905c5 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 16:25:23 -0500 Subject: [PATCH 35/48] update CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a0528e4011..fc4fcd458b 100755 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,3 +18,4 @@ conda/ @rapidsai/ops-codeowners **/Dockerfile @rapidsai/ops-codeowners **/.dockerignore @rapidsai/ops-codeowners docker/ @rapidsai/ops-codeowners +dependencies.yaml @rapidsai/ops-codeowners From 6e14104b57b4313cd7a757ad71c8fce54b102ec9 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 16:25:51 -0500 Subject: [PATCH 36/48] rm fallback value for `CONDA_PY` --- conda/recipes/pylibraft/meta.yaml | 2 +- conda/recipes/raft-dask/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index f227b11857..4b3c94729c 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -4,7 +4,7 @@ # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ.get('CONDA_PY', 38) %} +{% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index eb9f96ea56..1336386409 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -4,7 +4,7 @@ # conda build . -c conda-forge -c numba -c rapidsai -c pytorch {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ.get('CONDA_PY', 38) %} +{% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} From 92c3b6b0e21bddb2c71aad3f111bc546bcf8f8f1 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 16:28:08 -0500 Subject: [PATCH 37/48] disable Jenkins uploads --- ci/cpu/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 92dc663f33..2f0e2b94ca 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -130,5 +130,6 @@ fi # UPLOAD - Conda packages ################################################################################ -gpuci_logger "Upload conda packages" -source ci/cpu/upload.sh +# Uploads disabled due to new GH Actions implementation +# gpuci_logger "Upload conda packages" +# source ci/cpu/upload.sh From bb560e6e19bdc349b7494df215b22dd1cff04ef1 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 16:29:31 -0500 Subject: [PATCH 38/48] replace `-c` w/ `--channel` --- ci/test_cpp.sh | 2 +- ci/test_python.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 3f0fbbf13a..d8538bdf47 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -26,7 +26,7 @@ SUITEERROR=0 rapids-print-env rapids-mamba-retry install \ - -c "${CPP_CHANNEL}" \ + --channel "${CPP_CHANNEL}" \ libraft-headers libraft-distance libraft-nn libraft-tests rapids-logger "Check GPU usage" diff --git a/ci/test_python.sh b/ci/test_python.sh index cae8a4e497..597046b364 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -30,8 +30,8 @@ SUITEERROR=0 rapids-print-env rapids-mamba-retry install \ - -c "${CPP_CHANNEL}" \ - -c "${PYTHON_CHANNEL}" \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ pylibraft raft-dask rapids-logger "Check GPU usage" From 1826f610efb3a6a88a66b1e6e6ff518ff7abf0d7 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 16:32:56 -0500 Subject: [PATCH 39/48] explicitly include `raft` C++ pkgs in `test_python.sh` --- ci/test_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index 597046b364..eb458d2a5a 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -32,7 +32,7 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - pylibraft raft-dask + libraft-distance libraft-headers pylibraft raft-dask rapids-logger "Check GPU usage" nvidia-smi From cc3d796f7e8dde23665ea726b80fbf44ab3e2c60 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 17:30:28 -0500 Subject: [PATCH 40/48] split existing `libcu*` dependencies into `host`/`run` versions --- conda/recipes/libraft/conda_build_config.yaml | 47 ++++++++++++++----- conda/recipes/libraft/meta.yaml | 36 +++++++------- 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 71e486bd05..399dd198eb 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -19,20 +19,43 @@ nccl_version: gtest_version: - "=1.10.0" -cuda_profiler_api_version: - - ">=11.4.240,<12" +libfaiss_version: + - "1.7.0 *_cuda" -libcublas_version: - - ">=11.7.3.1,<12" +# The CTK libraries below are missing from the conda-forge::cudatoolkit +# package. The "*_host_*" version specifiers correspond to `11.5` packages and the +# "*_run_*" version specifiers correspond to `11.x` packages. -libcurand_version: - - ">=10.2.7.107,<10.3.1.50" +libcublas_host_version: + - ">=11.7.3.1,<=11.7.4.6" -libcusolver_version: - - ">=11.2.1,<11.4.2.57" +libcublas_run_version: + - ">=11.5.2.43,<=11.11.3.6" -libcusparse_version: - - ">=11.6.0.43,<12" +libcurand_host_version: + - ">=10.2.6.48,<=10.2.7.107" -libfaiss_version: - - "1.7.0 *_cuda" +libcurand_run_version: + - ">=10.2.5.43,<=10.3.0.86" + +libcusolver_host_version: + - ">=11.2.1.48,<=11.3.2.107" + +libcusolver_run_version: + - ">=11.2.0.43,<=11.4.1.48" + +libcusparse_host_version: + - ">=11.7.0.31,<=11.7.0.107" + +libcusparse_run_version: + - ">=11.6.0.43,<=11.7.5.86" + +# `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all +# architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the +# "*_run_*" version specifiers correspond to `11.x` packages. + +cuda_profiler_api_host_version: + - ">=11.8.86,<12" + +cuda_profiler_api_run_version: + - ">=11.4.240,<12" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 8cc6680707..29f6c62469 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -46,27 +46,27 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cudatoolkit {{ cuda_version }}.* - - cuda-profiler-api {{ cuda_profiler_api_version }} - - libcublas-dev {{ libcublas_version }} - - libcublas {{ libcublas_version }} - - libcurand-dev {{ libcurand_version }} - - libcurand {{ libcurand_version }} - - libcusolver-dev {{ libcusolver_version }} - - libcusolver {{ libcusolver_version }} - - libcusparse-dev {{ libcusparse_version }} - - libcusparse {{ libcusparse_version }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcublas {{ libcublas_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcurand {{ libcurand_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} + - libcusparse {{ libcusparse_host_version }} - librmm {{ minor_version }}.* run: - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - cuda-profiler-api {{ cuda_profiler_api_version }} - - libcublas-dev {{ libcublas_version }} - - libcublas {{ libcublas_version }} - - libcurand-dev {{ libcurand_version }} - - libcurand {{ libcurand_version }} - - libcusolver-dev {{ libcusolver_version }} - - libcusolver {{ libcusolver_version }} - - libcusparse-dev {{ libcusparse_version }} - - libcusparse {{ libcusparse_version }} + - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - libcublas-dev {{ libcublas_run_version }} + - libcublas {{ libcublas_run_version }} + - libcurand-dev {{ libcurand_run_version }} + - libcurand {{ libcurand_run_version }} + - libcusolver-dev {{ libcusolver_run_version }} + - libcusolver {{ libcusolver_run_version }} + - libcusparse-dev {{ libcusparse_run_version }} + - libcusparse {{ libcusparse_run_version }} - librmm {{ minor_version }}.* about: home: https://rapids.ai/ From 0d72fb1051d8522aed444d8a245cf979d14e5b42 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 17:31:08 -0500 Subject: [PATCH 41/48] add `libcu*` packages as `host` dependencies for other `libraft` packages This is to ensure that the other `raft` packages get built with the correct `libcu*` versioned packages. This commit also alphabetizes the lists. --- conda/recipes/libraft/meta.yaml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 29f6c62469..0d5ea82b72 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -91,6 +91,15 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - {{ pin_subpackage('libraft-headers', exact=True) }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcublas {{ libcublas_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcurand {{ libcurand_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} + - libcusparse {{ libcusparse_host_version }} run: - {{ pin_subpackage('libraft-headers', exact=True) }} about: @@ -115,10 +124,19 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: + - {{ pin_subpackage('libraft-headers', exact=True) }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} - faiss-proc=*=cuda - lapack + - libcublas {{ libcublas_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcurand {{ libcurand_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} - libfaiss {{ libfaiss_version }} - - {{ pin_subpackage('libraft-headers', exact=True) }} run: - faiss-proc=*=cuda - libfaiss {{ libfaiss_version }} @@ -145,11 +163,20 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - gmock {{ gtest_version }} - - gtest {{ gtest_version }} - {{ pin_subpackage('libraft-distance', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - {{ pin_subpackage('libraft-nn', exact=True) }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} + - gmock {{ gtest_version }} + - gtest {{ gtest_version }} + - libcublas {{ libcublas_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcurand {{ libcurand_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} run: - gmock {{ gtest_version }} - gtest {{ gtest_version }} From 1bd89656696fd017471b5de78ce9057aebdfe7f4 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 17:40:40 -0500 Subject: [PATCH 42/48] update `dependencies.yaml` this commit updates `dependencies.yaml` to: - include both `-dev` and non `-dev` packages of the `libcu*` packages - use a version range instead of a specific version pin for the `libcu*` packages --- dependencies.yaml | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index e8e81039b6..1aefbf61e9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -82,28 +82,42 @@ dependencies: packages: - cudatoolkit=11.2 - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - - libcublas-dev=11.5.2.43 # belongs to cuda-11.4.0 but it's the oldest available - - libcurand-dev=10.2.5.43 # belongs to cuda-11.4.0 but it's the oldest available - - libcusolver-dev=11.2.0.43 # belongs to cuda-11.4.0 but it's the oldest available - - libcusparse-dev=11.6.0.43 # belongs to cuda-11.4.0 but it's the oldest available + # The NVIDIA channel doesn't publish pkgs older than 11.4 for these libs, + # so 11.2 uses 11.4 packages (the oldest available). + - *libcublas_dev114 + - *libcublas114 + - *libcurand_dev114 + - *libcurand114 + - *libcusolver_dev114 + - *libcusolver114 + - *libcusparse_dev114 + - *libcusparse114 - matrix: cuda: "11.4" packages: - cudatoolkit=11.4 - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - - libcublas-dev=11.6.5.2 - - libcurand-dev=10.2.5.120 - - libcusolver-dev=11.2.0.120 - - libcusparse-dev=11.6.0.120 + - &libcublas_dev114 libcublas-dev>=11.5.2.43,<=11.6.5.2 + - &libcublas114 libcublas>=11.5.2.43,<=11.6.5.2 + - &libcurand_dev114 libcurand-dev>=10.2.5.43,<=10.2.5.120 + - &libcurand114 libcurand>=10.2.5.43,<=10.2.5.120 + - &libcusolver_dev114 libcusolver-dev>=11.2.0.43,<=11.2.0.120 + - &libcusolver114 libcusolver>=11.2.0.43,<=11.2.0.120 + - &libcusparse_dev114 libcusparse-dev>=11.6.0.43,<=11.6.0.120 + - &libcusparse114 libcusparse>=11.6.0.43,<=11.6.0.120 - matrix: cuda: "11.5" packages: - cudatoolkit=11.5 - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - - libcublas-dev=11.7.4.6 - - libcurand-dev=10.2.7.107 - - libcusolver-dev=11.3.2.107 - - libcusparse-dev=11.7.0.107 + - libcublas-dev>=11.7.3.1,<=11.7.4.6 + - libcublas>=11.7.3.1,<=11.7.4.6 + - libcurand-dev>=10.2.6.48,<=10.2.7.107 + - libcurand>=10.2.6.48,<=10.2.7.107 + - libcusolver-dev>=11.2.1.48,<=11.3.2.107 + - libcusolver>=11.2.1.48,<=11.3.2.107 + - libcusparse-dev>=11.7.0.31,<=11.7.0.107 + - libcusparse>=11.7.0.31,<=11.7.0.107 doc: common: - output_types: [conda, requirements] From db58ca3ff41645dfc20baff431797e1bfcc1459a Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 17:42:23 -0500 Subject: [PATCH 43/48] reorder `cudatoolkit` list items in `dependencies.yaml` this moves `11.2` after `11.4` so that the `11.4` YAML anchors work with `11.2` --- dependencies.yaml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 1aefbf61e9..52054d9c7d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -78,20 +78,18 @@ dependencies: - output_types: conda matrices: - matrix: - cuda: "11.2" + cuda: "11.5" packages: - - cudatoolkit=11.2 + - cudatoolkit=11.5 - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - # The NVIDIA channel doesn't publish pkgs older than 11.4 for these libs, - # so 11.2 uses 11.4 packages (the oldest available). - - *libcublas_dev114 - - *libcublas114 - - *libcurand_dev114 - - *libcurand114 - - *libcusolver_dev114 - - *libcusolver114 - - *libcusparse_dev114 - - *libcusparse114 + - libcublas-dev>=11.7.3.1,<=11.7.4.6 + - libcublas>=11.7.3.1,<=11.7.4.6 + - libcurand-dev>=10.2.6.48,<=10.2.7.107 + - libcurand>=10.2.6.48,<=10.2.7.107 + - libcusolver-dev>=11.2.1.48,<=11.3.2.107 + - libcusolver>=11.2.1.48,<=11.3.2.107 + - libcusparse-dev>=11.7.0.31,<=11.7.0.107 + - libcusparse>=11.7.0.31,<=11.7.0.107 - matrix: cuda: "11.4" packages: @@ -106,18 +104,20 @@ dependencies: - &libcusparse_dev114 libcusparse-dev>=11.6.0.43,<=11.6.0.120 - &libcusparse114 libcusparse>=11.6.0.43,<=11.6.0.120 - matrix: - cuda: "11.5" + cuda: "11.2" packages: - - cudatoolkit=11.5 + - cudatoolkit=11.2 - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - - libcublas-dev>=11.7.3.1,<=11.7.4.6 - - libcublas>=11.7.3.1,<=11.7.4.6 - - libcurand-dev>=10.2.6.48,<=10.2.7.107 - - libcurand>=10.2.6.48,<=10.2.7.107 - - libcusolver-dev>=11.2.1.48,<=11.3.2.107 - - libcusolver>=11.2.1.48,<=11.3.2.107 - - libcusparse-dev>=11.7.0.31,<=11.7.0.107 - - libcusparse>=11.7.0.31,<=11.7.0.107 + # The NVIDIA channel doesn't publish pkgs older than 11.4 for these libs, + # so 11.2 uses 11.4 packages (the oldest available). + - *libcublas_dev114 + - *libcublas114 + - *libcurand_dev114 + - *libcurand114 + - *libcusolver_dev114 + - *libcusolver114 + - *libcusparse_dev114 + - *libcusparse114 doc: common: - output_types: [conda, requirements] From 338636a361af4a7bce72efd6d70873886af87aeb Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 17:45:25 -0500 Subject: [PATCH 44/48] regenerate dependency files --- conda/environments/all_cuda-115_arch-x86_64.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index b969018b08..18e0a8187f 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -24,10 +24,14 @@ dependencies: - doxygen>=1.8.20 - faiss-proc=*=cuda - gcc_linux-64=9.* -- libcublas-dev=11.7.4.6 -- libcurand-dev=10.2.7.107 -- libcusolver-dev=11.3.2.107 -- libcusparse-dev=11.7.0.107 +- libcublas-dev>=11.7.3.1,<=11.7.4.6 +- libcublas>=11.7.3.1,<=11.7.4.6 +- libcurand-dev>=10.2.6.48,<=10.2.7.107 +- libcurand>=10.2.6.48,<=10.2.7.107 +- libcusolver-dev>=11.2.1.48,<=11.3.2.107 +- libcusolver>=11.2.1.48,<=11.3.2.107 +- libcusparse-dev>=11.7.0.31,<=11.7.0.107 +- libcusparse>=11.7.0.31,<=11.7.0.107 - libfaiss>=1.7.0=cuda* - ninja - pytest From f0d9e8d0db9d70bb8144bd8432b8903517e10523 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 19 Dec 2022 23:55:38 -0500 Subject: [PATCH 45/48] change `.*` to `=` for consistency --- conda/recipes/libraft/meta.yaml | 6 +++--- conda/recipes/pylibraft/meta.yaml | 6 +++--- conda/recipes/raft-dask/meta.yaml | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 0d5ea82b72..f0f67658f7 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -45,7 +45,7 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit {{ cuda_version }}.* + - cudatoolkit={{ cuda_version }} - cuda-profiler-api {{ cuda_profiler_api_host_version }} - libcublas-dev {{ libcublas_host_version }} - libcublas {{ libcublas_host_version }} @@ -55,7 +55,7 @@ outputs: - libcusolver {{ libcusolver_host_version }} - libcusparse-dev {{ libcusparse_host_version }} - libcusparse {{ libcusparse_host_version }} - - librmm {{ minor_version }}.* + - librmm={{ minor_version }} run: - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda_profiler_api_run_version }} @@ -67,7 +67,7 @@ outputs: - libcusolver {{ libcusolver_run_version }} - libcusparse-dev {{ libcusparse_run_version }} - libcusparse {{ libcusparse_run_version }} - - librmm {{ minor_version }}.* + - librmm={{ minor_version }} about: home: https://rapids.ai/ license: Apache-2.0 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 4b3c94729c..63e858dc04 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -34,10 +34,10 @@ requirements: - setuptools - cython>=0.29,<0.30 - scikit-build>=0.13.1 - - rmm {{ minor_version }}.* + - rmm={{ minor_version }} - libraft-headers {{ version }} - libraft-distance {{ version }} - - cudatoolkit {{ cuda_version }}.* + - cudatoolkit={{ cuda_version }} - cuda-python >=11.7.1,<12.0 run: - python x.x @@ -48,7 +48,7 @@ requirements: tests: # [linux64] requirements: # [linux64] - - cudatoolkit {{ cuda_version }}.* # [linux64] + - cudatoolkit={{ cuda_version }} # [linux64] imports: # [linux64] - pylibraft # [linux64] diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 1336386409..31f6e0e92f 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -34,9 +34,9 @@ requirements: - setuptools - cython>=0.29,<0.30 - scikit-build>=0.13.1 - - rmm {{ minor_version }}.* + - rmm={{ minor_version }} - pylibraft {{ version }} - - cudatoolkit {{ cuda_version }}.* + - cudatoolkit={{ cuda_version }} - cuda-python >=11.7.1,<12.0 - nccl>=2.9.9 - ucx {{ ucx_version }} @@ -44,10 +44,10 @@ requirements: - ucx-proc=*=gpu run: - python x.x - - dask-cuda {{ minor_version }}.* + - dask-cuda={{ minor_version }} - pylibraft {{ version }} - nccl>=2.9.9 - - rmm {{ minor_version }}.* + - rmm={{ minor_version }} - ucx >={{ ucx_version }} - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu @@ -59,7 +59,7 @@ requirements: tests: # [linux64] requirements: # [linux64] - - cudatoolkit {{ cuda_version }}.* # [linux64] + - cudatoolkit={{ cuda_version }} # [linux64] imports: # [linux64] - raft_dask # [linux64] From 435a6c392aadfd1fe86b1e7445d516213918d7be Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 20 Dec 2022 12:19:32 -0500 Subject: [PATCH 46/48] add space between pkg & version specifiers --- conda/recipes/libraft/meta.yaml | 6 +++--- conda/recipes/pylibraft/meta.yaml | 10 +++++----- conda/recipes/raft-dask/meta.yaml | 22 +++++++++++----------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index f0f67658f7..659c4f5d8d 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -45,7 +45,7 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit={{ cuda_version }} + - cudatoolkit ={{ cuda_version }} - cuda-profiler-api {{ cuda_profiler_api_host_version }} - libcublas-dev {{ libcublas_host_version }} - libcublas {{ libcublas_host_version }} @@ -55,7 +55,7 @@ outputs: - libcusolver {{ libcusolver_host_version }} - libcusparse-dev {{ libcusparse_host_version }} - libcusparse {{ libcusparse_host_version }} - - librmm={{ minor_version }} + - librmm ={{ minor_version }} run: - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda_profiler_api_run_version }} @@ -67,7 +67,7 @@ outputs: - libcusolver {{ libcusolver_run_version }} - libcusparse-dev {{ libcusparse_run_version }} - libcusparse {{ libcusparse_run_version }} - - librmm={{ minor_version }} + - librmm ={{ minor_version }} about: home: https://rapids.ai/ license: Apache-2.0 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 63e858dc04..158221ecff 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -32,12 +32,12 @@ requirements: host: - python x.x - setuptools - - cython>=0.29,<0.30 - - scikit-build>=0.13.1 - - rmm={{ minor_version }} + - cython >=0.29,<0.30 + - scikit-build >=0.13.1 + - rmm ={{ minor_version }} - libraft-headers {{ version }} - libraft-distance {{ version }} - - cudatoolkit={{ cuda_version }} + - cudatoolkit ={{ cuda_version }} - cuda-python >=11.7.1,<12.0 run: - python x.x @@ -48,7 +48,7 @@ requirements: tests: # [linux64] requirements: # [linux64] - - cudatoolkit={{ cuda_version }} # [linux64] + - cudatoolkit ={{ cuda_version }} # [linux64] imports: # [linux64] - pylibraft # [linux64] diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 31f6e0e92f..8015dd2ae9 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -32,34 +32,34 @@ requirements: host: - python x.x - setuptools - - cython>=0.29,<0.30 - - scikit-build>=0.13.1 - - rmm={{ minor_version }} + - cython >=0.29,<0.30 + - scikit-build >=0.13.1 + - rmm ={{ minor_version }} - pylibraft {{ version }} - - cudatoolkit={{ cuda_version }} + - cudatoolkit ={{ cuda_version }} - cuda-python >=11.7.1,<12.0 - - nccl>=2.9.9 + - nccl >=2.9.9 - ucx {{ ucx_version }} - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu run: - python x.x - - dask-cuda={{ minor_version }} + - dask-cuda ={{ minor_version }} - pylibraft {{ version }} - - nccl>=2.9.9 - - rmm={{ minor_version }} + - nccl >=2.9.9 + - rmm ={{ minor_version }} - ucx >={{ ucx_version }} - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - - dask>=2022.12.0 - - distributed>=2022.12.0 + - dask >=2022.12.0 + - distributed >=2022.12.0 - cuda-python >=11.7.1,<12.0 - joblib >=0.11 - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} tests: # [linux64] requirements: # [linux64] - - cudatoolkit={{ cuda_version }} # [linux64] + - cudatoolkit ={{ cuda_version }} # [linux64] imports: # [linux64] - raft_dask # [linux64] From d102c812f820ace7449df938e19a2531f57aec25 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 20 Dec 2022 13:12:40 -0500 Subject: [PATCH 47/48] alphabetize `meta.yaml` lists --- conda/recipes/libraft/meta.yaml | 52 +++++++++++++++---------------- conda/recipes/pylibraft/meta.yaml | 22 ++++++------- conda/recipes/raft-dask/meta.yaml | 32 +++++++++---------- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 659c4f5d8d..b0d6c47ee9 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -20,18 +20,18 @@ outputs: script: build_libraft_headers.sh build: script_env: &script_env - - PARALLEL_LEVEL - - CMAKE_GENERATOR + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_GENERATOR + - PARALLEL_LEVEL - SCCACHE_BUCKET - - SCCACHE_REGION - SCCACHE_IDLE_TIMEOUT - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY + - SCCACHE_REGION + - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: @@ -45,28 +45,28 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit ={{ cuda_version }} - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - libcublas-dev {{ libcublas_host_version }} + - cudatoolkit ={{ cuda_version }} - libcublas {{ libcublas_host_version }} - - libcurand-dev {{ libcurand_host_version }} + - libcublas-dev {{ libcublas_host_version }} - libcurand {{ libcurand_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} + - libcurand-dev {{ libcurand_host_version }} - libcusolver {{ libcusolver_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} - librmm ={{ minor_version }} run: - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda_profiler_api_run_version }} - - libcublas-dev {{ libcublas_run_version }} - libcublas {{ libcublas_run_version }} - - libcurand-dev {{ libcurand_run_version }} + - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - - libcusolver-dev {{ libcusolver_run_version }} + - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} - - libcusparse-dev {{ libcusparse_run_version }} + - libcusolver-dev {{ libcusolver_run_version }} - libcusparse {{ libcusparse_run_version }} + - libcusparse-dev {{ libcusparse_run_version }} - librmm ={{ minor_version }} about: home: https://rapids.ai/ @@ -84,22 +84,22 @@ outputs: requirements: build: - {{ compiler('c') }} - - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - {{ pin_subpackage('libraft-headers', exact=True) }} - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - libcublas-dev {{ libcublas_host_version }} - libcublas {{ libcublas_host_version }} - - libcurand-dev {{ libcurand_host_version }} + - libcublas-dev {{ libcublas_host_version }} - libcurand {{ libcurand_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} + - libcurand-dev {{ libcurand_host_version }} - libcusolver {{ libcusolver_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} run: - {{ pin_subpackage('libraft-headers', exact=True) }} about: @@ -118,8 +118,8 @@ outputs: requirements: build: - {{ compiler('c') }} - - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} @@ -157,8 +157,8 @@ outputs: requirements: build: - {{ compiler('c') }} - - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} @@ -178,11 +178,11 @@ outputs: - libcusparse {{ libcusparse_host_version }} - libcusparse-dev {{ libcusparse_host_version }} run: - - gmock {{ gtest_version }} - - gtest {{ gtest_version }} - {{ pin_subpackage('libraft-distance', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - {{ pin_subpackage('libraft-nn', exact=True) }} + - gmock {{ gtest_version }} + - gtest {{ gtest_version }} about: home: https://rapids.ai/ license: Apache-2.0 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 158221ecff..56c662e099 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -30,21 +30,21 @@ requirements: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - python x.x - - setuptools + - cuda-python >=11.7.1,<12.0 + - cudatoolkit ={{ cuda_version }} - cython >=0.29,<0.30 - - scikit-build >=0.13.1 - - rmm ={{ minor_version }} - - libraft-headers {{ version }} - libraft-distance {{ version }} - - cudatoolkit ={{ cuda_version }} - - cuda-python >=11.7.1,<12.0 - run: - - python x.x - libraft-headers {{ version }} - - libraft-distance {{ version }} - - cuda-python >=11.7.1,<12.0 + - python x.x + - rmm ={{ minor_version }} + - scikit-build >=0.13.1 + - setuptools + run: - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-python >=11.7.1,<12.0 + - libraft-distance {{ version }} + - libraft-headers {{ version }} + - python x.x tests: # [linux64] requirements: # [linux64] diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 8015dd2ae9..0d7a9de1ab 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -30,32 +30,32 @@ requirements: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - python x.x - - setuptools - - cython >=0.29,<0.30 - - scikit-build >=0.13.1 - - rmm ={{ minor_version }} - - pylibraft {{ version }} - - cudatoolkit ={{ cuda_version }} - cuda-python >=11.7.1,<12.0 + - cudatoolkit ={{ cuda_version }} + - cython >=0.29,<0.30 - nccl >=2.9.9 + - pylibraft {{ version }} + - python x.x + - rmm ={{ minor_version }} + - scikit-build >=0.13.1 + - setuptools - ucx {{ ucx_version }} - - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu + - ucx-py {{ ucx_py_version }} run: - - python x.x + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-python >=11.7.1,<12.0 + - dask >=2022.12.0 - dask-cuda ={{ minor_version }} - - pylibraft {{ version }} + - distributed >=2022.12.0 + - joblib >=0.11 - nccl >=2.9.9 + - pylibraft {{ version }} + - python x.x - rmm ={{ minor_version }} - ucx >={{ ucx_version }} - - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - - dask >=2022.12.0 - - distributed >=2022.12.0 - - cuda-python >=11.7.1,<12.0 - - joblib >=0.11 - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - ucx-py {{ ucx_py_version }} tests: # [linux64] requirements: # [linux64] From 47ce3a86c05be17e62ceef7647e9136c71d23c57 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 20 Dec 2022 16:21:15 -0600 Subject: [PATCH 48/48] Minor updates. --- ci/build_python.sh | 4 +--- conda/recipes/pylibraft/meta.yaml | 3 ++- conda/recipes/raft-dask/meta.yaml | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 9f5b117f5d..b20fd51bca 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -13,15 +13,13 @@ rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -# TODO: Remove `--no-test` flag once importing on a CPU +# TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-mamba-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ conda/recipes/pylibraft -# TODO: Remove `--no-test` flag once importing on a CPU -# node works correctly rapids-mamba-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 56c662e099..6bc091a219 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -46,9 +46,10 @@ requirements: - libraft-headers {{ version }} - python x.x +# TODO: Remove the linux64 tags on tests after disabling gpuCI / Jenkins tests: # [linux64] requirements: # [linux64] - - cudatoolkit ={{ cuda_version }} # [linux64] + - cudatoolkit ={{ cuda_version }} # [linux64] imports: # [linux64] - pylibraft # [linux64] diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 0d7a9de1ab..3e41e0df17 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -57,9 +57,10 @@ requirements: - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} +# TODO: Remove the linux64 tags on tests after disabling gpuCI / Jenkins tests: # [linux64] requirements: # [linux64] - - cudatoolkit ={{ cuda_version }} # [linux64] + - cudatoolkit ={{ cuda_version }} # [linux64] imports: # [linux64] - raft_dask # [linux64]