From 3c3cefd0ad6245d2e88971c7969a911378a15659 Mon Sep 17 00:00:00 2001 From: Jordan Jacobelli Date: Fri, 3 Jun 2022 17:00:52 +0200 Subject: [PATCH] Consolidate C++ conda recipes and add libraft-tests package (#641) This PR includes the following changes: - Adds a `libraft-tests` package to the `libraft-split` recipe - This is a prerequisite for removing "Project Flash" from our build/CI scripts - The `libraft-tests` package was added as an additional output to the `libraft-split` recipe - Consolidates remaining C++ recipes into `libraft-split` recipe - This gets rid of a lot of duplicate code between the recipes and reduces the number of times we have to call conda build in our CI scripts - Migrate the "from sources" builds done in GPU tests by building packages using conda. This is done for the following reasons: - This is required step to improve the Ops CI/CD setup to a more convenient pipeline - This is required to start using conda compilers and `mamba` to build RAPIDS packages - This prevent us from manually managing and installing the dependencies in GPU job - This ensure the packages can be installed (no conda conflict while installing) - This ensure the tests are running and working against the package content and not the build results. Currently the Python packages are not tested Dependency version specs are now specified in `conda/recipes/libraft/conda_build_config.yaml`. The exception here is the version spec used for cudatoolkit since that comes from an environment variable in the CI process. Authors: - Jordan Jacobelli (https://github.com/Ethyling) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/raft/pull/641 --- ci/cpu/build.sh | 35 ++--- ci/cpu/upload.sh | 34 +--- ci/gpu/build.sh | 75 +++------ .../build_libraft_distance.sh} | 0 .../build_libraft_headers.sh} | 0 .../build.sh => libraft/build_libraft_nn.sh} | 0 conda/recipes/libraft/build_libraft_tests.sh | 5 + conda/recipes/libraft/conda_build_config.yaml | 14 ++ conda/recipes/libraft/meta.yaml | 146 ++++++++++++++++++ conda/recipes/libraft_distance/meta.yaml | 61 -------- conda/recipes/libraft_headers/meta.yaml | 60 ------- conda/recipes/libraft_nn/meta.yaml | 60 ------- cpp/CMakeLists.txt | 3 +- python/pylibraft/pylibraft/test/__init__.py | 14 -- python/raft/raft/test/__init__.py | 14 -- 15 files changed, 204 insertions(+), 317 deletions(-) rename conda/recipes/{libraft_distance/build.sh => libraft/build_libraft_distance.sh} (100%) rename conda/recipes/{libraft_headers/build.sh => libraft/build_libraft_headers.sh} (100%) rename conda/recipes/{libraft_nn/build.sh => libraft/build_libraft_nn.sh} (100%) create mode 100644 conda/recipes/libraft/build_libraft_tests.sh create mode 100644 conda/recipes/libraft/conda_build_config.yaml create mode 100644 conda/recipes/libraft/meta.yaml delete mode 100644 conda/recipes/libraft_distance/meta.yaml delete mode 100644 conda/recipes/libraft_headers/meta.yaml delete mode 100644 conda/recipes/libraft_nn/meta.yaml delete mode 100644 python/pylibraft/pylibraft/test/__init__.py delete mode 100644 python/raft/raft/test/__init__.py diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 438647963a..e8e6c6535c 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -22,7 +22,9 @@ cd $WORKSPACE # If nightly build, append current YYMMDD to version if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then - export VERSION_SUFFIX=`date +%y%m%d` + export VERSION_SUFFIX=$(date +%y%m%d) +else + export VERSION_SUFFIX="" fi # Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) @@ -65,13 +67,8 @@ conda list --show-channel-urls # FIX Added to deal with Anancoda SSL verification issues during conda builds conda config --set ssl_verify False -# FIXME: for now, force the building of all packages so they are built on a -# machine with a single CUDA version, then have the gpu/build.sh script simply -# install. This should eliminate a mismatch between different CUDA versions on -# cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat. if [ "$BUILD_LIBRAFT" == "1" ]; then - BUILD_RAFT=1 - # If we are doing CUDA + Python builds, libraft package is located at ${CONDA_BLD_DIR} + # If we are doing CUDA builds, libraft package is located at ${CONDA_BLD_DIR} CONDA_LOCAL_CHANNEL="${CONDA_BLD_DIR}" else # If we are doing Python builds only, libraft package is placed here by Project Flash @@ -85,29 +82,17 @@ gpuci_mamba_retry install -c conda-forge boa ############################################################################### if [ "$BUILD_LIBRAFT" == "1" ]; then - gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers" + gpuci_logger "Building conda packages for libraft-nn, libraft-distance, libraft-headers and libraft-tests" if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance + gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft else - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers - gpuci_logger "`ls ${CONDA_BLD_DIR}/work`" - mkdir -p ${CONDA_BLD_DIR}/libraft_headers/work - mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_headers/work - - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn - gpuci_logger "`ls ${CONDA_BLD_DIR}/work`" - mkdir -p ${CONDA_BLD_DIR}/libraft_nn/work - mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_nn/work - - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance + gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft gpuci_logger "`ls ${CONDA_BLD_DIR}/work`" - mkdir -p ${CONDA_BLD_DIR}/libraft_distance/work - mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_distance/work + mkdir -p ${CONDA_BLD_DIR}/libraft/work + mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work fi else - gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers" + gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance, libraft-headers and libraft-tests" fi if [ "$BUILD_RAFT" == '1' ]; then diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index 9126423008..dfd6ea3986 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -10,7 +10,7 @@ export GPUCI_RETRY_MAX=3 export GPUCI_RETRY_SLEEP=30 # Set label option. -# LABEL_OPTION="--label testing" +#LABEL_OPTION="--label testing" LABEL_OPTION="--label main" # Skip uploads unless BUILD_MODE == "branch" @@ -25,18 +25,6 @@ if [ -z "$MY_UPLOAD_KEY" ]; then return 0 fi -################################################################################ -# SETUP - Get conda file output locations -################################################################################ - -gpuci_logger "Get conda file output locations" - -export LIBRAFT_HEADERS_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_headers --output` -export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_nn --output` -export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_distance --output` -export PYRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pyraft --python=$PYTHON --output` -export PYLIBRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output` - ################################################################################ # UPLOAD - Conda packages ################################################################################ @@ -44,24 +32,14 @@ export PYLIBRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${C gpuci_logger "Starting conda uploads" if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then - - test -e ${LIBRAFT_HEADERS_FILE} - echo "Upload libraft-headers" - echo ${LIBRAFT_HEADERS_FILE} - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress - - test -e ${LIBRAFT_NN_FILE} - echo "Upload libraft-nn" - echo ${LIBRAFT_NN_FILE} - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_NN_FILE} --no-progress - - test -e ${LIBRAFT_DISTANCE_FILE} - echo "Upload libraft-distance" - echo ${LIBRAFT_DISTANCE_FILE} - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress + LIBRAFT_FILES=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft --output) + echo "Upload libraft-headers, libraft-nn, libraft-distance and libraft-tests" + gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress ${LIBRAFT_FILES} fi if [[ "$BUILD_RAFT" == "1" ]]; then + PYRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pyraft --python=$PYTHON --output) + PYLIBRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output) test -e ${PYRAFT_FILE} echo "Upload pyraft" echo ${PYRAFT_FILE} diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 05e36613a4..79521a9969 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -25,18 +25,11 @@ export HOME="$WORKSPACE" cd "$WORKSPACE" export GIT_DESCRIBE_TAG=`git describe --tags` export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` +unset GIT_DESCRIBE_TAG # ucx-py version export UCX_PY_VERSION='0.27.*' -export CMAKE_CUDA_COMPILER_LAUNCHER="sccache" -export CMAKE_CXX_COMPILER_LAUNCHER="sccache" -export CMAKE_C_COMPILER_LAUNCHER="sccache" -export SCCACHE_S3_KEY_PREFIX="libraft-$(uname -m)" -export SCCACHE_BUCKET="rapids-sccache" -export SCCACHE_REGION="us-west-2" -export SCCACHE_IDLE_TIMEOUT="32768" - ################################################################################ # SETUP - Check environment ################################################################################ @@ -47,40 +40,14 @@ env gpuci_logger "Check GPU usage" nvidia-smi -# temporary usage of gpuci_mamba_retry install with packages listed here, looking into -# using the repos yaml files for this gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids -gpuci_logger "Installing packages needed for RAFT" -gpuci_mamba_retry install -y -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \ - "cudatoolkit=${CUDA_REL}" \ - "libcusolver>=11.2.1" \ - "cudf=${MINOR_VERSION}" \ - "rmm=${MINOR_VERSION}" \ - "breathe" \ - "dask-cudf=${MINOR_VERSION}" \ - "dask-cuda=${MINOR_VERSION}" \ - "ucx-py=${UCX_PY_VERSION}" \ - "rapids-build-env=${MINOR_VERSION}.*" \ - "rapids-notebook-env=${MINOR_VERSION}.*" - -if [ "$(arch)" = "x86_64" ]; then - gpuci_mamba_retry install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \ - "rapids-doc-env=${MINOR_VERSION}.*" -fi - -# Install the master version of dask, distributed, and dask-ml -gpuci_logger "Install the master version of dask and distributed" -set -x -pip install "git+https://github.com/dask/distributed.git@2022.05.2" --upgrade --no-deps -pip install "git+https://github.com/dask/dask.git@2022.05.2" --upgrade --no-deps -set +x # Install pre-built conda packages from previous CI step gpuci_logger "Install libraft conda packages from CPU job" -export LIBRAFT_CONDA_PACKAGES="$WORKSPACE/ci/artifacts/raft/cpu/.conda-bld/" # notice there is no `linux-64` here -gpuci_mamba_retry install -c "${LIBRAFT_CONDA_PACKAGES}" libraft-headers libraft-distance libraft-nn +CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/raft/cpu/.conda-bld/" # notice there is no `linux-64` here +gpuci_mamba_retry install -c "${CONDA_ARTIFACT_PATH}" libraft-headers libraft-distance libraft-nn libraft-tests gpuci_logger "Check compiler versions" python --version @@ -96,24 +63,16 @@ conda list --show-channel-urls # BUILD - Build RAFT tests ################################################################################ -gpuci_logger "Adding ${CONDA_PREFIX}/lib to LD_LIBRARY_PATH" - -export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH - -gpuci_logger "Build C++ and Python targets" -# These should link against the existing shared libs -if hasArg --skip-tests; then - "$WORKSPACE/build.sh" pyraft pylibraft libraft -v -else - "$WORKSPACE/build.sh" pyraft pylibraft libraft tests bench -v -fi +gpuci_logger "Build and install Python targets" +CONDA_BLD_DIR="$WORKSPACE/.conda-bld" +gpuci_mamba_retry install boa +gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pyraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}" +gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pylibraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}" +gpuci_mamba_retry install -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" pyraft pylibraft gpuci_logger "sccache stats" sccache --show-stats -gpuci_logger "Building docs" -"$WORKSPACE/build.sh" docs -v - ################################################################################ # TEST - Run GoogleTest and py.tests for RAFT ################################################################################ @@ -123,22 +82,30 @@ if hasArg --skip-tests; then exit 0 fi +# Install the master version of dask, distributed, and dask-ml +gpuci_logger "Install the master version of dask and distributed" +set -x +pip install "git+https://github.com/dask/distributed.git@2022.05.2" --upgrade --no-deps +pip install "git+https://github.com/dask/dask.git@2022.05.2" --upgrade --no-deps +set +x + gpuci_logger "Check GPU usage" nvidia-smi gpuci_logger "GoogleTest for raft" -cd "$WORKSPACE/cpp/build" -GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" ./test_raft +cd "$WORKSPACE" +GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" "$CONDA_PREFIX/bin/libraft/gtests/test_raft" gpuci_logger "Python pytest for pyraft" -cd "$WORKSPACE/python/raft" +cd "$WORKSPACE/python/raft/raft/test" python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-pyraft.xml" -v -s gpuci_logger "Python pytest for pylibraft" -cd "$WORKSPACE/python/pylibraft" +cd "$WORKSPACE/python/pylibraft/pylibraft/test" python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-pylibraft.xml" -v -s if [ "$(arch)" = "x86_64" ]; then gpuci_logger "Building docs" + gpuci_mamba_retry install "rapids-doc-env=${MINOR_VERSION}.*" "$WORKSPACE/build.sh" docs -v fi diff --git a/conda/recipes/libraft_distance/build.sh b/conda/recipes/libraft/build_libraft_distance.sh similarity index 100% rename from conda/recipes/libraft_distance/build.sh rename to conda/recipes/libraft/build_libraft_distance.sh diff --git a/conda/recipes/libraft_headers/build.sh b/conda/recipes/libraft/build_libraft_headers.sh similarity index 100% rename from conda/recipes/libraft_headers/build.sh rename to conda/recipes/libraft/build_libraft_headers.sh diff --git a/conda/recipes/libraft_nn/build.sh b/conda/recipes/libraft/build_libraft_nn.sh similarity index 100% rename from conda/recipes/libraft_nn/build.sh rename to conda/recipes/libraft/build_libraft_nn.sh diff --git a/conda/recipes/libraft/build_libraft_tests.sh b/conda/recipes/libraft/build_libraft_tests.sh new file mode 100644 index 0000000000..040a2f8b8c --- /dev/null +++ b/conda/recipes/libraft/build_libraft_tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Copyright (c) 2022, NVIDIA CORPORATION. + +./build.sh tests bench -v --allgpuarch --no-nvtx +cmake --install cpp/build --component testing diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml new file mode 100644 index 0000000000..874335c231 --- /dev/null +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -0,0 +1,14 @@ +cmake_version: + - ">=3.20.1,!=3.23.0" + +nccl_version: + - ">=2.9.9" + +gtest_version: + - "=1.10.0" + +libcusolver_version: + - ">=11.2.1" + +libfaiss_version: + - "1.7.0 *_cuda" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml new file mode 100644 index 0000000000..5321cf0187 --- /dev/null +++ b/conda/recipes/libraft/meta.yaml @@ -0,0 +1,146 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +# Usage: +# conda build . -c conda-forge -c nvidia -c rapidsai +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set cuda_version = '.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} +{% set cuda_major = cuda_version.split('.')[0] %} +{% set ucx_py_version = environ.get('UCX_PY_VERSION') %} +{% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 + +package: + name: libraft-split + +source: + git_url: ../../.. + +outputs: + - name: libraft-headers + version: {{ version }} + script: build_libraft_headers.sh + build: + script_env: &script_env + - CC + - CXX + - CUDAHOSTCXX + - PARALLEL_LEVEL + - VERSION_SUFFIX + - PROJECT_FLASH + - CMAKE_GENERATOR + - CMAKE_C_COMPILER_LAUNCHER + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_CUDA_COMPILER_LAUNCHER + - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] + - SCCACHE_BUCKET=rapids-sccache + - SCCACHE_REGION=us-west-2 + - SCCACHE_IDLE_TIMEOUT=32768 + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + requirements: + build: + - cmake {{ cmake_version }} + host: + - cudatoolkit {{ cuda_version }}.* + - libcusolver {{ libcusolver_version }} + - librmm {{ minor_version }} + - nccl {{ nccl_version }} + - ucx-proc=*=gpu + - ucx-py {{ ucx_py_version }} + run: + - cudatoolkit {{ cuda_spec }} + - libcusolver {{ libcusolver_version }} + - librmm {{ minor_version }} + - nccl {{ nccl_version }} + - ucx-proc=*=gpu + - ucx-py {{ ucx_py_version }} + about: + home: http://rapids.ai/ + license: Apache-2.0 + summary: libraft-headers library + - name: libraft-distance + version: {{ version }} + script: build_libraft_distance.sh + build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: *script_env + requirements: + build: + - cmake {{ cmake_version }} + host: + - cudatoolkit {{ cuda_version }}.* + - librmm {{ minor_version }} + - nccl {{ nccl_version }} + - ucx-proc=*=gpu + - ucx-py {{ ucx_py_version }} + - {{ pin_subpackage('libraft-headers', exact=True) }} + run: + - cudatoolkit {{ cuda_spec }} + - libcusolver {{ libcusolver_version }} + - librmm {{ minor_version }} + - nccl {{ nccl_version }} + - ucx-proc=*=gpu + - ucx-py {{ ucx_py_version }} + - {{ pin_subpackage('libraft-headers', exact=True) }} + about: + home: http://rapids.ai/ + license: Apache-2.0 + summary: libraft-distance library + - name: libraft-nn + version: {{ version }} + script: build_libraft_nn.sh + build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: *script_env + requirements: + build: + - cmake {{ cmake_version }} + host: + - cudatoolkit {{ cuda_version }}.* + - faiss-proc=*=cuda + - lapack + - libfaiss {{ libfaiss_version }} + - librmm {{ minor_version }} + - {{ pin_subpackage('libraft-headers', exact=True) }} + run: + - cudatoolkit {{ cuda_spec }} + - faiss-proc=*=cuda + - libcusolver {{ libcusolver_version }} + - libfaiss {{ libfaiss_version }} + - librmm {{ minor_version }} + - {{ pin_subpackage('libraft-headers', exact=True) }} + about: + home: http://rapids.ai/ + license: Apache-2.0 + summary: libraft-nn library + - name: libraft-tests + version: {{ version }} + script: build_libraft_tests.sh + build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: *script_env + requirements: + build: + - cmake {{ cmake_version }} + host: + - cudatoolkit {{ cuda_version }}.* + - gmock {{ gtest_version }} + - gtest {{ gtest_version }} + - {{ pin_subpackage('libraft-distance', exact=True) }} + - {{ pin_subpackage('libraft-headers', exact=True) }} + - {{ pin_subpackage('libraft-nn', exact=True) }} + run: + - cudatoolkit {{ cuda_spec }} + - gmock {{ gtest_version }} + - gtest {{ gtest_version }} + - {{ pin_subpackage('libraft-distance', exact=True) }} + - {{ pin_subpackage('libraft-headers', exact=True) }} + - {{ pin_subpackage('libraft-nn', exact=True) }} + about: + home: http://rapids.ai/ + license: Apache-2.0 + summary: libraft tests diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml deleted file mode 100644 index 9b78bd15f3..0000000000 --- a/conda/recipes/libraft_distance/meta.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# Usage: -# conda build . -c conda-forge -c nvidia -c rapidsai -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} -{% set cuda_major=cuda_version.split('.')[0] %} -{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} -package: - name: libraft-distance - version: {{ version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - CC - - CXX - - CUDAHOSTCXX - - PARALLEL_LEVEL - - VERSION_SUFFIX - - PROJECT_FLASH - - CMAKE_GENERATOR - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_BUCKET=rapids-sccache - - SCCACHE_REGION=us-west-2 - - SCCACHE_IDLE_TIMEOUT=32768 - -requirements: - build: - - cmake>=3.20.1,!=3.23.0 - host: - - libraft-headers {{ version }} - - nccl>=2.9.9 - - cudatoolkit {{ cuda_version }}.* - - ucx-py {{ ucx_py_version }} - - ucx-proc=*=gpu - - gtest=1.10.0 - - librmm {{ minor_version }} - run: - - libraft-headers {{ version }} - - nccl>=2.9.9 - - ucx-py {{ ucx_py_version }} - - ucx-proc=*=gpu - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - libcusolver>=11.2.1 - - librmm {{ minor_version }} - -about: - home: http://rapids.ai/ - license: Apache-2.0 - # license_file: LICENSE - summary: libraft-distance library diff --git a/conda/recipes/libraft_headers/meta.yaml b/conda/recipes/libraft_headers/meta.yaml deleted file mode 100644 index fd95da66ee..0000000000 --- a/conda/recipes/libraft_headers/meta.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# Usage: - # conda build . -c conda-forge -c nvidia -c rapidsai -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} -{% set cuda_major=cuda_version.split('.')[0] %} -{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} -package: - name: libraft-headers - version: {{ version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - CC - - CXX - - CUDAHOSTCXX - - PARALLEL_LEVEL - - VERSION_SUFFIX - - PROJECT_FLASH - - CMAKE_GENERATOR - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_BUCKET=rapids-sccache - - SCCACHE_REGION=us-west-2 - - SCCACHE_IDLE_TIMEOUT=32768 - -requirements: - build: - - cmake>=3.20.1,!=3.23.0 - host: - - nccl>=2.9.9 - - cudatoolkit {{ cuda_version }}.* - - ucx-py {{ ucx_py_version }} - - ucx-proc=*=gpu - - gtest=1.10.0 - - librmm {{ minor_version}} - - libcusolver>=11.2.1 - run: - - nccl>=2.9.9 - - librmm {{ minor_version}} - - ucx-py {{ ucx_py_version }} - - ucx-proc=*=gpu - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - libcusolver>=11.2.1 - -about: - home: http://rapids.ai/ - license: Apache-2.0 - # license_file: LICENSE - summary: libraft-headers library diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml deleted file mode 100644 index fa3392ddc8..0000000000 --- a/conda/recipes/libraft_nn/meta.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# Usage: -# conda build . -c conda-forge -c nvidia -c rapidsai -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} -{% set cuda_major=cuda_version.split('.')[0] %} -{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} -package: - name: libraft-nn - version: {{ version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - CC - - CXX - - CUDAHOSTCXX - - PARALLEL_LEVEL - - VERSION_SUFFIX - - PROJECT_FLASH - - CMAKE_GENERATOR - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] - - SCCACHE_BUCKET=rapids-sccache - - SCCACHE_REGION=us-west-2 - - SCCACHE_IDLE_TIMEOUT=32768 - -requirements: - build: - - cmake>=3.20.1,!=3.23.0 - host: - - libraft-headers {{ version }} - - cudatoolkit {{ cuda_version }}.* - - lapack - - faiss-proc=*=cuda - - libfaiss 1.7.0 *_cuda - - gtest=1.10.0 - - librmm {{ minor_version }} - run: - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - libraft-headers {{ version }} - - faiss-proc=*=cuda - - libfaiss 1.7.0 *_cuda - - libcusolver>=11.2.1 - - librmm {{ minor_version }} - -about: - home: http://rapids.ai/ - license: Apache-2.0 - # license_file: LICENSE - summary: libraft-nn library diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f7f6d4ff49..3d2d2ff3d3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -369,7 +369,8 @@ if(TARGET raft_distance_lib) COMPONENT distance EXPORT raft-distance-lib-exports) install(DIRECTORY include/raft_distance - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT distance) endif() if(TARGET raft_nn_lib) diff --git a/python/pylibraft/pylibraft/test/__init__.py b/python/pylibraft/pylibraft/test/__init__.py deleted file mode 100644 index 273b4497cc..0000000000 --- a/python/pylibraft/pylibraft/test/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/python/raft/raft/test/__init__.py b/python/raft/raft/test/__init__.py deleted file mode 100644 index 99e0b7fac2..0000000000 --- a/python/raft/raft/test/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# \ No newline at end of file