From aeb2979bc6bf146fce82ece1b9e265188b3e68f0 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Fri, 28 Oct 2022 16:10:39 -0700 Subject: [PATCH 1/8] Run linters using pre-commit This change adds pre-commit hook to run on each commit, that will automatically run code linters and not allow you commit changes if the linters fail. This is similar to how cudf uses pre-commit, as described in https://github.com/rapidsai/cudf/issues/9309 . The pre-commit checks will also be run in CI as part of the check style script. This change also adds several new linters that weren't been run in CI before: * pydocstyle * mypy * black * isort * cmake-format * cmake-lint Of these new linters - mypy caught an issue where the test_comms.py would always be skipped (`python/raft-dask/raft_dask/test/test_comms.py:23: error: Module "raft_dask" has no attribute "Comms"` ), and Pydocstyle caught some minor formatting inconsistencies in the python docstrings. black/isort ensure consistent code formatting for the python raft code --- .pre-commit-config.yaml | 98 ++++ ci/checks/style.sh | 69 +-- cpp/CMakeLists.txt | 534 ++++++++++-------- cpp/bench/CMakeLists.txt | 238 ++++---- cpp/cmake/config.json | 43 ++ cpp/cmake/doxygen.cmake | 22 +- cpp/cmake/modules/ConfigureCUDA.cmake | 39 +- cpp/cmake/modules/raft_export.cmake | 87 +-- .../__clang_cuda_additional_intrinsics.h | 116 ++-- cpp/scripts/run-cmake-format.sh | 84 +++ cpp/test/CMakeLists.txt | 469 ++++++++------- pyproject.toml | 19 + python/.flake8.cython | 2 +- python/pylibraft/CMakeLists.txt | 30 +- python/pylibraft/pylibraft/_version.py | 16 +- .../pylibraft/cluster/CMakeLists.txt | 10 +- python/pylibraft/pylibraft/cluster/kmeans.pyx | 8 +- .../pylibraft/pylibraft/common/CMakeLists.txt | 10 +- python/pylibraft/pylibraft/common/cuda.pxd | 1 + python/pylibraft/pylibraft/common/cuda.pyx | 12 +- python/pylibraft/pylibraft/common/handle.pxd | 8 +- python/pylibraft/pylibraft/common/handle.pyx | 5 +- .../pylibraft/common/interruptible.pxd | 2 + .../pylibraft/common/interruptible.pyx | 4 +- .../pylibraft/distance/CMakeLists.txt | 13 +- .../pylibraft/pylibraft/distance/__init__.py | 3 +- .../pylibraft/distance/fused_l2_nn.pyx | 6 +- .../pylibraft/distance/pairwise_distance.pyx | 5 +- .../pylibraft/pylibraft/random/CMakeLists.txt | 14 +- .../random/rmat_rectangular_generator.pyx | 8 +- .../pylibraft/pylibraft/random/rng_state.pxd | 1 + .../pylibraft/pylibraft/test/test_distance.py | 25 +- .../pylibraft/test/test_fused_l2_argmin.py | 10 +- .../pylibraft/pylibraft/test/test_kmeans.py | 41 +- .../pylibraft/pylibraft/test/test_random.py | 4 +- .../pylibraft/test/test_z_interruptible.py | 25 +- python/pylibraft/pylibraft/testing/utils.py | 18 +- python/pylibraft/setup.py | 62 +- python/pylibraft/setuputils.py | 10 +- python/pylibraft/versioneer.py | 285 ++++++---- python/raft-dask/CMakeLists.txt | 19 +- .../raft-dask/cmake/thirdparty/get_nccl.cmake | 45 +- python/raft-dask/raft_dask/_version.py | 16 +- .../raft-dask/raft_dask/common/CMakeLists.txt | 5 +- python/raft-dask/raft_dask/common/__init__.py | 35 +- python/raft-dask/raft_dask/common/comms.py | 32 +- .../raft_dask/common/comms_utils.pyx | 8 +- python/raft-dask/raft_dask/common/nccl.pyx | 6 +- python/raft-dask/raft_dask/common/utils.py | 6 +- .../raft_dask/include_test/CMakeLists.txt | 5 +- python/raft-dask/raft_dask/test/conftest.py | 27 +- python/raft-dask/raft_dask/test/test_comms.py | 37 +- python/raft-dask/raft_dask/test/test_raft.py | 5 +- python/raft-dask/setup.py | 55 +- python/raft-dask/setuputils.py | 10 +- python/raft-dask/versioneer.py | 285 ++++++---- setup.cfg | 23 + 57 files changed, 1761 insertions(+), 1314 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 cpp/cmake/config.json create mode 100755 cpp/scripts/run-cmake-format.sh create mode 100644 pyproject.toml create mode 100644 setup.cfg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..429219b601 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,98 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +repos: + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + # Use the config file specific to each subproject so that each + # project can specify its own first/third-party packages. + args: ["--config-root=python/", "--resolve-all-configs"] + files: python/.* + types_or: [python, cython, pyi] + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + files: python/.* + # Explicitly specify the pyproject.toml at the repo root, not per-project. + args: ["--config", "pyproject.toml"] + exclude: .*_version.py,.*versioneer.py + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + args: ["--config=python/.flake8.cython"] + files: python/.*$ + types: [file] + types_or: [python, cython] + additional_dependencies: ["flake8-force"] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.971' + hooks: + - id: mypy + additional_dependencies: [types-cachetools] + args: ["--config-file=setup.cfg", + "python/pylibraft/pylibraft", + "python/raft-dask/raft_dask"] + pass_filenames: false + exclude: .*_version.py + - repo: https://github.com/PyCQA/pydocstyle + rev: 6.1.1 + hooks: + - id: pydocstyle + args: ["--config=setup.cfg"] + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v11.1.0 + hooks: + - id: clang-format + types_or: [c, c++, cuda] + args: ["-fallback-style=none", "-style=file", "-i"] + exclude: cpp/include/raft/thirdparty/.* + - repo: local + hooks: + - id: no-deprecationwarning + name: no-deprecationwarning + description: 'Enforce that DeprecationWarning is not introduced (use FutureWarning instead)' + entry: '(category=|\s)DeprecationWarning[,)]' + language: pygrep + types_or: [python, cython] + - id: cmake-format + name: cmake-format + entry: ./cpp/scripts/run-cmake-format.sh cmake-format + language: python + types: [cmake] + exclude: .*/thirdparty/.* + # Note that pre-commit autoupdate does not update the versions + # of dependencies, so we'll have to update this manually. + additional_dependencies: + - cmakelang==0.6.13 + verbose: true + require_serial: true + - id: cmake-lint + name: cmake-lint + entry: ./cpp/scripts/run-cmake-format.sh cmake-lint + language: python + types: [cmake] + # Note that pre-commit autoupdate does not update the versions + # of dependencies, so we'll have to update this manually. + additional_dependencies: + - cmakelang==0.6.13 + verbose: true + require_serial: true + exclude: .*/thirdparty/.* + - id: copyright-check + name: copyright-check + entry: python ./ci/checks/copyright.py --git-modified-only --update-current-year + language: python + pass_filenames: false + additional_dependencies: [gitpython] + - id: include-check + name: include-check + entry: python ./cpp/scripts/include_checker.py cpp/bench cpp/include cpp/test + pass_filenames: false + language: python + additional_dependencies: [gitpython] + +default_language_version: + python: python3 diff --git a/ci/checks/style.sh b/ci/checks/style.sh index fb5a64fdac..f8fcbe19f8 100644 --- a/ci/checks/style.sh +++ b/ci/checks/style.sh @@ -12,69 +12,12 @@ PATH=/opt/conda/bin:$PATH . /opt/conda/etc/profile.d/conda.sh conda activate rapids -# Run flake8 and get results/return code -FLAKE=`flake8 --exclude=cpp,thirdparty,__init__.py,versioneer.py && flake8 --config=python/.flake8.cython` -RETVAL=$? +FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.12/cmake-format-rapids-cmake.json +export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json +mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) +wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} -# Output results if failure otherwise show pass -if [ "$FLAKE" != "" ]; then - echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" - echo -e "$FLAKE" - echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: flake8 style check\n\n" -fi - -# Check for copyright headers in the files modified currently -COPYRIGHT=`python ci/checks/copyright.py --git-modified-only 2>&1` -CR_RETVAL=$? -if [ "$RETVAL" = "0" ]; then - RETVAL=$CR_RETVAL -fi - -# Output results if failure otherwise show pass -if [ "$CR_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: copyright check; begin output\n\n" - echo -e "$COPYRIGHT" - echo -e "\n\n>>>> FAILED: copyright check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: copyright check\n\n" -fi - -# Check for a consistent #include syntax -HASH_INCLUDE=`python cpp/scripts/include_checker.py \ - cpp/bench \ - cpp/include \ - cpp/test \ - 2>&1` -HASH_RETVAL=$? -if [ "$RETVAL" = "0" ]; then - RETVAL=$HASH_RETVAL -fi - -# Output results if failure otherwise show pass -if [ "$HASH_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: #include check; begin output\n\n" - echo -e "$HASH_INCLUDE" - echo -e "\n\n>>>> FAILED: #include check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: #include check\n\n" -fi - -# Check for a consistent code format -FORMAT=`python cpp/scripts/run-clang-format.py 2>&1` -FORMAT_RETVAL=$? -if [ "$RETVAL" = "0" ]; then - RETVAL=$FORMAT_RETVAL -fi - -# Output results if failure otherwise show pass -if [ "$FORMAT_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: clang format check; begin output\n\n" - echo -e "$FORMAT" - echo -e "\n\n>>>> FAILED: clang format check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: clang format check\n\n" -fi +# Run pre-commit checks +pre-commit run --hook-stage manual --all-files exit $RETVAL diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 552ee8db4c..6f0afa4eb0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,18 +1,16 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2020-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= set(RAPIDS_VERSION "22.12") set(RAFT_VERSION "22.12.00") @@ -26,7 +24,11 @@ include(rapids-find) rapids_cuda_init_architectures(RAFT) -project(RAFT VERSION ${RAFT_VERSION} LANGUAGES CXX CUDA) +project( + RAFT + VERSION ${RAFT_VERSION} + LANGUAGES CXX CUDA +) # Needed because GoogleBenchmark changes the state of FindThreads.cmake, causing subsequent runs to # have different values for the `Threads::Threads` target. Setting this flag ensures @@ -36,8 +38,8 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) # Write the version header rapids_cmake_write_version_file(include/raft/version_config.hpp) -############################################################################## -# - build type --------------------------------------------------------------- +# ################################################################################################## +# * build type --------------------------------------------------------------- # Set a default build type if none was specified rapids_cmake_build_type(Release) @@ -45,14 +47,16 @@ rapids_cmake_build_type(Release) # this is needed for clang-tidy runs set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -############################################################################## -# - User Options ------------------------------------------------------------ +# ################################################################################################## +# * User Options ------------------------------------------------------------ option(BUILD_SHARED_LIBS "Build raft shared libraries" ON) option(BUILD_TESTS "Build raft unit-tests" ON) option(BUILD_BENCH "Build raft C++ benchmark tests" OFF) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) -option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) +option(CUDA_ENABLE_LINEINFO + "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF +) option(CUDA_STATIC_RUNTIME "Statically link the CUDA toolkit runtime and libraries" OFF) option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) option(DISABLE_DEPRECATION_WARNINGS "Disable deprecaction warnings " ON) @@ -60,9 +64,16 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF) option(RAFT_NVTX "Enable nvtx markers" OFF) option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ${BUILD_TESTS}) -option(RAFT_COMPILE_NN_LIBRARY "Enable building raft nearest neighbors shared library instantiations" ${RAFT_COMPILE_LIBRARIES}) -option(RAFT_COMPILE_DIST_LIBRARY "Enable building raft distant shared library instantiations" ${RAFT_COMPILE_LIBRARIES}) -option(RAFT_ENABLE_NN_DEPENDENCIES "Search for raft::nn dependencies like faiss" ${RAFT_COMPILE_LIBRARIES}) +option( + RAFT_COMPILE_NN_LIBRARY "Enable building raft nearest neighbors shared library instantiations" + ${RAFT_COMPILE_LIBRARIES} +) +option(RAFT_COMPILE_DIST_LIBRARY "Enable building raft distant shared library instantiations" + ${RAFT_COMPILE_LIBRARIES} +) +option(RAFT_ENABLE_NN_DEPENDENCIES "Search for raft::nn dependencies like faiss" + ${RAFT_COMPILE_LIBRARIES} +) option(RAFT_ENABLE_thrust_DEPENDENCY "Enable Thrust dependency" ON) @@ -74,7 +85,13 @@ endif() option(RAFT_EXCLUDE_FAISS_FROM_ALL "Exclude FAISS targets from RAFT's 'all' target" ON) include(CMakeDependentOption) -cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_COMPILE_LIBRARIES OFF) +cmake_dependent_option( + RAFT_USE_FAISS_STATIC + "Build and statically link the FAISS library for nearest neighbors search on GPU" + ON + RAFT_COMPILE_LIBRARIES + OFF +) message(VERBOSE "RAFT: Building optional components: ${raft_FIND_COMPONENTS}") message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") @@ -85,35 +102,42 @@ message(VERBOSE "RAFT: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${RAFT_NVTX}") -message(VERBOSE "RAFT: Statically link the CUDA toolkit runtime and libraries: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE + "RAFT: Statically link the CUDA toolkit runtime and libraries: ${CUDA_STATIC_RUNTIME}" +) # Set RMM logging level -set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") -set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") +set(RMM_LOGGING_LEVEL + "INFO" + CACHE STRING "Choose the logging level." +) +set_property( + CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF" +) message(VERBOSE "RAFT: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") -############################################################################## -# - Conda environment detection ---------------------------------------------- +# ################################################################################################## +# * Conda environment detection ---------------------------------------------- if(DETECT_CONDA_ENV) - rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) - if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) - message(STATUS "RAFT: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) + if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) + message( + STATUS "RAFT: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}" + ) set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") endif() endif() -############################################################################## -# - compiler options --------------------------------------------------------- +# ################################################################################################## +# * compiler options --------------------------------------------------------- set(_ctk_static_suffix "") if(CUDA_STATIC_RUNTIME) - # If we're statically linking CTK cuBLAS, - # we also want to statically link BLAS + # If we're statically linking CTK cuBLAS, we also want to statically link BLAS set(BLA_STATIC ON) set(_ctk_static_suffix "_static") - # Control legacy FindCUDA.cmake behavior too - # Remove this after we push it into rapids-cmake: + # Control legacy FindCUDA.cmake behavior too Remove this after we push it into rapids-cmake: # https://github.com/rapidsai/rapids-cmake/pull/259 set(CUDA_USE_STATIC_CUDA_RUNTIME ON) endif() @@ -121,7 +145,7 @@ endif() # CUDA runtime rapids_cuda_init_runtime(USE_STATIC ${CUDA_STATIC_RUNTIME}) -if (NOT DISABLE_OPENMP) +if(NOT DISABLE_OPENMP) find_package(OpenMP) if(OPENMP_FOUND) message(VERBOSE "RAFT: OpenMP found in ${OpenMP_CXX_INCLUDE_DIRS}") @@ -132,13 +156,15 @@ endif() # * determine GPU architectures # * enable the CMake CUDA language # * set other CUDA compilation flags -rapids_find_package(CUDAToolkit REQUIRED - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports) +rapids_find_package( + CUDAToolkit REQUIRED + BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports +) include(cmake/modules/ConfigureCUDA.cmake) -############################################################################## -# - Requirements ------------------------------------------------------------- +# ################################################################################################## +# * Requirements ------------------------------------------------------------- if(RAFT_COMPILE_LIBRARIES) set(RAFT_COMPILE_DIST_LIBRARY ON) @@ -159,7 +185,9 @@ include(cmake/thirdparty/get_faiss.cmake) if(RAFT_ENABLE_cuco_DEPENDENCY) include(${rapids-cmake-dir}/cpm/cuco.cmake) - rapids_cpm_cuco(BUILD_EXPORT_SET raft-distance-lib-exports INSTALL_EXPORT_SET raft-distance-lib-exports) + rapids_cpm_cuco( + BUILD_EXPORT_SET raft-distance-lib-exports INSTALL_EXPORT_SET raft-distance-lib-exports + ) endif() if(BUILD_TESTS) @@ -171,69 +199,77 @@ if(BUILD_BENCH) rapids_cpm_gbench() endif() -############################################################################## -# - raft --------------------------------------------------------------------- +# ################################################################################################## +# * raft --------------------------------------------------------------------- add_library(raft INTERFACE) add_library(raft::raft ALIAS raft) -target_include_directories(raft INTERFACE - "$" - "$") - -# Keep RAFT as lightweight as possible. -# Only CUDA libs and rmm should -# be used in global target. -target_link_libraries(raft INTERFACE - rmm::rmm - CUDA::cublas${_ctk_static_suffix} - CUDA::curand${_ctk_static_suffix} - CUDA::cusolver${_ctk_static_suffix} - CUDA::cusparse${_ctk_static_suffix} - $<$:raft::Thrust> +target_include_directories( + raft INTERFACE "$" "$" +) + +# Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. +target_link_libraries( + raft + INTERFACE rmm::rmm + CUDA::cublas${_ctk_static_suffix} + CUDA::curand${_ctk_static_suffix} + CUDA::cusolver${_ctk_static_suffix} + CUDA::cusparse${_ctk_static_suffix} + $<$:raft::Thrust> ) target_compile_features(raft INTERFACE cxx_std_17 $) if(RAFT_COMPILE_DIST_LIBRARY OR RAFT_COMPILE_NN_LIBRARY) - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld" - [=[ + file( + WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld" + [=[ SECTIONS { .nvFatBinSegment : { *(.nvFatBinSegment) } .nv_fatbin : { *(.nv_fatbin) } } -]=]) +]=] + ) endif() -############################################################################## -# - NVTX support in raft ----------------------------------------------------- +# ################################################################################################## +# * NVTX support in raft ----------------------------------------------------- -if (RAFT_NVTX) +if(RAFT_NVTX) # This enables NVTX within the project with no option to disable it downstream. target_link_libraries(raft INTERFACE CUDA::nvToolsExt) target_compile_definitions(raft INTERFACE NVTX_ENABLED) else() - # Allow enable NVTX downstream if not set here. - # This creates a new option at build/install time, which is set by default to OFF, - # but can be enabled in the dependent project. - get_property(nvtx_option_help_string CACHE RAFT_NVTX PROPERTY HELPSTRING) - string(CONCAT nvtx_export_string - "option(RAFT_NVTX \"" ${nvtx_option_help_string} "\" OFF)" - [=[ + # Allow enable NVTX downstream if not set here. This creates a new option at build/install time, + # which is set by default to OFF, but can be enabled in the dependent project. + get_property( + nvtx_option_help_string + CACHE RAFT_NVTX + PROPERTY HELPSTRING + ) + string( + CONCAT + nvtx_export_string + "option(RAFT_NVTX \"" + ${nvtx_option_help_string} + "\" OFF)" + [=[ target_link_libraries(raft::raft INTERFACE $<$:CUDA::nvToolsExt>) target_compile_definitions(raft::raft INTERFACE $<$:NVTX_ENABLED>) - ]=]) + ]=] + ) endif() -############################################################################## -# - raft_distance ------------------------------------------------------------ -# TODO: -# Currently, this package also contains the 'random' namespace (for rmat logic) -# We couldn't get this to work properly due to strange CI failures as noticed -# in the PR#778. In the long term, we should rename this package to `raft_compiled` -# in order to have a single pre-compiled raft package for those who need it. +# ################################################################################################## +# * raft_distance ------------------------------------------------------------ TODO: Currently, this +# package also contains the 'random' namespace (for rmat logic) We couldn't get this to work +# properly due to strange CI failures as noticed in the PR#778. In the long term, we should rename +# this package to `raft_compiled` in order to have a single pre-compiled raft package for those +# who need it. add_library(raft_distance INTERFACE) if(TARGET raft_distance AND (NOT TARGET raft::distance)) @@ -243,7 +279,8 @@ endif() set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance) if(RAFT_COMPILE_DIST_LIBRARY) - add_library(raft_distance_lib + add_library( + raft_distance_lib src/distance/pairwise_distance.cu src/distance/fused_l2_min_arg.cu src/distance/update_centroids_float.cu @@ -262,9 +299,9 @@ if(RAFT_COMPILE_DIST_LIBRARY) src/distance/specializations/detail/kernels/gram_matrix_base_float.cu src/distance/specializations/detail/kernels/polynomial_kernel_double_int.cu src/distance/specializations/detail/kernels/polynomial_kernel_float_int.cu -# These are somehow missing a kernel definition which is causing a compile error. -# src/distance/specializations/detail/kernels/rbf_kernel_double.cu -# src/distance/specializations/detail/kernels/rbf_kernel_float.cu + # These are somehow missing a kernel definition which is causing a compile error. + # src/distance/specializations/detail/kernels/rbf_kernel_double.cu + # src/distance/specializations/detail/kernels/rbf_kernel_float.cu src/distance/specializations/detail/kernels/tanh_kernel_double.cu src/distance/specializations/detail/kernels/tanh_kernel_float.cu src/distance/specializations/detail/kl_divergence_float_float_float_int.cu @@ -302,26 +339,23 @@ if(RAFT_COMPILE_DIST_LIBRARY) ) set_target_properties( raft_distance_lib - PROPERTIES OUTPUT_NAME raft_distance - BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON) - - target_link_libraries(raft_distance_lib - PUBLIC raft::raft - cuco::cuco - ) - target_compile_options(raft_distance_lib - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - target_compile_definitions(raft_distance_lib - INTERFACE "RAFT_DISTANCE_COMPILED") + PROPERTIES OUTPUT_NAME raft_distance + BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) + + target_link_libraries(raft_distance_lib PUBLIC raft::raft cuco::cuco) + target_compile_options( + raft_distance_lib PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) + target_compile_definitions(raft_distance_lib INTERFACE "RAFT_DISTANCE_COMPILED") # ensure CUDA symbols aren't relocated to the middle of the debug build binaries target_link_options(raft_distance_lib PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") @@ -332,13 +366,12 @@ if(TARGET raft_distance_lib AND (NOT TARGET raft::raft_distance_lib)) add_library(raft::raft_distance_lib ALIAS raft_distance_lib) endif() -target_link_libraries(raft_distance INTERFACE - raft::raft - $ +target_link_libraries( + raft_distance INTERFACE raft::raft $ ) -############################################################################## -# - raft_nn ------------------------------------------------------------------ +# ################################################################################################## +# * raft_nn ------------------------------------------------------------------ add_library(raft_nn INTERFACE) if(TARGET raft_nn AND (NOT TARGET raft::nn)) @@ -348,121 +381,135 @@ endif() set_target_properties(raft_nn PROPERTIES EXPORT_NAME nn) if(RAFT_COMPILE_NN_LIBRARY) - add_library(raft_nn_lib - src/nn/specializations/ball_cover.cu - src/nn/specializations/detail/ball_cover_lowdim_pass_one_2d.cu - src/nn/specializations/detail/ball_cover_lowdim_pass_two_2d.cu - src/nn/specializations/detail/ball_cover_lowdim_pass_one_3d.cu - src/nn/specializations/detail/ball_cover_lowdim_pass_two_3d.cu - src/nn/specializations/detail/ivfpq_compute_similarity_float_fast.cu - src/nn/specializations/detail/ivfpq_compute_similarity_float_no_basediff.cu - src/nn/specializations/detail/ivfpq_compute_similarity_float_no_smem_lut.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_fast.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_no_basediff.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_no_smem_lut.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_fast.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_no_basediff.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_no_smem_lut.cu - src/nn/specializations/detail/ivfpq_compute_similarity_half_fast.cu - src/nn/specializations/detail/ivfpq_compute_similarity_half_no_basediff.cu - src/nn/specializations/detail/ivfpq_compute_similarity_half_no_smem_lut.cu - src/nn/specializations/detail/ivfpq_search_float_int64_t.cu - src/nn/specializations/detail/ivfpq_search_float_uint32_t.cu - src/nn/specializations/detail/ivfpq_search_float_uint64_t.cu - src/nn/specializations/fused_l2_knn_long_float_true.cu - src/nn/specializations/fused_l2_knn_long_float_false.cu - src/nn/specializations/fused_l2_knn_int_float_true.cu - src/nn/specializations/fused_l2_knn_int_float_false.cu - src/nn/specializations/knn.cu - ) + add_library( + raft_nn_lib + src/nn/specializations/ball_cover.cu + src/nn/specializations/detail/ball_cover_lowdim_pass_one_2d.cu + src/nn/specializations/detail/ball_cover_lowdim_pass_two_2d.cu + src/nn/specializations/detail/ball_cover_lowdim_pass_one_3d.cu + src/nn/specializations/detail/ball_cover_lowdim_pass_two_3d.cu + src/nn/specializations/detail/ivfpq_compute_similarity_float_fast.cu + src/nn/specializations/detail/ivfpq_compute_similarity_float_no_basediff.cu + src/nn/specializations/detail/ivfpq_compute_similarity_float_no_smem_lut.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_fast.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_no_basediff.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_no_smem_lut.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_fast.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_no_basediff.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_no_smem_lut.cu + src/nn/specializations/detail/ivfpq_compute_similarity_half_fast.cu + src/nn/specializations/detail/ivfpq_compute_similarity_half_no_basediff.cu + src/nn/specializations/detail/ivfpq_compute_similarity_half_no_smem_lut.cu + src/nn/specializations/detail/ivfpq_search_float_int64_t.cu + src/nn/specializations/detail/ivfpq_search_float_uint32_t.cu + src/nn/specializations/detail/ivfpq_search_float_uint64_t.cu + src/nn/specializations/fused_l2_knn_long_float_true.cu + src/nn/specializations/fused_l2_knn_long_float_false.cu + src/nn/specializations/fused_l2_knn_int_float_true.cu + src/nn/specializations/fused_l2_knn_int_float_false.cu + src/nn/specializations/knn.cu + ) set_target_properties( raft_nn_lib - PROPERTIES OUTPUT_NAME raft_nn - BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON) - - target_link_libraries(raft_nn_lib - PUBLIC faiss::faiss - raft::raft) - target_compile_options(raft_nn_lib - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) + PROPERTIES OUTPUT_NAME raft_nn + BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) + + target_link_libraries(raft_nn_lib PUBLIC faiss::faiss raft::raft) + target_compile_options( + raft_nn_lib PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) # ensure CUDA symbols aren't relocated to the middle of the debug build binaries target_link_options(raft_nn_lib PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") - target_compile_definitions(raft_nn_lib - INTERFACE "RAFT_NN_COMPILED") + target_compile_definitions(raft_nn_lib INTERFACE "RAFT_NN_COMPILED") endif() if(TARGET raft_nn_lib AND (NOT TARGET raft::raft_nn_lib)) add_library(raft::raft_nn_lib ALIAS raft_nn_lib) endif() -target_link_libraries(raft_nn INTERFACE - raft::raft - $) +target_link_libraries(raft_nn INTERFACE raft::raft $) -############################################################################## -# - install targets----------------------------------------------------------- -rapids_cmake_install_lib_dir( lib_dir ) +# ################################################################################################## +# * install targets----------------------------------------------------------- +rapids_cmake_install_lib_dir(lib_dir) include(GNUInstallDirs) include(CPack) -install(TARGETS raft - DESTINATION ${lib_dir} - COMPONENT raft - EXPORT raft-exports) +install( + TARGETS raft + DESTINATION ${lib_dir} + COMPONENT raft + EXPORT raft-exports +) -install(TARGETS raft_distance - DESTINATION ${lib_dir} - COMPONENT raft - EXPORT raft-distance-exports) +install( + TARGETS raft_distance + DESTINATION ${lib_dir} + COMPONENT raft + EXPORT raft-distance-exports +) -install(TARGETS raft_nn - DESTINATION ${lib_dir} - COMPONENT raft - EXPORT raft-nn-exports) +install( + TARGETS raft_nn + DESTINATION ${lib_dir} + COMPONENT raft + EXPORT raft-nn-exports +) if(TARGET raft_distance_lib) - install(TARGETS raft_distance_lib - DESTINATION ${lib_dir} - COMPONENT distance - EXPORT raft-distance-lib-exports) - install(DIRECTORY include/raft_distance - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - COMPONENT distance) + install( + TARGETS raft_distance_lib + DESTINATION ${lib_dir} + COMPONENT distance + EXPORT raft-distance-lib-exports + ) + install( + DIRECTORY include/raft_distance + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT distance + ) endif() if(TARGET raft_nn_lib) - install(TARGETS raft_nn_lib - DESTINATION ${lib_dir} - COMPONENT nn - EXPORT raft-nn-lib-exports) + install( + TARGETS raft_nn_lib + DESTINATION ${lib_dir} + COMPONENT nn + EXPORT raft-nn-lib-exports + ) endif() - -install(DIRECTORY include/raft - COMPONENT raft - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install( + DIRECTORY include/raft + COMPONENT raft + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) # Temporary install of raft.hpp while the file is removed -install(FILES include/raft.hpp - COMPONENT raft - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft) +install( + FILES include/raft.hpp + COMPONENT raft + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft +) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/include/raft/version_config.hpp - COMPONENT raft - DESTINATION include/raft) +install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/include/raft/version_config.hpp + COMPONENT raft + DESTINATION include/raft +) -############################################################################## -# - export/install optional components -------------------------------------- +# ################################################################################################## +# * export/install optional components -------------------------------------- include("${rapids-cmake-dir}/export/write_dependencies.cmake") @@ -479,30 +526,31 @@ endif() foreach(comp install_comp IN ZIP_LISTS raft_components raft_install_comp) install( - EXPORT raft-${comp}-exports - FILE raft-${comp}-targets.cmake - NAMESPACE raft:: - DESTINATION "${lib_dir}/cmake/raft" - COMPONENT ${install_comp} + EXPORT raft-${comp}-exports + FILE raft-${comp}-targets.cmake + NAMESPACE raft:: + DESTINATION "${lib_dir}/cmake/raft" + COMPONENT ${install_comp} ) export( - EXPORT raft-${comp}-exports - FILE ${RAFT_BINARY_DIR}/raft-${comp}-targets.cmake - NAMESPACE raft:: + EXPORT raft-${comp}-exports + FILE ${RAFT_BINARY_DIR}/raft-${comp}-targets.cmake + NAMESPACE raft:: ) rapids_export_write_dependencies( - BUILD raft-${comp}-exports "${PROJECT_BINARY_DIR}/raft-${comp}-dependencies.cmake" + BUILD raft-${comp}-exports "${PROJECT_BINARY_DIR}/raft-${comp}-dependencies.cmake" ) rapids_export_write_dependencies( - INSTALL raft-${comp}-exports "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/${install_comp}/raft-${comp}-dependencies.cmake" + INSTALL raft-${comp}-exports + "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/${install_comp}/raft-${comp}-dependencies.cmake" ) endforeach() -############################################################################## -# - install export ----------------------------------------------------------- +# ################################################################################################## +# * install export ----------------------------------------------------------- set(doc_string - [=[ + [=[ Provide targets for the RAFT: Reusable Accelerated Functions and Tools RAFT contains fundamental widely-used algorithms and primitives @@ -517,21 +565,27 @@ Imported Targets: - raft::nn brought in by the `nn` optional component - raft::distance brought in by the `distance` optional component -]=]) +]=] +) set(code_string ${nvtx_export_string}) if(RAFT_ENABLE_thrust_DEPENDENCY) - string(APPEND code_string - [=[ + string( + APPEND + code_string + [=[ if(NOT TARGET raft::Thrust) thrust_create_target(raft::Thrust FROM_OPTIONS) endif() - ]=]) + ]=] + ) endif() -string(APPEND code_string -[=[ +string( + APPEND + code_string + [=[ if(distance IN_LIST raft_FIND_COMPONENTS) enable_language(CUDA) endif() @@ -545,45 +599,41 @@ if(nn IN_LIST raft_FIND_COMPONENTS) add_library(faiss ALIAS faiss::faiss) endif() endif() -]=]) +]=] +) # Use `rapids_export` for 22.04 as it will have COMPONENT support include(cmake/modules/raft_export.cmake) -raft_export(INSTALL raft - COMPONENTS nn distance - EXPORT_SET raft-exports - GLOBAL_TARGETS raft nn distance - NAMESPACE raft:: - DOCUMENTATION doc_string - FINAL_CODE_BLOCK code_string) - -############################################################################## -# - build export ------------------------------------------------------------- -raft_export(BUILD raft - EXPORT_SET raft-exports - COMPONENTS nn distance - GLOBAL_TARGETS raft raft_distance raft_nn - DOCUMENTATION doc_string - NAMESPACE raft:: - FINAL_CODE_BLOCK code_string) - -############################################################################## -# - build test executable ---------------------------------------------------- +raft_export( + INSTALL raft COMPONENTS nn distance EXPORT_SET raft-exports GLOBAL_TARGETS raft nn distance + NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string +) + +# ################################################################################################## +# * build export ------------------------------------------------------------- +raft_export( + BUILD raft EXPORT_SET raft-exports COMPONENTS nn distance GLOBAL_TARGETS raft raft_distance + raft_nn DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string +) + +# ################################################################################################## +# * build test executable ---------------------------------------------------- if(BUILD_TESTS) include(test/CMakeLists.txt) endif() -############################################################################## -# - build benchmark executable ----------------------------------------------- +# ################################################################################################## +# * build benchmark executable ----------------------------------------------- if(BUILD_BENCH) include(bench/CMakeLists.txt) endif() -############################################################################## -# - doxygen targets ---------------------------------------------------------- +# ################################################################################################## +# * doxygen targets ---------------------------------------------------------- include(cmake/doxygen.cmake) -add_doxygen_target(IN_DOXYFILE doxygen/Doxyfile.in - OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile - CWD ${CMAKE_CURRENT_BINARY_DIR}) +add_doxygen_target( + IN_DOXYFILE doxygen/Doxyfile.in OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile CWD + ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/cpp/bench/CMakeLists.txt b/cpp/bench/CMakeLists.txt index ef91fe4e6c..c236e430e3 100644 --- a/cpp/bench/CMakeLists.txt +++ b/cpp/bench/CMakeLists.txt @@ -1,135 +1,129 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= -################################################################################################### -# - compiler function ----------------------------------------------------------------------------- +# ################################################################################################## +# * compiler function ----------------------------------------------------------------------------- function(ConfigureBench) -set(options OPTIONAL DIST NN) -set(oneValueArgs NAME ) -set(multiValueArgs PATH TARGETS CONFIGURATIONS) - -cmake_parse_arguments(ConfigureBench "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - -set(BENCH_NAME ${ConfigureBench_NAME}) - -add_executable(${BENCH_NAME} ${ConfigureBench_PATH}) - -target_link_libraries(${BENCH_NAME} - PRIVATE - raft::raft - $<$:raft::distance> - $<$:raft::nn> - benchmark::benchmark - Threads::Threads - $ - $ - ) - -set_target_properties(${BENCH_NAME} - PROPERTIES - # set target compile options - INSTALL_RPATH "\$ORIGIN/../../../lib" - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - ) - -target_compile_options(${BENCH_NAME} - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - -target_include_directories(${BENCH_NAME} - PUBLIC "$" - ) - -install( - TARGETS ${BENCH_NAME} - COMPONENT testing - DESTINATION bin/gbench/libraft - EXCLUDE_FROM_ALL) + set(options OPTIONAL DIST NN) + set(oneValueArgs NAME) + set(multiValueArgs PATH TARGETS CONFIGURATIONS) + + cmake_parse_arguments(ConfigureBench "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(BENCH_NAME ${ConfigureBench_NAME}) + + add_executable(${BENCH_NAME} ${ConfigureBench_PATH}) + + target_link_libraries( + ${BENCH_NAME} + PRIVATE raft::raft + $<$:raft::distance> + $<$:raft::nn> + benchmark::benchmark + Threads::Threads + $ + $ + ) + + set_target_properties( + ${BENCH_NAME} + PROPERTIES # set target compile options + INSTALL_RPATH "\$ORIGIN/../../../lib" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) + + target_compile_options( + ${BENCH_NAME} PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) + + target_include_directories(${BENCH_NAME} PUBLIC "$") + + install( + TARGETS ${BENCH_NAME} + COMPONENT testing + DESTINATION bin/gbench/libraft + EXCLUDE_FROM_ALL + ) endfunction() if(BUILD_BENCH) - ConfigureBench(NAME CLUSTER_BENCH - PATH - bench/cluster/kmeans_balanced.cu - bench/cluster/kmeans.cu - bench/main.cpp - OPTIONAL DIST NN - ) - - ConfigureBench(NAME DISTANCE_BENCH - PATH - bench/distance/distance_cosine.cu - bench/distance/distance_exp_l2.cu - bench/distance/distance_l1.cu - bench/distance/distance_unexp_l2.cu - bench/distance/fused_l2_nn.cu - bench/distance/kernels.cu - bench/main.cpp - OPTIONAL DIST - ) - - ConfigureBench(NAME LINALG_BENCH - PATH - bench/linalg/add.cu - bench/linalg/map_then_reduce.cu - bench/linalg/matrix_vector_op.cu - bench/linalg/reduce_rows_by_key.cu - bench/linalg/reduce.cu - bench/main.cpp - ) - - ConfigureBench(NAME RANDOM_BENCH - PATH - bench/random/make_blobs.cu - bench/random/permute.cu - bench/random/rng.cu - bench/main.cpp - ) - - ConfigureBench(NAME SPARSE_BENCH - PATH - bench/sparse/convert_csr.cu - bench/main.cpp - ) - - ConfigureBench(NAME NEIGHBORS_BENCH - PATH - bench/neighbors/knn/brute_force_float_int64_t.cu - bench/neighbors/knn/brute_force_float_uint32_t.cu - bench/neighbors/knn/ivf_flat_float_int64_t.cu - bench/neighbors/knn/ivf_flat_float_uint32_t.cu - bench/neighbors/knn/ivf_flat_int8_t_int64_t.cu - bench/neighbors/knn/ivf_flat_uint8_t_uint32_t.cu - bench/neighbors/knn/ivf_pq_float_int64_t.cu - bench/neighbors/knn/ivf_pq_float_uint32_t.cu - bench/neighbors/knn/ivf_pq_int8_t_int64_t.cu - bench/neighbors/knn/ivf_pq_uint8_t_uint32_t.cu - bench/neighbors/selection.cu - bench/main.cpp - OPTIONAL DIST NN - ) + ConfigureBench( + NAME CLUSTER_BENCH PATH bench/cluster/kmeans_balanced.cu bench/cluster/kmeans.cu bench/main.cpp + OPTIONAL DIST NN + ) + + ConfigureBench( + NAME + DISTANCE_BENCH + PATH + bench/distance/distance_cosine.cu + bench/distance/distance_exp_l2.cu + bench/distance/distance_l1.cu + bench/distance/distance_unexp_l2.cu + bench/distance/fused_l2_nn.cu + bench/distance/kernels.cu + bench/main.cpp + OPTIONAL + DIST + ) + + ConfigureBench( + NAME + LINALG_BENCH + PATH + bench/linalg/add.cu + bench/linalg/map_then_reduce.cu + bench/linalg/matrix_vector_op.cu + bench/linalg/reduce_rows_by_key.cu + bench/linalg/reduce.cu + bench/main.cpp + ) + + ConfigureBench( + NAME RANDOM_BENCH PATH bench/random/make_blobs.cu bench/random/permute.cu bench/random/rng.cu + bench/main.cpp + ) + + ConfigureBench(NAME SPARSE_BENCH PATH bench/sparse/convert_csr.cu bench/main.cpp) + + ConfigureBench( + NAME + NEIGHBORS_BENCH + PATH + bench/neighbors/knn/brute_force_float_int64_t.cu + bench/neighbors/knn/brute_force_float_uint32_t.cu + bench/neighbors/knn/ivf_flat_float_int64_t.cu + bench/neighbors/knn/ivf_flat_float_uint32_t.cu + bench/neighbors/knn/ivf_flat_int8_t_int64_t.cu + bench/neighbors/knn/ivf_flat_uint8_t_uint32_t.cu + bench/neighbors/knn/ivf_pq_float_int64_t.cu + bench/neighbors/knn/ivf_pq_float_uint32_t.cu + bench/neighbors/knn/ivf_pq_int8_t_int64_t.cu + bench/neighbors/knn/ivf_pq_uint8_t_uint32_t.cu + bench/neighbors/selection.cu + bench/main.cpp + OPTIONAL + DIST + NN + ) endif() - diff --git a/cpp/cmake/config.json b/cpp/cmake/config.json new file mode 100644 index 0000000000..f7cc50e513 --- /dev/null +++ b/cpp/cmake/config.json @@ -0,0 +1,43 @@ +{ + "parse": { + "additional_commands": { + "CPMFindPackage": { + "kwargs": { + "NAME": 1, + "GITHUB_REPOSITORY": "?", + "GIT_TAG": "?", + "VERSION": "?", + "GIT_SHALLOW": "?", + "OPTIONS": "*", + "FIND_PACKAGE_ARGUMENTS": "*" + } + }, + "ConfigureTest": { + "flags": ["TEST_NAME", "TEST_SRC"] + }, + "ConfigureBench": { + "flags": ["BENCH_NAME", "BENCH_SRC"] + } + } + }, + "format": { + "line_width": 100, + "tab_size": 2, + "command_case": "unchanged", + "max_lines_hwrap": 1, + "max_pargs_hwrap": 999, + "dangle_parens": true + }, + "lint": { + "disabled_codes": ["C0301", "C0111", "C0113"], + "function_pattern": "[0-9A-z_]+", + "macro_pattern": "[0-9A-z_]+", + "global_var_pattern": "[A-z][0-9A-z_]+", + "internal_var_pattern": "_[A-z][0-9A-z_]+", + "local_var_pattern": "[A-z][A-z0-9_]+", + "private_var_pattern": "_[0-9A-z_]+", + "public_var_pattern": "[A-z][0-9A-z_]+", + "argument_var_pattern": "[A-z][A-z0-9_]+", + "keyword_pattern": "[A-z][0-9A-z_]+" + } +} diff --git a/cpp/cmake/doxygen.cmake b/cpp/cmake/doxygen.cmake index 7d06ec194c..a6fddbacd6 100644 --- a/cpp/cmake/doxygen.cmake +++ b/cpp/cmake/doxygen.cmake @@ -1,16 +1,14 @@ # Copyright (c) 2020-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. # find_package(Doxygen 1.8.11) @@ -24,11 +22,13 @@ function(add_doxygen_target) configure_file(${dox_IN_DOXYFILE} ${dox_OUT_DOXYFILE} @ONLY) message("Command: ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE}") - add_custom_target(docs_raft + add_custom_target( + docs_raft ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE} WORKING_DIRECTORY ${dox_CWD} VERBATIM - COMMENT "Generate doxygen docs") + COMMENT "Generate doxygen docs" + ) else() message("add_doxygen_target: doxygen exe not found") endif() diff --git a/cpp/cmake/modules/ConfigureCUDA.cmake b/cpp/cmake/modules/ConfigureCUDA.cmake index 440c8c4f3a..9b029a5242 100644 --- a/cpp/cmake/modules/ConfigureCUDA.cmake +++ b/cpp/cmake/modules/ConfigureCUDA.cmake @@ -1,26 +1,24 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2018-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= if(DISABLE_DEPRECATION_WARNINGS) - list(APPEND RAFT_CXX_FLAGS -Wno-deprecated-declarations) - list(APPEND RAFT_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) + list(APPEND RAFT_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND RAFT_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) endif() if(CMAKE_COMPILER_IS_GNUCXX) - list(APPEND RAFT_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) + list(APPEND RAFT_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) endif() list(APPEND RAFT_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) @@ -31,21 +29,22 @@ list(APPEND RAFT_CUDA_FLAGS -Xfatbin=-compress-all) # set warnings as errors if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) - list(APPEND RAFT_CUDA_FLAGS -Werror=all-warnings) + list(APPEND RAFT_CUDA_FLAGS -Werror=all-warnings) endif() list(APPEND RAFT_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) -# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking +# Option to enable line info in CUDA device compilation to allow introspection when profiling / +# memchecking if(CUDA_ENABLE_LINEINFO) - list(APPEND RAFT_CUDA_FLAGS -lineinfo) + list(APPEND RAFT_CUDA_FLAGS -lineinfo) endif() if(OpenMP_FOUND) - list(APPEND RAFT_CUDA_FLAGS -Xcompiler=${OpenMP_CXX_FLAGS}) + list(APPEND RAFT_CUDA_FLAGS -Xcompiler=${OpenMP_CXX_FLAGS}) endif() # Debug options if(CMAKE_BUILD_TYPE MATCHES Debug) - message(VERBOSE "RAFT: Building with debugging flags") - list(APPEND RAFT_CUDA_FLAGS -G -Xcompiler=-rdynamic) + message(VERBOSE "RAFT: Building with debugging flags") + list(APPEND RAFT_CUDA_FLAGS -G -Xcompiler=-rdynamic) endif() diff --git a/cpp/cmake/modules/raft_export.cmake b/cpp/cmake/modules/raft_export.cmake index 748fa8ad26..41fe6d73da 100644 --- a/cpp/cmake/modules/raft_export.cmake +++ b/cpp/cmake/modules/raft_export.cmake @@ -1,18 +1,16 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2021-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= include_guard(GLOBAL) #[=======================================================================[.rst: @@ -174,19 +172,26 @@ function(raft_export type project_name) set(scratch_dir "${PROJECT_BINARY_DIR}/rapids-cmake/${project_name}/export/raft/") - configure_package_config_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" - "${scratch_dir}/${project_name}-config.cmake" - INSTALL_DESTINATION "${install_location}") + configure_package_config_file( + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" + "${scratch_dir}/${project_name}-config.cmake" INSTALL_DESTINATION "${install_location}" + ) if(rapids_version_set) write_basic_package_version_file( - "${scratch_dir}/${project_name}-config-version.cmake" VERSION ${rapids_project_version} - COMPATIBILITY ${rapids_project_version_compat}) + "${scratch_dir}/${project_name}-config-version.cmake" + VERSION ${rapids_project_version} + COMPATIBILITY ${rapids_project_version_compat} + ) endif() - install(EXPORT ${RAPIDS_EXPORT_SET} FILE ${project_name}-targets.cmake - NAMESPACE ${RAPIDS_PROJECT_VERSION} DESTINATION "${install_location}" - COMPONENT raft) + install( + EXPORT ${RAPIDS_EXPORT_SET} + FILE ${project_name}-targets.cmake + NAMESPACE ${RAPIDS_PROJECT_VERSION} + DESTINATION "${install_location}" + COMPONENT raft + ) if(TARGET rapids_export_install_${RAPIDS_EXPORT_SET}) include("${rapids-cmake-dir}/export/write_dependencies.cmake") @@ -203,41 +208,55 @@ function(raft_export type project_name) endif() # Install everything we have generated - install(DIRECTORY "${scratch_dir}/" DESTINATION "${install_location}" - COMPONENT raft) + install( + DIRECTORY "${scratch_dir}/" + DESTINATION "${install_location}" + COMPONENT raft + ) foreach(comp nn distance) set(scratch_dir "${PROJECT_BINARY_DIR}/rapids-cmake/${project_name}/export/${comp}/") file(MAKE_DIRECTORY "${scratch_dir}") - install(DIRECTORY "${scratch_dir}" DESTINATION "${install_location}" - COMPONENT ${comp}) + install( + DIRECTORY "${scratch_dir}" + DESTINATION "${install_location}" + COMPONENT ${comp} + ) endforeach() else() set(install_location "${PROJECT_BINARY_DIR}") - configure_package_config_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" - "${install_location}/${project_name}-config.cmake" - INSTALL_DESTINATION "${install_location}") + configure_package_config_file( + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" + "${install_location}/${project_name}-config.cmake" INSTALL_DESTINATION "${install_location}" + ) if(rapids_version_set) write_basic_package_version_file( - "${install_location}/${project_name}-config-version.cmake" VERSION ${rapids_project_version} - COMPATIBILITY ${rapids_project_version_compat}) + "${install_location}/${project_name}-config-version.cmake" + VERSION ${rapids_project_version} + COMPATIBILITY ${rapids_project_version_compat} + ) endif() - export(EXPORT ${RAPIDS_EXPORT_SET} NAMESPACE ${RAPIDS_PROJECT_VERSION} - FILE "${install_location}/${project_name}-targets.cmake") + export( + EXPORT ${RAPIDS_EXPORT_SET} + NAMESPACE ${RAPIDS_PROJECT_VERSION} + FILE "${install_location}/${project_name}-targets.cmake" + ) if(TARGET rapids_export_build_${RAPIDS_EXPORT_SET}) include("${rapids-cmake-dir}/export/write_dependencies.cmake") - rapids_export_write_dependencies(BUILD ${RAPIDS_EXPORT_SET} - "${install_location}/${project_name}-dependencies.cmake") + rapids_export_write_dependencies( + BUILD ${RAPIDS_EXPORT_SET} "${install_location}/${project_name}-dependencies.cmake" + ) endif() if(DEFINED RAPIDS_LANGUAGES) include("${rapids-cmake-dir}/export/write_language.cmake") foreach(lang IN LISTS RAPIDS_LANGUAGES) - rapids_export_write_language(BUILD ${lang} - "${install_location}/${project_name}-${lang}-language.cmake") + rapids_export_write_language( + BUILD ${lang} "${install_location}/${project_name}-${lang}-language.cmake" + ) endforeach() endif() diff --git a/cpp/scripts/__clang_cuda_additional_intrinsics.h b/cpp/scripts/__clang_cuda_additional_intrinsics.h index 8964d210bf..b9c032dc45 100644 --- a/cpp/scripts/__clang_cuda_additional_intrinsics.h +++ b/cpp/scripts/__clang_cuda_additional_intrinsics.h @@ -1,3 +1,4 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. #ifndef __CLANG_CUDA_ADDITIONAL_INTRINSICS_H__ #define __CLANG_CUDA_ADDITIONAL_INTRINSICS_H__ #ifndef __CUDA__ @@ -7,46 +8,49 @@ // for some of these macros, see cuda_fp16.hpp #if defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320)) #if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__) -#define __LDG_PTR "l" -#define __LBITS "64" +#define __LDG_PTR "l" +#define __LBITS "64" #else -#define __LDG_PTR "r" -#define __LBITS "32" -#endif // (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__) +#define __LDG_PTR "r" +#define __LBITS "32" +#endif // (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__) #define __NOARG -#define __MAKE_LD(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ - __device__ __forceinline__ c_typ __ld ## cop (const c_typ* addr) { \ - int_typ out; \ - asm("ld." #cop "." ptx_typ " %0, [%1];" \ - : "=" inl_typ(out) : __LDG_PTR(addr)mem); \ - return (c_typ)out; \ +#define __MAKE_LD(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ + __device__ __forceinline__ c_typ __ld##cop(const c_typ* addr) \ + { \ + int_typ out; \ + asm("ld." #cop "." ptx_typ " %0, [%1];" : "=" inl_typ(out) : __LDG_PTR(addr) mem); \ + return (c_typ)out; \ } -#define __MAKE_LD2(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ - __device__ __forceinline__ c_typ __ld ## cop (const c_typ* addr) { \ - int_typ out1, out2; \ - asm("ld." #cop ".v2." ptx_typ " {%0, %1}, [%2];" \ - : "=" inl_typ(out1), "=" inl_typ(out2) : __LDG_PTR(addr)mem); \ - c_typ out; \ - out.x = out1; \ - out.y = out2; \ - return out; \ +#define __MAKE_LD2(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ + __device__ __forceinline__ c_typ __ld##cop(const c_typ* addr) \ + { \ + int_typ out1, out2; \ + asm("ld." #cop ".v2." ptx_typ " {%0, %1}, [%2];" \ + : "=" inl_typ(out1), "=" inl_typ(out2) \ + : __LDG_PTR(addr) mem); \ + c_typ out; \ + out.x = out1; \ + out.y = out2; \ + return out; \ } -#define __MAKE_LD4(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ - __device__ __forceinline__ c_typ __ld ## cop (const c_typ* addr) { \ - int_typ out1, out2, out3, out4; \ - asm("ld." #cop".v4." ptx_typ " {%0, %1, %2, %3}, [%4];" \ - : "=" inl_typ(out1), "=" inl_typ(out2), \ - "=" inl_typ(out3), "=" inl_typ(out4) : __LDG_PTR(addr)mem); \ - c_typ out; \ - out.x = out1; \ - out.y = out2; \ - out.z = out3; \ - out.w = out4; \ - return out; \ +#define __MAKE_LD4(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ + __device__ __forceinline__ c_typ __ld##cop(const c_typ* addr) \ + { \ + int_typ out1, out2, out3, out4; \ + asm("ld." #cop ".v4." ptx_typ " {%0, %1, %2, %3}, [%4];" \ + : "=" inl_typ(out1), "=" inl_typ(out2), "=" inl_typ(out3), "=" inl_typ(out4) \ + : __LDG_PTR(addr) mem); \ + c_typ out; \ + out.x = out1; \ + out.y = out2; \ + out.z = out3; \ + out.w = out4; \ + return out; \ } __MAKE_LD(cg, char, short, "s8", "h", __NOARG) @@ -82,7 +86,6 @@ __MAKE_LD4(cg, int4, int, "s32", "r", __NOARG) __MAKE_LD4(cg, uint4, unsigned int, "u32", "r", __NOARG) __MAKE_LD4(cg, float4, float, "f32", "f", __NOARG) - __MAKE_LD(ca, char, short, "s8", "h", __NOARG) __MAKE_LD(ca, signed char, short, "s8", "h", __NOARG) __MAKE_LD(ca, unsigned char, short, "u8", "h", __NOARG) @@ -116,7 +119,6 @@ __MAKE_LD4(ca, int4, int, "s32", "r", __NOARG) __MAKE_LD4(ca, uint4, unsigned int, "u32", "r", __NOARG) __MAKE_LD4(ca, float4, float, "f32", "f", __NOARG) - __MAKE_LD(cs, char, short, "s8", "h", __NOARG) __MAKE_LD(cs, signed char, short, "s8", "h", __NOARG) __MAKE_LD(cs, unsigned char, short, "u8", "h", __NOARG) @@ -150,7 +152,6 @@ __MAKE_LD4(cs, int4, int, "s32", "r", __NOARG) __MAKE_LD4(cs, uint4, unsigned int, "u32", "r", __NOARG) __MAKE_LD4(cs, float4, float, "f32", "f", __NOARG) - __MAKE_LD(lu, char, short, "s8", "h", : "memory") __MAKE_LD(lu, signed char, short, "s8", "h", : "memory") __MAKE_LD(lu, unsigned char, short, "u8", "h", : "memory") @@ -184,7 +185,6 @@ __MAKE_LD4(lu, int4, int, "s32", "r", : "memory") __MAKE_LD4(lu, uint4, unsigned int, "u32", "r", : "memory") __MAKE_LD4(lu, float4, float, "f32", "f", : "memory") - __MAKE_LD(cv, char, short, "s8", "h", : "memory") __MAKE_LD(cv, signed char, short, "s8", "h", : "memory") __MAKE_LD(cv, unsigned char, short, "u8", "h", : "memory") @@ -218,26 +218,30 @@ __MAKE_LD4(cv, int4, int, "s32", "r", : "memory") __MAKE_LD4(cv, uint4, unsigned int, "u32", "r", : "memory") __MAKE_LD4(cv, float4, float, "f32", "f", : "memory") - -#define __MAKE_ST(cop, c_typ, int_typ, ptx_typ, inl_typ) \ - __device__ __forceinline__ void __st ## cop (c_typ* addr, c_typ v) { \ - asm("st." #cop "." ptx_typ " [%0], %1;" \ - :: __LDG_PTR(addr), inl_typ((int_typ)v) : "memory"); \ +#define __MAKE_ST(cop, c_typ, int_typ, ptx_typ, inl_typ) \ + __device__ __forceinline__ void __st##cop(c_typ* addr, c_typ v) \ + { \ + asm("st." #cop "." ptx_typ " [%0], %1;" ::__LDG_PTR(addr), inl_typ((int_typ)v) : "memory"); \ } -#define __MAKE_ST2(cop, c_typ, int_typ, ptx_typ, inl_typ) \ - __device__ __forceinline__ void __st ## cop (c_typ* addr, c_typ v) { \ - int_typ v1 = v.x, v2 = v.y; \ - asm("st." #cop ".v2." ptx_typ " [%0], {%1, %2};" \ - :: __LDG_PTR(addr), inl_typ(v1), inl_typ(v2) : "memory"); \ +#define __MAKE_ST2(cop, c_typ, int_typ, ptx_typ, inl_typ) \ + __device__ __forceinline__ void __st##cop(c_typ* addr, c_typ v) \ + { \ + int_typ v1 = v.x, v2 = v.y; \ + asm("st." #cop ".v2." ptx_typ " [%0], {%1, %2};" ::__LDG_PTR(addr), inl_typ(v1), inl_typ(v2) \ + : "memory"); \ } -#define __MAKE_ST4(cop, c_typ, int_typ, ptx_typ, inl_typ) \ - __device__ __forceinline__ c_typ __st ## cop (c_typ* addr, c_typ v) { \ - int_typ v1 = v.x, v2 = v.y, v3 = v.z, v4 = v.w; \ - asm("st." #cop ".v4." ptx_typ " [%0], {%1, %2, %3, %4};" \ - :: __LDG_PTR(addr), inl_typ(v1), inl_typ(v2), \ - inl_typ(v3), inl_typ(v4) : "memory"); \ +#define __MAKE_ST4(cop, c_typ, int_typ, ptx_typ, inl_typ) \ + __device__ __forceinline__ c_typ __st##cop(c_typ* addr, c_typ v) \ + { \ + int_typ v1 = v.x, v2 = v.y, v3 = v.z, v4 = v.w; \ + asm("st." #cop ".v4." ptx_typ " [%0], {%1, %2, %3, %4};" ::__LDG_PTR(addr), \ + inl_typ(v1), \ + inl_typ(v2), \ + inl_typ(v3), \ + inl_typ(v4) \ + : "memory"); \ } __MAKE_ST(wb, char, short, "s8", "h") @@ -273,7 +277,6 @@ __MAKE_ST4(wb, int4, int, "s32", "r") __MAKE_ST4(wb, uint4, unsigned int, "u32", "r") __MAKE_ST4(wb, float4, float, "f32", "f") - __MAKE_ST(cg, char, short, "s8", "h") __MAKE_ST(cg, signed char, short, "s8", "h") __MAKE_ST(cg, unsigned char, short, "u8", "h") @@ -307,7 +310,6 @@ __MAKE_ST4(cg, int4, int, "s32", "r") __MAKE_ST4(cg, uint4, unsigned int, "u32", "r") __MAKE_ST4(cg, float4, float, "f32", "f") - __MAKE_ST(cs, char, short, "s8", "h") __MAKE_ST(cs, signed char, short, "s8", "h") __MAKE_ST(cs, unsigned char, short, "u8", "h") @@ -341,7 +343,6 @@ __MAKE_ST4(cs, int4, int, "s32", "r") __MAKE_ST4(cs, uint4, unsigned int, "u32", "r") __MAKE_ST4(cs, float4, float, "f32", "f") - __MAKE_ST(wt, char, short, "s8", "h") __MAKE_ST(wt, signed char, short, "s8", "h") __MAKE_ST(wt, unsigned char, short, "u8", "h") @@ -375,7 +376,6 @@ __MAKE_ST4(wt, int4, int, "s32", "r") __MAKE_ST4(wt, uint4, unsigned int, "u32", "r") __MAKE_ST4(wt, float4, float, "f32", "f") - #undef __MAKE_ST4 #undef __MAKE_ST2 #undef __MAKE_ST @@ -386,6 +386,6 @@ __MAKE_ST4(wt, float4, float, "f32", "f") #undef __LBITS #undef __LDG_PTR -#endif // defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320)) +#endif // defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320)) -#endif // defined(__CLANG_CUDA_ADDITIONAL_INTRINSICS_H__) +#endif // defined(__CLANG_CUDA_ADDITIONAL_INTRINSICS_H__) diff --git a/cpp/scripts/run-cmake-format.sh b/cpp/scripts/run-cmake-format.sh new file mode 100755 index 0000000000..afe6e47c8e --- /dev/null +++ b/cpp/scripts/run-cmake-format.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Copyright (c) 2022, NVIDIA CORPORATION. + +# This script is a wrapper for cmakelang that may be used with pre-commit. The +# wrapping is necessary because RAPIDS libraries split configuration for +# cmakelang linters between a local config file and a second config file that's +# shared across all of RAPIDS via rapids-cmake. In order to keep it up to date +# this file is only maintained in one place (the rapids-cmake repo) and +# pulled down during builds. We need a way to invoke CMake linting commands +# without causing pre-commit failures (which could block local commits or CI), +# while also being sufficiently flexible to allow users to maintain the config +# file independently of a build directory. +# +# This script provides the minimal functionality to enable those use cases. It +# searches in a number of predefined locations for the rapids-cmake config file +# and exits gracefully if the file is not found. If a user wishes to specify a +# config file at a nonstandard location, they may do so by setting the +# environment variable RAPIDS_CMAKE_FORMAT_FILE. +# +# This script can be invoked directly anywhere within the project repository. +# Alternatively, it may be invoked as a pre-commit hook via +# `pre-commit run (cmake-format)|(cmake-lint)`. +# +# Usage: +# bash run-cmake-format.sh {cmake-format,cmake-lint} infile [infile ...] + +status=0 +if [ -z ${CUDF_ROOT:+PLACEHOLDER} ]; then + CUDF_BUILD_DIR=$(git rev-parse --show-toplevel 2>&1)/cpp/build + status=$? +else + CUDF_BUILD_DIR=${CUDF_ROOT} +fi + +if ! [ ${status} -eq 0 ]; then + if [[ ${CUDF_BUILD_DIR} == *"not a git repository"* ]]; then + echo "This script must be run inside the cudf repository, or the CUDF_ROOT environment variable must be set." + else + echo "Script failed with unknown error attempting to determine project root:" + echo ${CUDF_BUILD_DIR} + fi + exit 1 +fi + +DEFAULT_FORMAT_FILE_LOCATIONS=( + "${CUDF_BUILD_DIR:-${HOME}}/_deps/rapids-cmake-src/cmake-format-rapids-cmake.json" + "cpp/libcudf_kafka/build/_deps/rapids-cmake-src/cmake-format-rapids-cmake.json" +) + +if [ -z ${RAPIDS_CMAKE_FORMAT_FILE:+PLACEHOLDER} ]; then + for file_path in ${DEFAULT_FORMAT_FILE_LOCATIONS[@]}; do + if [ -f ${file_path} ]; then + RAPIDS_CMAKE_FORMAT_FILE=${file_path} + break + fi + done +fi + +if [ -z ${RAPIDS_CMAKE_FORMAT_FILE:+PLACEHOLDER} ]; then + echo "The rapids-cmake cmake-format configuration file was not found at any of the default search locations: " + echo "" + ( IFS=$'\n'; echo "${DEFAULT_FORMAT_FILE_LOCATIONS[*]}" ) + echo "" + echo "Try setting the environment variable RAPIDS_CMAKE_FORMAT_FILE to the path to the config file." + exit 0 +else + echo "Using format file ${RAPIDS_CMAKE_FORMAT_FILE}" +fi + +if [[ $1 == "cmake-format" ]]; then + cmake-format -i --config-files cpp/cmake/config.json ${RAPIDS_CMAKE_FORMAT_FILE} -- ${@:2} +elif [[ $1 == "cmake-lint" ]]; then + # Since the pre-commit hook is verbose, we have to be careful to only + # present cmake-lint's output (which is quite verbose) if we actually + # observe a failure. + OUTPUT=$(cmake-lint --config-files cpp/cmake/config.json ${RAPIDS_CMAKE_FORMAT_FILE} -- ${@:2}) + status=$? + + if ! [ ${status} -eq 0 ]; then + echo "${OUTPUT}" + fi + exit ${status} +fi diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 4280be91ff..86d1029988 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -1,41 +1,37 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2021-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= - -################################################################################################### -# - compiler function ----------------------------------------------------------------------------- +# ################################################################################################## +# * compiler function ----------------------------------------------------------------------------- function(ConfigureTest) - set(options OPTIONAL DIST NN) - set(oneValueArgs NAME ) - set(multiValueArgs PATH TARGETS CONFIGURATIONS) + set(options OPTIONAL DIST NN) + set(oneValueArgs NAME) + set(multiValueArgs PATH TARGETS CONFIGURATIONS) - cmake_parse_arguments(ConfigureTest "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) + cmake_parse_arguments(ConfigureTest "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(TEST_NAME ${ConfigureTest_NAME}) + set(TEST_NAME ${ConfigureTest_NAME}) - add_executable(${TEST_NAME} ${ConfigureTest_PATH}) + add_executable(${TEST_NAME} ${ConfigureTest_PATH}) - message("TEST PATH: ${ConfigureTest_PATH}") + message("TEST PATH: ${ConfigureTest_PATH}") - target_link_libraries(${TEST_NAME} - PRIVATE - raft::raft + target_link_libraries( + ${TEST_NAME} + PRIVATE raft::raft $<$:raft::distance> $<$:raft::nn> GTest::gtest @@ -43,244 +39,233 @@ function(ConfigureTest) Threads::Threads $ $ - ) + ) - add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) - set_target_properties(${TEST_NAME} - PROPERTIES - # set target compile options - INSTALL_RPATH "\$ORIGIN/../../../lib" - ) + set_target_properties( + ${TEST_NAME} PROPERTIES # set target compile options + INSTALL_RPATH "\$ORIGIN/../../../lib" + ) - target_compile_options(${TEST_NAME} - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) + target_compile_options( + ${TEST_NAME} PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) - target_include_directories(${TEST_NAME} - PUBLIC "$" - ) + target_include_directories(${TEST_NAME} PUBLIC "$") - install( - TARGETS ${TEST_NAME} - COMPONENT testing - DESTINATION bin/gtests/libraft - EXCLUDE_FROM_ALL) + install( + TARGETS ${TEST_NAME} + COMPONENT testing + DESTINATION bin/gtests/libraft + EXCLUDE_FROM_ALL + ) endfunction() +# ################################################################################################## +# test sources ################################################################################## +# ################################################################################################## -################################################################################################### -### test sources ################################################################################## -################################################################################################### - -################################################################################################### -# - distance tests ------------------------------------------------------------------------- +# ################################################################################################## +# * distance tests ------------------------------------------------------------------------- if(BUILD_TESTS) - ConfigureTest(NAME CLUSTER_TEST - PATH - test/cluster/kmeans.cu - test/cluster_solvers.cu - test/cluster/linkage.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME CLUSTER_TEST PATH test/cluster/kmeans.cu test/cluster_solvers.cu test/cluster/linkage.cu + OPTIONAL DIST NN + ) - ConfigureTest(NAME CORE_TEST - PATH - test/common/logger.cpp - test/handle.cpp - test/interruptible.cu - test/nvtx.cpp - test/mdarray.cu - test/mdspan_utils.cu - test/span.cpp - test/span.cu - test/test.cpp - ) + ConfigureTest( + NAME CORE_TEST PATH test/common/logger.cpp test/handle.cpp test/interruptible.cu test/nvtx.cpp + test/mdarray.cu test/mdspan_utils.cu test/span.cpp test/span.cu test/test.cpp + ) - ConfigureTest(NAME DISTANCE_TEST - PATH - test/distance/dist_adj.cu - test/distance/dist_canberra.cu - test/distance/dist_chebyshev.cu - test/distance/dist_correlation.cu - test/distance/dist_cos.cu - test/distance/dist_euc_exp.cu - test/distance/dist_euc_unexp.cu - test/distance/dist_hamming.cu - test/distance/dist_hellinger.cu - test/distance/dist_jensen_shannon.cu - test/distance/dist_kl_divergence.cu - test/distance/dist_l1.cu - test/distance/dist_minkowski.cu - test/distance/dist_russell_rao.cu - test/distance/fused_l2_nn.cu - test/distance/gram.cu - OPTIONAL DIST - ) + ConfigureTest( + NAME + DISTANCE_TEST + PATH + test/distance/dist_adj.cu + test/distance/dist_canberra.cu + test/distance/dist_chebyshev.cu + test/distance/dist_correlation.cu + test/distance/dist_cos.cu + test/distance/dist_euc_exp.cu + test/distance/dist_euc_unexp.cu + test/distance/dist_hamming.cu + test/distance/dist_hellinger.cu + test/distance/dist_jensen_shannon.cu + test/distance/dist_kl_divergence.cu + test/distance/dist_l1.cu + test/distance/dist_minkowski.cu + test/distance/dist_russell_rao.cu + test/distance/fused_l2_nn.cu + test/distance/gram.cu + OPTIONAL + DIST + ) - ConfigureTest(NAME LABEL_TEST - PATH - test/label/label.cu - test/label/merge_labels.cu - ) + ConfigureTest(NAME LABEL_TEST PATH test/label/label.cu test/label/merge_labels.cu) - ConfigureTest(NAME LINALG_TEST - PATH - test/linalg/add.cu - test/linalg/axpy.cu - test/linalg/binary_op.cu - test/linalg/cholesky_r1.cu - test/linalg/coalesced_reduction.cu - test/linalg/divide.cu - test/linalg/eig.cu - test/linalg/eig_sel.cu - test/linalg/gemm_layout.cu - test/linalg/gemv.cu - test/linalg/map.cu - test/linalg/map_then_reduce.cu - test/linalg/matrix_vector.cu - test/linalg/matrix_vector_op.cu - test/linalg/mean_squared_error.cu - test/linalg/multiply.cu - test/linalg/norm.cu - test/linalg/power.cu - test/linalg/reduce.cu - test/linalg/reduce_cols_by_key.cu - test/linalg/reduce_rows_by_key.cu - test/linalg/rsvd.cu - test/linalg/sqrt.cu - test/linalg/strided_reduction.cu - test/linalg/subtract.cu - test/linalg/svd.cu - test/linalg/ternary_op.cu - test/linalg/transpose.cu - test/linalg/unary_op.cu - ) + ConfigureTest( + NAME + LINALG_TEST + PATH + test/linalg/add.cu + test/linalg/axpy.cu + test/linalg/binary_op.cu + test/linalg/cholesky_r1.cu + test/linalg/coalesced_reduction.cu + test/linalg/divide.cu + test/linalg/eig.cu + test/linalg/eig_sel.cu + test/linalg/gemm_layout.cu + test/linalg/gemv.cu + test/linalg/map.cu + test/linalg/map_then_reduce.cu + test/linalg/matrix_vector.cu + test/linalg/matrix_vector_op.cu + test/linalg/mean_squared_error.cu + test/linalg/multiply.cu + test/linalg/norm.cu + test/linalg/power.cu + test/linalg/reduce.cu + test/linalg/reduce_cols_by_key.cu + test/linalg/reduce_rows_by_key.cu + test/linalg/rsvd.cu + test/linalg/sqrt.cu + test/linalg/strided_reduction.cu + test/linalg/subtract.cu + test/linalg/svd.cu + test/linalg/ternary_op.cu + test/linalg/transpose.cu + test/linalg/unary_op.cu + ) - ConfigureTest(NAME MATRIX_TEST - PATH - test/matrix/argmax.cu - test/matrix/columnSort.cu - test/matrix/diagonal.cu - test/matrix/gather.cu - test/matrix/linewise_op.cu - test/matrix/math.cu - test/matrix/matrix.cu - test/matrix/norm.cu - test/matrix/reverse.cu - test/matrix/slice.cu - test/matrix/triangular.cu - test/spectral_matrix.cu - ) + ConfigureTest( + NAME + MATRIX_TEST + PATH + test/matrix/argmax.cu + test/matrix/columnSort.cu + test/matrix/diagonal.cu + test/matrix/gather.cu + test/matrix/linewise_op.cu + test/matrix/math.cu + test/matrix/matrix.cu + test/matrix/norm.cu + test/matrix/reverse.cu + test/matrix/slice.cu + test/matrix/triangular.cu + test/spectral_matrix.cu + ) - ConfigureTest(NAME RANDOM_TEST - PATH - test/random/make_blobs.cu - test/random/make_regression.cu - test/random/multi_variable_gaussian.cu - test/random/permute.cu - test/random/rng.cu - test/random/rng_int.cu - test/random/rmat_rectangular_generator.cu - test/random/sample_without_replacement.cu - ) + ConfigureTest( + NAME + RANDOM_TEST + PATH + test/random/make_blobs.cu + test/random/make_regression.cu + test/random/multi_variable_gaussian.cu + test/random/permute.cu + test/random/rng.cu + test/random/rng_int.cu + test/random/rmat_rectangular_generator.cu + test/random/sample_without_replacement.cu + ) - ConfigureTest(NAME SOLVERS_TEST - PATH - test/cluster_solvers_deprecated.cu - test/eigen_solvers.cu - test/lap/lap.cu - test/mst.cu - ) + ConfigureTest( + NAME SOLVERS_TEST PATH test/cluster_solvers_deprecated.cu test/eigen_solvers.cu test/lap/lap.cu + test/mst.cu + ) - ConfigureTest(NAME SPARSE_TEST - PATH - test/sparse/add.cu - test/sparse/convert_coo.cu - test/sparse/convert_csr.cu - test/sparse/csr_row_slice.cu - test/sparse/csr_to_dense.cu - test/sparse/csr_transpose.cu - test/sparse/degree.cu - test/sparse/filter.cu - test/sparse/norm.cu - test/sparse/reduce.cu - test/sparse/row_op.cu - test/sparse/sort.cu - test/sparse/symmetrize.cu - ) + ConfigureTest( + NAME + SPARSE_TEST + PATH + test/sparse/add.cu + test/sparse/convert_coo.cu + test/sparse/convert_csr.cu + test/sparse/csr_row_slice.cu + test/sparse/csr_to_dense.cu + test/sparse/csr_transpose.cu + test/sparse/degree.cu + test/sparse/filter.cu + test/sparse/norm.cu + test/sparse/reduce.cu + test/sparse/row_op.cu + test/sparse/sort.cu + test/sparse/symmetrize.cu + ) - ConfigureTest(NAME SPARSE_DIST_TEST - PATH - test/sparse/dist_coo_spmv.cu - test/sparse/distance.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME SPARSE_DIST_TEST PATH test/sparse/dist_coo_spmv.cu test/sparse/distance.cu OPTIONAL DIST + NN + ) - ConfigureTest(NAME SPARSE_NEIGHBORS_TEST - PATH - test/sparse/neighbors/connect_components.cu - test/sparse/neighbors/brute_force.cu - test/sparse/neighbors/knn_graph.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME SPARSE_NEIGHBORS_TEST PATH test/sparse/neighbors/connect_components.cu + test/sparse/neighbors/brute_force.cu test/sparse/neighbors/knn_graph.cu OPTIONAL DIST NN + ) - ConfigureTest(NAME NEIGHBORS_TEST - PATH - test/neighbors/ann_ivf_flat.cu - test/neighbors/ann_ivf_pq/test_float_int64_t.cu - test/neighbors/ann_ivf_pq/test_float_uint32_t.cu - test/neighbors/ann_ivf_pq/test_float_uint64_t.cu - test/neighbors/ann_ivf_pq/test_int8_t_uint64_t.cu - test/neighbors/ann_ivf_pq/test_uint8_t_uint64_t.cu - test/neighbors/knn.cu - test/neighbors/fused_l2_knn.cu - test/neighbors/haversine.cu - test/neighbors/ball_cover.cu - test/neighbors/epsilon_neighborhood.cu - test/neighbors/faiss_mr.cu - test/neighbors/selection.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME + NEIGHBORS_TEST + PATH + test/neighbors/ann_ivf_flat.cu + test/neighbors/ann_ivf_pq/test_float_int64_t.cu + test/neighbors/ann_ivf_pq/test_float_uint32_t.cu + test/neighbors/ann_ivf_pq/test_float_uint64_t.cu + test/neighbors/ann_ivf_pq/test_int8_t_uint64_t.cu + test/neighbors/ann_ivf_pq/test_uint8_t_uint64_t.cu + test/neighbors/knn.cu + test/neighbors/fused_l2_knn.cu + test/neighbors/haversine.cu + test/neighbors/ball_cover.cu + test/neighbors/epsilon_neighborhood.cu + test/neighbors/faiss_mr.cu + test/neighbors/selection.cu + OPTIONAL + DIST + NN + ) - ConfigureTest(NAME STATS_TEST - PATH - test/stats/accuracy.cu - test/stats/adjusted_rand_index.cu - test/stats/completeness_score.cu - test/stats/contingencyMatrix.cu - test/stats/cov.cu - test/stats/dispersion.cu - test/stats/entropy.cu - test/stats/histogram.cu - test/stats/homogeneity_score.cu - test/stats/information_criterion.cu - test/stats/kl_divergence.cu - test/stats/mean.cu - test/stats/meanvar.cu - test/stats/mean_center.cu - test/stats/minmax.cu - test/stats/mutual_info_score.cu - test/stats/r2_score.cu - test/stats/rand_index.cu - test/stats/regression_metrics.cu - test/stats/silhouette_score.cu - test/stats/stddev.cu - test/stats/sum.cu - test/stats/trustworthiness.cu - test/stats/weighted_mean.cu - test/stats/v_measure.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME + STATS_TEST + PATH + test/stats/accuracy.cu + test/stats/adjusted_rand_index.cu + test/stats/completeness_score.cu + test/stats/contingencyMatrix.cu + test/stats/cov.cu + test/stats/dispersion.cu + test/stats/entropy.cu + test/stats/histogram.cu + test/stats/homogeneity_score.cu + test/stats/information_criterion.cu + test/stats/kl_divergence.cu + test/stats/mean.cu + test/stats/meanvar.cu + test/stats/mean_center.cu + test/stats/minmax.cu + test/stats/mutual_info_score.cu + test/stats/r2_score.cu + test/stats/rand_index.cu + test/stats/regression_metrics.cu + test/stats/silhouette_score.cu + test/stats/stddev.cu + test/stats/sum.cu + test/stats/trustworthiness.cu + test/stats/weighted_mean.cu + test/stats/v_measure.cu + OPTIONAL + DIST + NN + ) - ConfigureTest(NAME UTILS_TEST - PATH - test/common/seive.cu - test/cudart_utils.cpp - test/device_atomics.cu - test/integer_utils.cpp - test/pow2_utils.cu - ) + ConfigureTest( + NAME UTILS_TEST PATH test/common/seive.cu test/cudart_utils.cpp test/device_atomics.cu + test/integer_utils.cpp test/pow2_utils.cu + ) endif() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..a756854eae --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[tool.black] +line-length = 79 +target-version = ["py38"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/.flake8.cython b/python/.flake8.cython index 3cd436d3f3..9dde36fe8a 100644 --- a/python/.flake8.cython +++ b/python/.flake8.cython @@ -16,7 +16,7 @@ [flake8] filename = *.pyx, *.pxd -exclude = *.egg, build, docs, .git +exclude = *.egg, build, docs, .git, __init__.py, cpp, thirdparty, versioneer.py, _version.py ignore = E999, E225, E226, E227, W503, W504 # Rules ignored: diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index f3dd7eaaa2..063139e8c7 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -25,26 +25,30 @@ project( # language to be enabled here. The test project that is built in scikit-build to verify # various linking options for the python library is hardcoded to build with C, so until # that is fixed we need to keep C. - C - CXX) + C CXX +) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" ON) +option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" + ON +) # If the user requested it we attempt to find RAFT. if(FIND_RAFT_CPP) - find_package(raft ${pylibraft_version} REQUIRED COMPONENTS distance) - if(NOT TARGET raft::raft_distance_lib) - message(FATAL_ERROR "Building against a preexisting libraft library requires the distance components of that library to have been built!") + find_package(raft ${pylibraft_version} REQUIRED COMPONENTS distance) + if(NOT TARGET raft::raft_distance_lib) + message( + FATAL_ERROR + "Building against a preexisting libraft library requires the distance components of that library to have been built!" + ) - endif() + endif() else() set(raft_FOUND OFF) endif() if(NOT raft_FOUND) - # TODO: This will not be necessary once we upgrade to CMake 3.22, which will - # pull in the required languages for the C++ project even if this project - # does not require those languges. + # TODO: This will not be necessary once we upgrade to CMake 3.22, which will pull in the required + # languages for the C++ project even if this project does not require those languges. include(rapids-cuda) rapids_cuda_init_architectures(pylibraft) enable_language(CUDA) @@ -58,9 +62,9 @@ if(NOT raft_FOUND) set(RAFT_COMPILE_DIST_LIBRARY ON) add_subdirectory(../../cpp raft-cpp) - # When building the C++ libraries from source we must copy - # libraft_distance.so alongside the pairwise_distance and random Cython libraries - # TODO: when we have a single 'compiled' raft library, we shouldn't need this + # When building the C++ libraries from source we must copy libraft_distance.so alongside the + # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft + # library, we shouldn't need this install(TARGETS raft_distance_lib DESTINATION pylibraft/distance) install(TARGETS raft_distance_lib DESTINATION pylibraft/random) endif() diff --git a/python/pylibraft/pylibraft/_version.py b/python/pylibraft/pylibraft/_version.py index 58cd44da3b..81de84e96e 100644 --- a/python/pylibraft/pylibraft/_version.py +++ b/python/pylibraft/pylibraft/_version.py @@ -70,7 +70,7 @@ def decorate(f): def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None ): """Call the given command(s).""" assert isinstance(commands, list) @@ -85,7 +85,7 @@ def run_command( env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), - ) + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -123,7 +123,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return { - "version": dirname[len(parentdir_prefix):], + "version": dirname[len(parentdir_prefix) :], "full-revisionid": None, "dirty": False, "error": None, @@ -193,7 +193,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -210,7 +210,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) return { @@ -264,7 +264,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): "--long", "--match", "%s*" % tag_prefix, - ], + ], cwd=root, ) # --long was added in git-1.5.5 @@ -299,7 +299,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if not mo: # unparseable. Maybe git-describe is misbehaving? pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out + "unable to parse git-describe output: '%s'" % describe_out ) return pieces @@ -314,7 +314,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): tag_prefix, ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index 44e34e0213..497fb52735 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -18,11 +18,11 @@ set(linked_libraries raft::raft raft::distance) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - MODULE_PREFIX cluster_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX cluster_ +) foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") endforeach() diff --git a/python/pylibraft/pylibraft/cluster/kmeans.pyx b/python/pylibraft/pylibraft/cluster/kmeans.pyx index c2d445f970..059194531c 100644 --- a/python/pylibraft/pylibraft/cluster/kmeans.pyx +++ b/python/pylibraft/pylibraft/cluster/kmeans.pyx @@ -20,14 +20,14 @@ import numpy as np -from libc.stdint cimport uintptr_t from cython.operator cimport dereference as deref - -from libcpp cimport bool -from libcpp cimport nullptr +from libc.stdint cimport uintptr_t +from libcpp cimport bool, nullptr from pylibraft.common import Handle + from pylibraft.common.handle cimport handle_t + from pylibraft.common.input_validation import * from pylibraft.distance import DISTANCE_TYPES diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index d9ed2d12fc..b8410d6c66 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -18,11 +18,11 @@ set(linked_libraries raft::raft) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - MODULE_PREFIX common_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX common_ +) foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") endforeach() diff --git a/python/pylibraft/pylibraft/common/cuda.pxd b/python/pylibraft/pylibraft/common/cuda.pxd index ae6246dee1..a44d9aeb63 100644 --- a/python/pylibraft/pylibraft/common/cuda.pxd +++ b/python/pylibraft/pylibraft/common/cuda.pxd @@ -16,6 +16,7 @@ from cuda.ccudart cimport cudaStream_t + cdef class Stream: cdef cudaStream_t s diff --git a/python/pylibraft/pylibraft/common/cuda.pyx b/python/pylibraft/pylibraft/common/cuda.pyx index eb48f64cf1..24102c34af 100644 --- a/python/pylibraft/pylibraft/common/cuda.pyx +++ b/python/pylibraft/pylibraft/common/cuda.pyx @@ -19,16 +19,16 @@ # cython: embedsignature = True # cython: language_level = 3 -from cuda.ccudart cimport( - cudaStream_t, +from cuda.ccudart cimport ( cudaError_t, - cudaSuccess, + cudaGetErrorName, + cudaGetErrorString, + cudaGetLastError, + cudaStream_t, cudaStreamCreate, cudaStreamDestroy, cudaStreamSynchronize, - cudaGetLastError, - cudaGetErrorString, - cudaGetErrorName + cudaSuccess, ) diff --git a/python/pylibraft/pylibraft/common/handle.pxd b/python/pylibraft/pylibraft/common/handle.pxd index d7abe236ba..e763768eac 100644 --- a/python/pylibraft/pylibraft/common/handle.pxd +++ b/python/pylibraft/pylibraft/common/handle.pxd @@ -20,11 +20,11 @@ # cython: language_level = 3 -from libcpp.memory cimport shared_ptr -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from libcpp.memory cimport shared_ptr, unique_ptr + from rmm._lib.cuda_stream_pool cimport cuda_stream_pool -from libcpp.memory cimport shared_ptr -from libcpp.memory cimport unique_ptr +from rmm._lib.cuda_stream_view cimport cuda_stream_view + cdef extern from "raft/core/handle.hpp" namespace "raft" nogil: cdef cppclass handle_t: diff --git a/python/pylibraft/pylibraft/common/handle.pyx b/python/pylibraft/pylibraft/common/handle.pyx index 83a4676076..60041cb2af 100644 --- a/python/pylibraft/pylibraft/common/handle.pyx +++ b/python/pylibraft/pylibraft/common/handle.pyx @@ -20,10 +20,11 @@ # cython: language_level = 3 # import raft -from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread -from rmm._lib.cuda_stream_view cimport cuda_stream_view + +from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread, cuda_stream_view from .cuda cimport Stream + from .cuda import CudaRuntimeError diff --git a/python/pylibraft/pylibraft/common/interruptible.pxd b/python/pylibraft/pylibraft/common/interruptible.pxd index a4c7d90ac2..aaccf8aeab 100644 --- a/python/pylibraft/pylibraft/common/interruptible.pxd +++ b/python/pylibraft/pylibraft/common/interruptible.pxd @@ -20,8 +20,10 @@ # cython: language_level = 3 from libcpp.memory cimport shared_ptr + from rmm._lib.cuda_stream_view cimport cuda_stream_view + cdef extern from "raft/core/interruptible.hpp" namespace "raft" nogil: cdef cppclass interruptible: void cancel() diff --git a/python/pylibraft/pylibraft/common/interruptible.pyx b/python/pylibraft/pylibraft/common/interruptible.pyx index dfc95490ed..fc2e6d9e1f 100644 --- a/python/pylibraft/pylibraft/common/interruptible.pyx +++ b/python/pylibraft/pylibraft/common/interruptible.pyx @@ -21,10 +21,12 @@ import contextlib import signal + +from cuda.ccudart cimport cudaStream_t from cython.operator cimport dereference from rmm._lib.cuda_stream_view cimport cuda_stream_view -from cuda.ccudart cimport cudaStream_t + from .cuda cimport Stream diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index d074171e58..c9f4e8f58e 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -13,17 +13,16 @@ # ============================================================================= # Set the list of Cython files to build -set(cython_sources pairwise_distance.pyx - fused_l2_nn.pyx) +set(cython_sources pairwise_distance.pyx fused_l2_nn.pyx) set(linked_libraries raft::raft raft::distance) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - MODULE_PREFIX distance_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX distance_ +) foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") endforeach() diff --git a/python/pylibraft/pylibraft/distance/__init__.py b/python/pylibraft/pylibraft/distance/__init__.py index 7d68f02936..b251e71ba3 100644 --- a/python/pylibraft/pylibraft/distance/__init__.py +++ b/python/pylibraft/pylibraft/distance/__init__.py @@ -14,5 +14,4 @@ # from .fused_l2_nn import fused_l2_nn_argmin -from .pairwise_distance import distance as pairwise_distance -from .pairwise_distance import DISTANCE_TYPES \ No newline at end of file +from .pairwise_distance import DISTANCE_TYPES, distance as pairwise_distance diff --git a/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx b/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx index 7abc32119b..7f59ee64d7 100644 --- a/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx +++ b/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx @@ -20,12 +20,14 @@ import numpy as np -from libc.stdint cimport uintptr_t from cython.operator cimport dereference as deref - +from libc.stdint cimport uintptr_t from libcpp cimport bool + from .distance_type cimport DistanceType + from pylibraft.common import Handle + from pylibraft.common.handle cimport handle_t diff --git a/python/pylibraft/pylibraft/distance/pairwise_distance.pyx b/python/pylibraft/pylibraft/distance/pairwise_distance.pyx index 0f7626e8d1..34ed1cfef2 100644 --- a/python/pylibraft/pylibraft/distance/pairwise_distance.pyx +++ b/python/pylibraft/pylibraft/distance/pairwise_distance.pyx @@ -20,13 +20,14 @@ import numpy as np -from libc.stdint cimport uintptr_t from cython.operator cimport dereference as deref - +from libc.stdint cimport uintptr_t from libcpp cimport bool + from .distance_type cimport DistanceType from pylibraft.common import Handle + from pylibraft.common.handle cimport handle_t diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index bc7357bb8a..d0e7b74ff5 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -14,17 +14,17 @@ # Set the list of Cython files to build set(cython_sources rmat_rectangular_generator.pyx) -# TODO: should finally be replaced with 'compiled' library to be more generic, -# when that is available +# TODO: should finally be replaced with 'compiled' library to be more generic, when that is +# available set(linked_libraries raft::raft raft::distance) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - MODULE_PREFIX random_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX random_ +) foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") endforeach() diff --git a/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx b/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx index cef19295ac..2127abb950 100644 --- a/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx +++ b/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx @@ -20,13 +20,15 @@ import numpy as np -from libc.stdint cimport uintptr_t, int64_t from cython.operator cimport dereference as deref +from libc.stdint cimport int64_t, uintptr_t + from pylibraft.common import Handle -from pylibraft.common.handle cimport handle_t -from .rng_state cimport RngState from libcpp cimport bool +from pylibraft.common.handle cimport handle_t + +from .rng_state cimport RngState cdef extern from "raft_distance/random/rmat_rectangular_generator.hpp" \ diff --git a/python/pylibraft/pylibraft/random/rng_state.pxd b/python/pylibraft/pylibraft/random/rng_state.pxd index 9d1e2d17e3..8c89db3618 100644 --- a/python/pylibraft/pylibraft/random/rng_state.pxd +++ b/python/pylibraft/pylibraft/random/rng_state.pxd @@ -20,6 +20,7 @@ from libc.stdint cimport uint64_t + cdef extern from "raft/random/rng_state.hpp" namespace "raft::random" nogil: ctypedef enum GeneratorType: diff --git a/python/pylibraft/pylibraft/test/test_distance.py b/python/pylibraft/pylibraft/test/test_distance.py index 7f35a25493..c0cd93d3b5 100644 --- a/python/pylibraft/pylibraft/test/test_distance.py +++ b/python/pylibraft/pylibraft/test/test_distance.py @@ -13,22 +13,31 @@ # limitations under the License. # -from scipy.spatial.distance import cdist -import pytest import numpy as np - +import pytest from pylibraft.common import Handle from pylibraft.distance import pairwise_distance - from pylibraft.testing.utils import TestDeviceBuffer +from scipy.spatial.distance import cdist @pytest.mark.parametrize("n_rows", [100]) @pytest.mark.parametrize("n_cols", [100]) -@pytest.mark.parametrize("metric", ["euclidean", "cityblock", "chebyshev", - "canberra", "correlation", "hamming", - "jensenshannon", "russellrao", "cosine", - "sqeuclidean"]) +@pytest.mark.parametrize( + "metric", + [ + "euclidean", + "cityblock", + "chebyshev", + "canberra", + "correlation", + "hamming", + "jensenshannon", + "russellrao", + "cosine", + "sqeuclidean", + ], +) @pytest.mark.parametrize("order", ["F", "C"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_distance(n_rows, n_cols, metric, order, dtype): diff --git a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py b/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py index 1ce1ee2d1f..2e760d4411 100644 --- a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py +++ b/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py @@ -13,13 +13,12 @@ # limitations under the License. # -from scipy.spatial.distance import cdist -import pytest import numpy as np - +import pytest from pylibraft.common import Handle from pylibraft.distance import fused_l2_nn_argmin from pylibraft.testing.utils import TestDeviceBuffer +from scipy.spatial.distance import cdist @pytest.mark.parametrize("n_rows", [10, 100]) @@ -43,8 +42,9 @@ def test_fused_l2_nn_minarg(n_rows, n_cols, n_clusters, dtype): output_device = TestDeviceBuffer(output, "C") handle = Handle() - fused_l2_nn_argmin(input1_device, input2_device, output_device, - True, handle=handle) + fused_l2_nn_argmin( + input1_device, input2_device, output_device, True, handle=handle + ) handle.sync() actual = output_device.copy_to_host() diff --git a/python/pylibraft/pylibraft/test/test_kmeans.py b/python/pylibraft/pylibraft/test/test_kmeans.py index 1d65470a82..b2dec5cb68 100644 --- a/python/pylibraft/pylibraft/test/test_kmeans.py +++ b/python/pylibraft/pylibraft/test/test_kmeans.py @@ -13,13 +13,11 @@ # limitations under the License. # -import pytest import numpy as np - -from pylibraft.common import Handle +import pytest from pylibraft.cluster.kmeans import compute_new_centroids +from pylibraft.common import Handle from pylibraft.distance import pairwise_distance - from pylibraft.testing.utils import TestDeviceBuffer @@ -29,8 +27,9 @@ @pytest.mark.parametrize("metric", ["euclidean", "sqeuclidean"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) @pytest.mark.parametrize("additional_args", [True, False]) -def test_compute_new_centroids(n_rows, n_cols, metric, n_clusters, dtype, - additional_args): +def test_compute_new_centroids( + n_rows, n_cols, metric, n_clusters, dtype, additional_args +): order = "C" @@ -44,16 +43,20 @@ def test_compute_new_centroids(n_rows, n_cols, metric, n_clusters, dtype, centroids = X[:n_clusters] centroids_device = TestDeviceBuffer(centroids, order) - weight_per_cluster = np.zeros((n_clusters, ), dtype=dtype) - weight_per_cluster_device = TestDeviceBuffer(weight_per_cluster, order) \ - if additional_args else None + weight_per_cluster = np.zeros((n_clusters,), dtype=dtype) + weight_per_cluster_device = ( + TestDeviceBuffer(weight_per_cluster, order) + if additional_args + else None + ) new_centroids = np.zeros((n_clusters, n_cols), dtype=dtype) new_centroids_device = TestDeviceBuffer(new_centroids, order) sample_weights = np.ones((n_rows,)).astype(dtype) / n_rows - sample_weights_device = TestDeviceBuffer(sample_weights, order) \ - if additional_args else None + sample_weights_device = ( + TestDeviceBuffer(sample_weights, order) if additional_args else None + ) # Compute new centroids naively dists = np.zeros((n_rows, n_clusters), dtype=dtype) @@ -72,13 +75,15 @@ def test_compute_new_centroids(n_rows, n_cols, metric, n_clusters, dtype, g = sample_weights[labels == i].sum() expected_centers[i, :] = j / g - compute_new_centroids(X_device, - centroids_device, - labels_device, - new_centroids_device, - sample_weights=sample_weights_device, - weight_per_cluster=weight_per_cluster_device, - handle=handle) + compute_new_centroids( + X_device, + centroids_device, + labels_device, + new_centroids_device, + sample_weights=sample_weights_device, + weight_per_cluster=weight_per_cluster_device, + handle=handle, + ) # pylibraft functions are often asynchronous so the # handle needs to be explicitly synchronized diff --git a/python/pylibraft/pylibraft/test/test_random.py b/python/pylibraft/pylibraft/test/test_random.py index e0b7140f1c..f6b338225b 100644 --- a/python/pylibraft/pylibraft/test/test_random.py +++ b/python/pylibraft/pylibraft/test/test_random.py @@ -13,12 +13,10 @@ # limitations under the License. # -import pytest import numpy as np - +import pytest from pylibraft.common import Handle from pylibraft.random import rmat - from pylibraft.testing.utils import TestDeviceBuffer diff --git a/python/pylibraft/pylibraft/test/test_z_interruptible.py b/python/pylibraft/pylibraft/test/test_z_interruptible.py index bda98edd13..adaac623ea 100644 --- a/python/pylibraft/pylibraft/test/test_z_interruptible.py +++ b/python/pylibraft/pylibraft/test/test_z_interruptible.py @@ -1,9 +1,10 @@ # Copyright (c) 2022, NVIDIA CORPORATION. import os -import pytest import signal import time + +import pytest from pylibraft.common.interruptible import cuda_interruptible, cuda_yield @@ -14,14 +15,15 @@ def send_ctrl_c(): def test_should_cancel_via_interruptible(): start_time = time.monotonic() - with pytest.raises(RuntimeError, match='this thread was cancelled'): + with pytest.raises(RuntimeError, match="this thread was cancelled"): with cuda_interruptible(): send_ctrl_c() cuda_yield() time.sleep(1.0) end_time = time.monotonic() - assert end_time < start_time + 0.5, \ - "The process seems to have waited, while it shouldn't have." + assert ( + end_time < start_time + 0.5 + ), "The process seems to have waited, while it shouldn't have." def test_should_cancel_via_python(): @@ -31,8 +33,9 @@ def test_should_cancel_via_python(): cuda_yield() time.sleep(1.0) end_time = time.monotonic() - assert end_time < start_time + 0.5, \ - "The process seems to have waited, while it shouldn't have." + assert ( + end_time < start_time + 0.5 + ), "The process seems to have waited, while it shouldn't have." def test_should_wait_no_interrupt(): @@ -41,8 +44,9 @@ def test_should_wait_no_interrupt(): cuda_yield() time.sleep(1.0) end_time = time.monotonic() - assert end_time > start_time + 0.5, \ - "The process seems to be cancelled, while it shouldn't be." + assert ( + end_time > start_time + 0.5 + ), "The process seems to be cancelled, while it shouldn't be." def test_should_wait_no_yield(): @@ -51,5 +55,6 @@ def test_should_wait_no_yield(): send_ctrl_c() time.sleep(1.0) end_time = time.monotonic() - assert end_time > start_time + 0.5, \ - "The process seems to be cancelled, while it shouldn't be." + assert ( + end_time > start_time + 0.5 + ), "The process seems to be cancelled, while it shouldn't be." diff --git a/python/pylibraft/pylibraft/testing/utils.py b/python/pylibraft/pylibraft/testing/utils.py index 979fbb5672..86cf4558db 100644 --- a/python/pylibraft/pylibraft/testing/utils.py +++ b/python/pylibraft/pylibraft/testing/utils.py @@ -19,12 +19,12 @@ class TestDeviceBuffer: - def __init__(self, ndarray, order): self.ndarray_ = ndarray - self.device_buffer_ = \ - rmm.DeviceBuffer.to_device(ndarray.ravel(order=order).tobytes()) + self.device_buffer_ = rmm.DeviceBuffer.to_device( + ndarray.ravel(order=order).tobytes() + ) @property def __cuda_array_interface__(self): @@ -35,8 +35,12 @@ def __cuda_array_interface__(self): return host_cai def copy_to_host(self): - return np.frombuffer(self.device_buffer_.tobytes(), - dtype=self.ndarray_.dtype, - like=self.ndarray_)\ - .astype(self.ndarray_.dtype)\ + return ( + np.frombuffer( + self.device_buffer_.tobytes(), + dtype=self.ndarray_.dtype, + like=self.ndarray_, + ) + .astype(self.ndarray_.dtype) .reshape(self.ndarray_.shape) + ) diff --git a/python/pylibraft/setup.py b/python/pylibraft/setup.py index 561c105d36..289214e856 100644 --- a/python/pylibraft/setup.py +++ b/python/pylibraft/setup.py @@ -14,40 +14,40 @@ # limitations under the License. # +import versioneer from setuptools import find_packages from skbuild import setup -import versioneer - -setup(name='pylibraft', - description="RAFT: Reusable Algorithms Functions and other Tools", - version=versioneer.get_version(), - classifiers=[ +setup( + name="pylibraft", + description="RAFT: Reusable Algorithms Functions and other Tools", + version=versioneer.get_version(), + classifiers=[ "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9" - ], - author="NVIDIA Corporation", - package_data={ - # Note: A dict comprehension with an explicit copy is necessary - # (rather than something simpler like a dict.fromkeys) because - # otherwise every package will refer to the same list and skbuild - # modifies it in place. - key: ["*.hpp", "*.pxd"] - for key in find_packages( - include=[ - "pylibraft.distance", - "pylibraft.distance.includes", - "pylibraft.common", - "pylibraft.common.includes", - "pylibraft.random", - "pylibraft.random.includes" - ] - ) - }, - packages=find_packages(include=['pylibraft', 'pylibraft.*']), - license="Apache", - cmdclass=versioneer.get_cmdclass(), - zip_safe=False - ) + "Programming Language :: Python :: 3.9", + ], + author="NVIDIA Corporation", + package_data={ + # Note: A dict comprehension with an explicit copy is necessary + # (rather than something simpler like a dict.fromkeys) because + # otherwise every package will refer to the same list and skbuild + # modifies it in place. + key: ["*.hpp", "*.pxd"] + for key in find_packages( + include=[ + "pylibraft.distance", + "pylibraft.distance.includes", + "pylibraft.common", + "pylibraft.common.includes", + "pylibraft.random", + "pylibraft.random.includes", + ] + ) + }, + packages=find_packages(include=["pylibraft", "pylibraft.*"]), + license="Apache", + cmdclass=versioneer.get_cmdclass(), + zip_safe=False, +) diff --git a/python/pylibraft/setuputils.py b/python/pylibraft/setuputils.py index d93e4b06a4..0a3f421856 100755 --- a/python/pylibraft/setuputils.py +++ b/python/pylibraft/setuputils.py @@ -51,15 +51,15 @@ def clean_folder(path): path : String Path to the folder to be cleaned. """ - shutil.rmtree(path + '/__pycache__', ignore_errors=True) + shutil.rmtree(path + "/__pycache__", ignore_errors=True) - folders = glob.glob(path + '/*/') + folders = glob.glob(path + "/*/") for folder in folders: - shutil.rmtree(folder + '/__pycache__', ignore_errors=True) + shutil.rmtree(folder + "/__pycache__", ignore_errors=True) clean_folder(folder) - cython_exts = glob.glob(folder + '/*.cpp') - cython_exts.extend(glob.glob(folder + '/*.cpython*')) + cython_exts = glob.glob(folder + "/*.cpp") + cython_exts.extend(glob.glob(folder + "/*.cpython*")) for file in cython_exts: os.remove(file) diff --git a/python/pylibraft/versioneer.py b/python/pylibraft/versioneer.py index b8c4bc423b..73e3259bf1 100644 --- a/python/pylibraft/versioneer.py +++ b/python/pylibraft/versioneer.py @@ -278,10 +278,12 @@ """ from __future__ import print_function + try: import configparser except ImportError: import ConfigParser as configparser + import errno import json import os @@ -309,11 +311,13 @@ def get_root(): setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -326,8 +330,10 @@ def get_root(): me_dir = os.path.normcase(os.path.splitext(me)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py) + ) except NameError: pass return root @@ -349,6 +355,7 @@ def get(parser, name): if parser.has_option("versioneer", name): return parser.get("versioneer", name) return None + cfg = VersioneerConfig() cfg.VCS = VCS cfg.style = get(parser, "style") or "" @@ -373,17 +380,20 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None +): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -391,10 +401,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -419,7 +432,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, p.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY[ + "git" +] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -994,7 +1009,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1003,7 +1018,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1011,19 +1026,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -1038,8 +1060,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command( + GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True + ) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1047,10 +1070,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1073,17 +1105,18 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = ( + "unable to parse git-describe output: '%s'" % describe_out + ) return pieces # tag @@ -1092,10 +1125,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1106,13 +1141,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command( + GITS, ["rev-list", "HEAD", "--count"], cwd=root + ) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1168,16 +1205,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1206,11 +1249,17 @@ def versions_from_file(filename): contents = f.read() except EnvironmentError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1219,8 +1268,9 @@ def versions_from_file(filename): def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps( + versions, sort_keys=True, indent=1, separators=(",", ": ") + ) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1252,8 +1302,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1367,11 +1416,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1391,9 +1442,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1416,8 +1471,9 @@ def get_versions(verbose=False): handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1471,9 +1527,13 @@ def get_versions(verbose=False): if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version(): @@ -1522,6 +1582,7 @@ def run(self): print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in both distutils and setuptools @@ -1554,14 +1615,17 @@ def run(self): # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join( + self.build_lib, cfg.versionfile_build + ) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -1582,17 +1646,21 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.distutils_buildexe import py2exe as _py2exe # py3 except ImportError: @@ -1611,13 +1679,17 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments @@ -1644,8 +1716,10 @@ def make_release_tree(self, base_dir, files): # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -1700,11 +1774,15 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except ( + EnvironmentError, + configparser.NoSectionError, + configparser.NoOptionError, + ) as e: if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print( + "Adding sample versioneer config to setup.cfg", file=sys.stderr + ) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -1713,15 +1791,18 @@ def do_setup(): print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: @@ -1763,8 +1844,10 @@ def do_setup(): else: print(" 'versioneer.py' already in MANIFEST.in") if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) + print( + " appending versionfile_source ('%s') to MANIFEST.in" + % cfg.versionfile_source + ) with open(manifest_in, "a") as f: f.write("include %s\n" % cfg.versionfile_source) else: diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 886371f222..a25b1b5b4a 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -25,23 +25,23 @@ project( # language to be enabled here. The test project that is built in scikit-build to verify # various linking options for the python library is hardcoded to build with C, so until # that is fixed we need to keep C. - C - CXX) + C CXX +) option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - OFF) + OFF +) # If the user requested it we attempt to find RAFT. if(FIND_RAFT_CPP) - find_package(raft ${raft_dask_version} REQUIRED) + find_package(raft ${raft_dask_version} REQUIRED) else() set(raft_FOUND OFF) endif() if(NOT raft_FOUND) - # TODO: This will not be necessary once we upgrade to CMake 3.22, which will - # pull in the required languages for the C++ project even if this project - # does not require those languges. + # TODO: This will not be necessary once we upgrade to CMake 3.22, which will pull in the required + # languages for the C++ project even if this project does not require those languges. include(rapids-cuda) rapids_cuda_init_architectures(raft-dask) enable_language(CUDA) @@ -49,9 +49,8 @@ if(NOT raft_FOUND) # rapids_cuda_init_architectures relies on `project` including. include("${CMAKE_PROJECT_raft_dask_INCLUDE}") - # raft-dask doesn't actually use raft libraries, it just needs the headers, so - # we can turn off all library compilation and we don't need to install - # anything here. + # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all + # library compilation and we don't need to install anything here. set(BUILD_TESTS OFF) set(BUILD_BENCH OFF) set(RAFT_COMPILE_LIBRARIES OFF) diff --git a/python/raft-dask/cmake/thirdparty/get_nccl.cmake b/python/raft-dask/cmake/thirdparty/get_nccl.cmake index c2cc17b399..bb4b0e4dae 100644 --- a/python/raft-dask/cmake/thirdparty/get_nccl.cmake +++ b/python/raft-dask/cmake/thirdparty/get_nccl.cmake @@ -1,39 +1,34 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2021-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= include(rapids-find) function(find_and_configure_nccl) - if(TARGET NCCL::NCCL) - return() - endif() + if(TARGET NCCL::NCCL) + return() + endif() - rapids_find_generate_module(NCCL - HEADER_NAMES nccl.h - LIBRARY_NAMES nccl - ) + rapids_find_generate_module( + NCCL + HEADER_NAMES nccl.h + LIBRARY_NAMES nccl + ) - # Currently NCCL has no CMake build-system so we require - # it built and installed on the machine already - rapids_find_package(NCCL REQUIRED) + # Currently NCCL has no CMake build-system so we require it built and installed on the machine + # already + rapids_find_package(NCCL REQUIRED) endfunction() find_and_configure_nccl() - - - - diff --git a/python/raft-dask/raft_dask/_version.py b/python/raft-dask/raft_dask/_version.py index 454b0fe7aa..4eb6056f17 100644 --- a/python/raft-dask/raft_dask/_version.py +++ b/python/raft-dask/raft_dask/_version.py @@ -70,7 +70,7 @@ def decorate(f): def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None ): """Call the given command(s).""" assert isinstance(commands, list) @@ -85,7 +85,7 @@ def run_command( env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), - ) + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -123,7 +123,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return { - "version": dirname[len(parentdir_prefix):], + "version": dirname[len(parentdir_prefix) :], "full-revisionid": None, "dirty": False, "error": None, @@ -193,7 +193,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -210,7 +210,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) return { @@ -264,7 +264,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): "--long", "--match", "%s*" % tag_prefix, - ], + ], cwd=root, ) # --long was added in git-1.5.5 @@ -299,7 +299,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if not mo: # unparseable. Maybe git-describe is misbehaving? pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out + "unable to parse git-describe output: '%s'" % describe_out ) return pieces @@ -314,7 +314,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): tag_prefix, ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 0e1da015fc..77b6695118 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -18,6 +18,5 @@ find_package(ucx REQUIRED) set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft NCCL::NCCL ucx::ucp) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - CXX) + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX +) diff --git a/python/raft-dask/raft_dask/common/__init__.py b/python/raft-dask/raft_dask/common/__init__.py index 8c25cdde90..c8ce695def 100644 --- a/python/raft-dask/raft_dask/common/__init__.py +++ b/python/raft-dask/raft_dask/common/__init__.py @@ -13,22 +13,21 @@ # limitations under the License. # -from .comms import Comms -from .comms import local_handle - -from .comms_utils import inject_comms_on_handle -from .comms_utils import inject_comms_on_handle_coll_only -from .comms_utils import perform_test_comms_allreduce -from .comms_utils import perform_test_comms_send_recv -from .comms_utils import perform_test_comms_device_send_or_recv -from .comms_utils import perform_test_comms_device_sendrecv -from .comms_utils import perform_test_comms_device_multicast_sendrecv -from .comms_utils import perform_test_comms_allgather -from .comms_utils import perform_test_comms_gather -from .comms_utils import perform_test_comms_gatherv -from .comms_utils import perform_test_comms_bcast -from .comms_utils import perform_test_comms_reduce -from .comms_utils import perform_test_comms_reducescatter -from .comms_utils import perform_test_comm_split - +from .comms import Comms, local_handle +from .comms_utils import ( + inject_comms_on_handle, + inject_comms_on_handle_coll_only, + perform_test_comm_split, + perform_test_comms_allgather, + perform_test_comms_allreduce, + perform_test_comms_bcast, + perform_test_comms_device_multicast_sendrecv, + perform_test_comms_device_send_or_recv, + perform_test_comms_device_sendrecv, + perform_test_comms_gather, + perform_test_comms_gatherv, + perform_test_comms_reduce, + perform_test_comms_reducescatter, + perform_test_comms_send_recv, +) from .ucx import UCX diff --git a/python/raft-dask/raft_dask/common/comms.py b/python/raft-dask/raft_dask/common/comms.py index d7260ece9b..0a7fb02da7 100644 --- a/python/raft-dask/raft_dask/common/comms.py +++ b/python/raft-dask/raft_dask/common/comms.py @@ -13,29 +13,28 @@ # limitations under the License. # -from .nccl import nccl -from .ucx import UCX - -from .comms_utils import inject_comms_on_handle -from .comms_utils import inject_comms_on_handle_coll_only - -from .utils import parse_host_port -from pylibraft.common.handle import Handle - -from dask.distributed import get_worker, default_client - -import warnings - import logging import time import uuid +import warnings from collections import OrderedDict +from pylibraft.common.handle import Handle + +from dask.distributed import default_client, get_worker + +from .comms_utils import ( + inject_comms_on_handle, + inject_comms_on_handle_coll_only, +) +from .nccl import nccl +from .ucx import UCX +from .utils import parse_host_port + logger = logging.getLogger(__name__) class Comms: - """ Initializes and manages underlying NCCL and UCX comms handles across the workers of a Dask cluster. It is expected that `init()` will be @@ -45,7 +44,7 @@ class Comms: Examples -------- - .. code-block:: python + .. code-block:: python # The following code block assumes we have wrapped a C++ # function in a Python function called `run_algorithm`, @@ -175,7 +174,6 @@ def init(self, workers=None): Parameters ---------- - workers : Sequence Unique collection of workers for initializing comms. """ @@ -256,7 +254,6 @@ def local_handle(sessionId): Returns ------- - handle : raft.Handle or None """ state = get_raft_comm_state(sessionId, get_worker()) @@ -277,7 +274,6 @@ def get_raft_comm_state(sessionId, state_object=None): Returns ------- - session state : str session state associated with sessionId """ diff --git a/python/raft-dask/raft_dask/common/comms_utils.pyx b/python/raft-dask/raft_dask/common/comms_utils.pyx index 2014284af1..7db04ef455 100644 --- a/python/raft-dask/raft_dask/common/comms_utils.pyx +++ b/python/raft-dask/raft_dask/common/comms_utils.pyx @@ -17,14 +17,12 @@ # cython: embedsignature = True # cython: language_level = 3 -from libc.stdlib cimport malloc, free -from cython.operator cimport dereference as deref - from cpython.long cimport PyLong_AsVoidPtr - +from cython.operator cimport dereference as deref +from libc.stdint cimport uintptr_t +from libc.stdlib cimport free, malloc from libcpp cimport bool -from libc.stdint cimport uintptr_t cdef extern from "nccl.h": diff --git a/python/raft-dask/raft_dask/common/nccl.pyx b/python/raft-dask/raft_dask/common/nccl.pyx index fd113e2222..a4d59610d3 100644 --- a/python/raft-dask/raft_dask/common/nccl.pyx +++ b/python/raft-dask/raft_dask/common/nccl.pyx @@ -19,11 +19,11 @@ # cython: embedsignature = True # cython: language_level = 3 -from libc.stdint cimport uintptr_t from cython.operator cimport dereference as deref - +from libc.stdint cimport uintptr_t +from libc.stdlib cimport free, malloc from libcpp cimport bool -from libc.stdlib cimport malloc, free + cdef extern from "raft/comms/std_comms.hpp" namespace "raft::comms": void get_unique_id(char *uid, int size) except + diff --git a/python/raft-dask/raft_dask/common/utils.py b/python/raft-dask/raft_dask/common/utils.py index daf51530be..78a899aa50 100644 --- a/python/raft-dask/raft_dask/common/utils.py +++ b/python/raft-dask/raft_dask/common/utils.py @@ -32,8 +32,8 @@ def parse_host_port(address): ------- tuple with host and port info : tuple(host, port) """ - if '://' in address: - address = address.rsplit('://', 1)[1] - host, port = address.split(':') + if "://" in address: + address = address.rsplit("://", 1)[1] + host, port = address.split(":") port = int(port) return host, port diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index 5e417f42ee..e0add1e55b 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -15,6 +15,5 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - CXX) + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX +) diff --git a/python/raft-dask/raft_dask/test/conftest.py b/python/raft-dask/raft_dask/test/conftest.py index f5cdc49700..39ee21cbaa 100644 --- a/python/raft-dask/raft_dask/test/conftest.py +++ b/python/raft-dask/raft_dask/test/conftest.py @@ -1,13 +1,12 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +import os + import pytest from dask.distributed import Client +from dask_cuda import LocalCUDACluster, initialize -from dask_cuda import initialize -from dask_cuda import LocalCUDACluster - -import os os.environ["UCX_LOG_LEVEL"] = "error" @@ -25,14 +24,18 @@ def cluster(): @pytest.fixture(scope="session") def ucx_cluster(): - initialize.initialize(create_cuda_context=True, - enable_tcp_over_ucx=enable_tcp_over_ucx, - enable_nvlink=enable_nvlink, - enable_infiniband=enable_infiniband) - cluster = LocalCUDACluster(protocol="ucx", - enable_tcp_over_ucx=enable_tcp_over_ucx, - enable_nvlink=enable_nvlink, - enable_infiniband=enable_infiniband) + initialize.initialize( + create_cuda_context=True, + enable_tcp_over_ucx=enable_tcp_over_ucx, + enable_nvlink=enable_nvlink, + enable_infiniband=enable_infiniband, + ) + cluster = LocalCUDACluster( + protocol="ucx", + enable_tcp_over_ucx=enable_tcp_over_ucx, + enable_nvlink=enable_nvlink, + enable_infiniband=enable_infiniband, + ) yield cluster cluster.close() diff --git a/python/raft-dask/raft_dask/test/test_comms.py b/python/raft-dask/raft_dask/test/test_comms.py index 29b4d963d9..74ec446e94 100644 --- a/python/raft-dask/raft_dask/test/test_comms.py +++ b/python/raft-dask/raft_dask/test/test_comms.py @@ -13,28 +13,29 @@ # limitations under the License. # -import pytest - from collections import OrderedDict -from dask.distributed import Client -from dask.distributed import wait +import pytest + +from dask.distributed import Client, wait try: - from raft_dask import Comms - from raft_dask.common import local_handle - from raft_dask.common import perform_test_comms_send_recv - from raft_dask.common import perform_test_comms_device_send_or_recv - from raft_dask.common import perform_test_comms_device_sendrecv - from raft_dask.common import perform_test_comms_device_multicast_sendrecv - from raft_dask.common import perform_test_comms_allreduce - from raft_dask.common import perform_test_comms_bcast - from raft_dask.common import perform_test_comms_reduce - from raft_dask.common import perform_test_comms_allgather - from raft_dask.common import perform_test_comms_gather - from raft_dask.common import perform_test_comms_gatherv - from raft_dask.common import perform_test_comms_reducescatter - from raft_dask.common import perform_test_comm_split + from raft_dask.common import ( + Comms, + local_handle, + perform_test_comm_split, + perform_test_comms_allgather, + perform_test_comms_allreduce, + perform_test_comms_bcast, + perform_test_comms_device_multicast_sendrecv, + perform_test_comms_device_send_or_recv, + perform_test_comms_device_sendrecv, + perform_test_comms_gather, + perform_test_comms_gatherv, + perform_test_comms_reduce, + perform_test_comms_reducescatter, + perform_test_comms_send_recv, + ) pytestmark = pytest.mark.mg except ImportError: diff --git a/python/raft-dask/raft_dask/test/test_raft.py b/python/raft-dask/raft_dask/test/test_raft.py index e1370dccc9..e1e1358f58 100644 --- a/python/raft-dask/raft_dask/test/test_raft.py +++ b/python/raft-dask/raft_dask/test/test_raft.py @@ -13,9 +13,10 @@ # limitations under the License. # -import pytest import sys +import pytest + try: import raft_dask except ImportError: @@ -23,7 +24,7 @@ pytestmart = pytest.mark.skip pytestmark = pytest.mark.skipif( - 'raft_dask' not in sys.argv, reason="marker to allow integration of RAFT" + "raft_dask" not in sys.argv, reason="marker to allow integration of RAFT" ) diff --git a/python/raft-dask/setup.py b/python/raft-dask/setup.py index 59bcc4a2dc..f5408e1f9e 100644 --- a/python/raft-dask/setup.py +++ b/python/raft-dask/setup.py @@ -14,37 +14,36 @@ # limitations under the License. # +import versioneer from setuptools import find_packages from skbuild import setup -import versioneer - - -setup(name='raft-dask', - description="Reusable Accelerated Functions & Tools Dask Infrastructure", - version=versioneer.get_version(), - classifiers=[ +setup( + name="raft-dask", + description="Reusable Accelerated Functions & Tools Dask Infrastructure", + version=versioneer.get_version(), + classifiers=[ "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9" - ], - author="NVIDIA Corporation", - package_data={ - # Note: A dict comprehension with an explicit copy is necessary - # (rather than something simpler like a dict.fromkeys) because - # otherwise every package will refer to the same list and skbuild - # modifies it in place. - key: ["*.hpp", "*.pxd"] - for key in find_packages( - include=[ - "raft_dask.common", - "raft_dask.common.includes", - ] - ) - }, - packages=find_packages(include=['raft_dask', 'raft_dask.*']), - license="Apache", - cmdclass=versioneer.get_cmdclass(), - zip_safe=False - ) + "Programming Language :: Python :: 3.9", + ], + author="NVIDIA Corporation", + package_data={ + # Note: A dict comprehension with an explicit copy is necessary + # (rather than something simpler like a dict.fromkeys) because + # otherwise every package will refer to the same list and skbuild + # modifies it in place. + key: ["*.hpp", "*.pxd"] + for key in find_packages( + include=[ + "raft_dask.common", + "raft_dask.common.includes", + ] + ) + }, + packages=find_packages(include=["raft_dask", "raft_dask.*"]), + license="Apache", + cmdclass=versioneer.get_cmdclass(), + zip_safe=False, +) diff --git a/python/raft-dask/setuputils.py b/python/raft-dask/setuputils.py index 8893e09fd3..9370d29876 100755 --- a/python/raft-dask/setuputils.py +++ b/python/raft-dask/setuputils.py @@ -51,15 +51,15 @@ def clean_folder(path): path : String Path to the folder to be cleaned. """ - shutil.rmtree(path + '/__pycache__', ignore_errors=True) + shutil.rmtree(path + "/__pycache__", ignore_errors=True) - folders = glob.glob(path + '/*/') + folders = glob.glob(path + "/*/") for folder in folders: - shutil.rmtree(folder + '/__pycache__', ignore_errors=True) + shutil.rmtree(folder + "/__pycache__", ignore_errors=True) clean_folder(folder) - cython_exts = glob.glob(folder + '/*.cpp') - cython_exts.extend(glob.glob(folder + '/*.cpython*')) + cython_exts = glob.glob(folder + "/*.cpp") + cython_exts.extend(glob.glob(folder + "/*.cpython*")) for file in cython_exts: os.remove(file) diff --git a/python/raft-dask/versioneer.py b/python/raft-dask/versioneer.py index b8c4bc423b..73e3259bf1 100644 --- a/python/raft-dask/versioneer.py +++ b/python/raft-dask/versioneer.py @@ -278,10 +278,12 @@ """ from __future__ import print_function + try: import configparser except ImportError: import ConfigParser as configparser + import errno import json import os @@ -309,11 +311,13 @@ def get_root(): setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -326,8 +330,10 @@ def get_root(): me_dir = os.path.normcase(os.path.splitext(me)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py) + ) except NameError: pass return root @@ -349,6 +355,7 @@ def get(parser, name): if parser.has_option("versioneer", name): return parser.get("versioneer", name) return None + cfg = VersioneerConfig() cfg.VCS = VCS cfg.style = get(parser, "style") or "" @@ -373,17 +380,20 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None +): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -391,10 +401,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -419,7 +432,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, p.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY[ + "git" +] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -994,7 +1009,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1003,7 +1018,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1011,19 +1026,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -1038,8 +1060,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command( + GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True + ) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1047,10 +1070,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1073,17 +1105,18 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = ( + "unable to parse git-describe output: '%s'" % describe_out + ) return pieces # tag @@ -1092,10 +1125,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1106,13 +1141,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command( + GITS, ["rev-list", "HEAD", "--count"], cwd=root + ) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1168,16 +1205,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1206,11 +1249,17 @@ def versions_from_file(filename): contents = f.read() except EnvironmentError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1219,8 +1268,9 @@ def versions_from_file(filename): def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps( + versions, sort_keys=True, indent=1, separators=(",", ": ") + ) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1252,8 +1302,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1367,11 +1416,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1391,9 +1442,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1416,8 +1471,9 @@ def get_versions(verbose=False): handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1471,9 +1527,13 @@ def get_versions(verbose=False): if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version(): @@ -1522,6 +1582,7 @@ def run(self): print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in both distutils and setuptools @@ -1554,14 +1615,17 @@ def run(self): # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join( + self.build_lib, cfg.versionfile_build + ) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -1582,17 +1646,21 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.distutils_buildexe import py2exe as _py2exe # py3 except ImportError: @@ -1611,13 +1679,17 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments @@ -1644,8 +1716,10 @@ def make_release_tree(self, base_dir, files): # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -1700,11 +1774,15 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except ( + EnvironmentError, + configparser.NoSectionError, + configparser.NoOptionError, + ) as e: if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print( + "Adding sample versioneer config to setup.cfg", file=sys.stderr + ) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -1713,15 +1791,18 @@ def do_setup(): print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: @@ -1763,8 +1844,10 @@ def do_setup(): else: print(" 'versioneer.py' already in MANIFEST.in") if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) + print( + " appending versionfile_source ('%s') to MANIFEST.in" + % cfg.versionfile_source + ) with open(manifest_in, "a") as f: f.write("include %s\n" % cfg.versionfile_source) else: diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..30b0ef0f7a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,23 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +[pydocstyle] +# Due to https://github.com/PyCQA/pydocstyle/issues/363, we must exclude rather +# than include using match-dir. Note that as discussed in +# https://stackoverflow.com/questions/65478393/how-to-filter-directories-using-the-match-dir-flag-for-pydocstyle, +# unlike the match option above this match-dir will have no effect when +# pydocstyle is invoked from pre-commit. Therefore this exclusion list must +# also be maintained in the pre-commit config file. +match-dir = ^(?!(ci|cpp|conda|docs|java|notebooks)).*$ +# Allow missing docstrings for docutils +ignore-decorators = .*(docutils|doc_apply|copy_docstring).* +select = + D201, D204, D206, D207, D208, D209, D210, D211, D214, D215, D300, D301, D302, D403, D405, D406, D407, D408, D409, D410, D411, D412, D414, D418 + # Would like to enable the following rules in the future: + # D200, D202, D205, D400 + +[mypy] +ignore_missing_imports = True +# If we don't specify this, then mypy will check excluded files if +# they are imported by a checked file. +follow_imports = skip +exclude=_version.py From 4f585504309f6c2af370ceb4508a50eafd81f729 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Fri, 28 Oct 2022 16:47:20 -0700 Subject: [PATCH 2/8] Fix pylibraft import order Add pylibraft to known first party imports for isort --- .../pylibraft/pylibraft/random/rmat_rectangular_generator.pyx | 1 + python/pylibraft/pylibraft/test/test_distance.py | 3 ++- python/pylibraft/pylibraft/test/test_fused_l2_argmin.py | 3 ++- python/pylibraft/pylibraft/test/test_kmeans.py | 1 + python/pylibraft/pylibraft/test/test_random.py | 1 + python/pylibraft/pylibraft/test/test_z_interruptible.py | 1 + python/pylibraft/setup.cfg | 1 + 7 files changed, 9 insertions(+), 2 deletions(-) diff --git a/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx b/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx index 2127abb950..36b0a1ba73 100644 --- a/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx +++ b/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx @@ -26,6 +26,7 @@ from libc.stdint cimport int64_t, uintptr_t from pylibraft.common import Handle from libcpp cimport bool + from pylibraft.common.handle cimport handle_t from .rng_state cimport RngState diff --git a/python/pylibraft/pylibraft/test/test_distance.py b/python/pylibraft/pylibraft/test/test_distance.py index c0cd93d3b5..670beb156e 100644 --- a/python/pylibraft/pylibraft/test/test_distance.py +++ b/python/pylibraft/pylibraft/test/test_distance.py @@ -15,10 +15,11 @@ import numpy as np import pytest +from scipy.spatial.distance import cdist + from pylibraft.common import Handle from pylibraft.distance import pairwise_distance from pylibraft.testing.utils import TestDeviceBuffer -from scipy.spatial.distance import cdist @pytest.mark.parametrize("n_rows", [100]) diff --git a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py b/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py index 2e760d4411..abe56f2b04 100644 --- a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py +++ b/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py @@ -15,10 +15,11 @@ import numpy as np import pytest +from scipy.spatial.distance import cdist + from pylibraft.common import Handle from pylibraft.distance import fused_l2_nn_argmin from pylibraft.testing.utils import TestDeviceBuffer -from scipy.spatial.distance import cdist @pytest.mark.parametrize("n_rows", [10, 100]) diff --git a/python/pylibraft/pylibraft/test/test_kmeans.py b/python/pylibraft/pylibraft/test/test_kmeans.py index b2dec5cb68..d198ac2f8f 100644 --- a/python/pylibraft/pylibraft/test/test_kmeans.py +++ b/python/pylibraft/pylibraft/test/test_kmeans.py @@ -15,6 +15,7 @@ import numpy as np import pytest + from pylibraft.cluster.kmeans import compute_new_centroids from pylibraft.common import Handle from pylibraft.distance import pairwise_distance diff --git a/python/pylibraft/pylibraft/test/test_random.py b/python/pylibraft/pylibraft/test/test_random.py index f6b338225b..77494ea277 100644 --- a/python/pylibraft/pylibraft/test/test_random.py +++ b/python/pylibraft/pylibraft/test/test_random.py @@ -15,6 +15,7 @@ import numpy as np import pytest + from pylibraft.common import Handle from pylibraft.random import rmat from pylibraft.testing.utils import TestDeviceBuffer diff --git a/python/pylibraft/pylibraft/test/test_z_interruptible.py b/python/pylibraft/pylibraft/test/test_z_interruptible.py index adaac623ea..aac428b2d5 100644 --- a/python/pylibraft/pylibraft/test/test_z_interruptible.py +++ b/python/pylibraft/pylibraft/test/test_z_interruptible.py @@ -5,6 +5,7 @@ import time import pytest + from pylibraft.common.interruptible import cuda_interruptible, cuda_yield diff --git a/python/pylibraft/setup.cfg b/python/pylibraft/setup.cfg index e1f4865ac9..630a597f3c 100644 --- a/python/pylibraft/setup.cfg +++ b/python/pylibraft/setup.cfg @@ -34,6 +34,7 @@ known_rapids= rmm known_first_party= raft + pylibraft default_section=THIRDPARTY sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER skip= From 319e32c65a5016b3e07090f0849eba0b66cd934c Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Mon, 7 Nov 2022 12:56:43 -0800 Subject: [PATCH 3/8] isort again --- python/pylibraft/pylibraft/common/handle.pyx | 3 +-- .../pylibraft/pylibraft/random/rmat_rectangular_generator.pyx | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/python/pylibraft/pylibraft/common/handle.pyx b/python/pylibraft/pylibraft/common/handle.pyx index ba735f5536..c14c22f5aa 100644 --- a/python/pylibraft/pylibraft/common/handle.pyx +++ b/python/pylibraft/pylibraft/common/handle.pyx @@ -21,8 +21,7 @@ import functools -from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread, cuda_stream_view from .cuda cimport Stream diff --git a/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx b/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx index 588f1a05c2..3651451f51 100644 --- a/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx +++ b/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx @@ -25,8 +25,6 @@ from libc.stdint cimport int64_t, uintptr_t from pylibraft.common import Handle from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.handle cimport handle_t -from .rng_state cimport RngState from libcpp cimport bool From a65aefd6a610f931abe3e198dea7813ea8746612 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Mon, 7 Nov 2022 13:07:54 -0800 Subject: [PATCH 4/8] Code review feedback --- CONTRIBUTING.md | 37 +++++++++++++++++++++++++++++++++ cpp/scripts/run-cmake-format.sh | 15 +++++++------ pyproject.toml | 2 +- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index faf777ba42..50fa034572 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,43 @@ into three categories: Remember, if you are unsure about anything, don't hesitate to comment on issues and ask for clarifications! + +### Python / Pre-commit hooks + +RAFT uses [pre-commit](https://pre-commit.com/) to execute code linters and formatters such as +[Black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/), and +[flake8](https://flake8.pycqa.org/en/latest/). These tools ensure a consistent code format +throughout the project. Using pre-commit ensures that linter versions and options are aligned for +all developers. Additionally, there is a CI check in place to enforce that committed code follows +our standards. + +To use `pre-commit`, install via `conda` or `pip`: + +```bash +conda install -c conda-forge pre-commit +``` + +```bash +pip install pre-commit +``` + +Then run pre-commit hooks before committing code: + +```bash +pre-commit run +``` + +Optionally, you may set up the pre-commit hooks to run automatically when you make a git commit. This can be done by running: + +```bash +pre-commit install +``` + +Now code linters and formatters will be run each time you commit changes. + +You can skip these checks with `git commit --no-verify` or with the short version `git commit -n`. + + ### Seasoned developers Once you have gotten your feet wet and are more comfortable with the code, you diff --git a/cpp/scripts/run-cmake-format.sh b/cpp/scripts/run-cmake-format.sh index afe6e47c8e..db5a8b5804 100755 --- a/cpp/scripts/run-cmake-format.sh +++ b/cpp/scripts/run-cmake-format.sh @@ -26,26 +26,25 @@ # bash run-cmake-format.sh {cmake-format,cmake-lint} infile [infile ...] status=0 -if [ -z ${CUDF_ROOT:+PLACEHOLDER} ]; then - CUDF_BUILD_DIR=$(git rev-parse --show-toplevel 2>&1)/cpp/build +if [ -z ${RAFT_ROOT:+PLACEHOLDER} ]; then + RAFT_BUILD_DIR=$(git rev-parse --show-toplevel 2>&1)/cpp/build status=$? else - CUDF_BUILD_DIR=${CUDF_ROOT} + RAFT_BUILD_DIR=${RAFT_ROOT} fi if ! [ ${status} -eq 0 ]; then - if [[ ${CUDF_BUILD_DIR} == *"not a git repository"* ]]; then - echo "This script must be run inside the cudf repository, or the CUDF_ROOT environment variable must be set." + if [[ ${RAFT_BUILD_DIR} == *"not a git repository"* ]]; then + echo "This script must be run inside the raft repository, or the RAFT_ROOT environment variable must be set." else echo "Script failed with unknown error attempting to determine project root:" - echo ${CUDF_BUILD_DIR} + echo ${RAFT_BUILD_DIR} fi exit 1 fi DEFAULT_FORMAT_FILE_LOCATIONS=( - "${CUDF_BUILD_DIR:-${HOME}}/_deps/rapids-cmake-src/cmake-format-rapids-cmake.json" - "cpp/libcudf_kafka/build/_deps/rapids-cmake-src/cmake-format-rapids-cmake.json" + "${RAFT_BUILD_DIR:-${HOME}}/_deps/rapids-cmake-src/cmake-format-rapids-cmake.json" ) if [ -z ${RAPIDS_CMAKE_FORMAT_FILE:+PLACEHOLDER} ]; then diff --git a/pyproject.toml b/pyproject.toml index a756854eae..dfd22f3378 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ line-length = 79 target-version = ["py38"] include = '\.py?$' -exclude = ''' +force-exclude = ''' /( thirdparty | \.eggs | From 2572ce153371611ff28d4354469d2429a087448c Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Mon, 7 Nov 2022 13:18:35 -0800 Subject: [PATCH 5/8] fix cmake format --- cpp/test/CMakeLists.txt | 18 +++++++++--------- python/pylibraft/CMakeLists.txt | 6 +++--- .../pylibraft/pylibraft/common/CMakeLists.txt | 9 ++++----- .../pylibraft/distance/CMakeLists.txt | 9 ++++----- .../pylibraft/pylibraft/random/CMakeLists.txt | 9 ++++----- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 0b479dc6c4..e737034161 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -43,15 +43,15 @@ function(ConfigureTest) add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) - set_target_properties(${TEST_NAME} - PROPERTIES - # set target compile options - INSTALL_RPATH "\$ORIGIN/../../../lib" - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - ) + set_target_properties( + ${TEST_NAME} + PROPERTIES # set target compile options + INSTALL_RPATH "\$ORIGIN/../../../lib" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + ) target_compile_options( ${TEST_NAME} PRIVATE "$<$:${RAFT_CXX_FLAGS}>" diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 19e9a5aba2..faea60bdb1 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -64,9 +64,9 @@ if(NOT raft_FOUND) set(RAFT_COMPILE_DIST_LIBRARY ON) add_subdirectory(../../cpp raft-cpp) - # When building the C++ libraries from source we must copy - # libraft_distance.so alongside the pairwise_distance and random Cython libraries - # TODO: when we have a single 'compiled' raft library, we shouldn't need this + # When building the C++ libraries from source we must copy libraft_distance.so alongside the + # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft + # library, we shouldn't need this set(cython_lib_dir pylibraft) install(TARGETS raft_distance_lib DESTINATION ${cython_lib_dir}) endif() diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index f8d86e0386..3b49cef429 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -18,8 +18,7 @@ set(linked_libraries raft::raft) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - ASSOCIATED_TARGETS raft - MODULE_PREFIX common_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ +) diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index 9247372984..cae00007d6 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -18,8 +18,7 @@ set(linked_libraries raft::raft raft::distance) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - ASSOCIATED_TARGETS raft - MODULE_PREFIX distance_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ +) diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index ca46f6507e..f6789cc894 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -20,8 +20,7 @@ set(linked_libraries raft::raft raft::distance) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - ASSOCIATED_TARGETS raft - MODULE_PREFIX random_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ +) From 6ca41a395090557387f0e6a94a781fefe2e91809 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Mon, 7 Nov 2022 14:32:02 -0800 Subject: [PATCH 6/8] use cudf flake8 config --- .pre-commit-config.yaml | 2 +- ci/checks/copyright.py | 1 - python/.flake8 | 9 --------- python/.flake8.cython | 28 ---------------------------- python/pylibraft/setup.cfg | 6 ------ python/raft-dask/setup.cfg | 6 ------ setup.cfg | 23 +++++++++++++++++++++++ 7 files changed, 24 insertions(+), 51 deletions(-) delete mode 100644 python/.flake8 delete mode 100644 python/.flake8.cython diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 429219b601..336c218f1d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: rev: 5.0.4 hooks: - id: flake8 - args: ["--config=python/.flake8.cython"] + args: ["--config=setup.cfg"] files: python/.*$ types: [file] types_or: [python, cython] diff --git a/ci/checks/copyright.py b/ci/checks/copyright.py index 6b808cc051..ab382e3b94 100644 --- a/ci/checks/copyright.py +++ b/ci/checks/copyright.py @@ -35,7 +35,6 @@ re.compile(r"CMakeLists[.]txt$"), re.compile(r"CMakeLists_standalone[.]txt$"), re.compile(r"setup[.]cfg$"), - re.compile(r"[.]flake8[.]cython$"), re.compile(r"meta[.]yaml$") ] ExemptFiles = [] diff --git a/python/.flake8 b/python/.flake8 deleted file mode 100644 index ef2e5a8495..0000000000 --- a/python/.flake8 +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -[flake8] -exclude = __init__.py -ignore = - # line break before binary operator - W503 - # whitespace before : - E203 \ No newline at end of file diff --git a/python/.flake8.cython b/python/.flake8.cython deleted file mode 100644 index 9dde36fe8a..0000000000 --- a/python/.flake8.cython +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -[flake8] -filename = *.pyx, *.pxd -exclude = *.egg, build, docs, .git, __init__.py, cpp, thirdparty, versioneer.py, _version.py -ignore = E999, E225, E226, E227, W503, W504 - -# Rules ignored: -# E999: invalid syntax (works for Python, not Cython) -# E225: Missing whitespace around operators (breaks cython casting syntax like ) -# E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) -# E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) -# W503: line break before binary operator (breaks lines that start with a pointer) -# W504: line break after binary operator (breaks lines that end with a pointer) diff --git a/python/pylibraft/setup.cfg b/python/pylibraft/setup.cfg index 630a597f3c..67cc148155 100644 --- a/python/pylibraft/setup.cfg +++ b/python/pylibraft/setup.cfg @@ -1,11 +1,5 @@ # Copyright (c) 2022, NVIDIA CORPORATION. -[flake8] -exclude = __init__.py,versioneer.py -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - [versioneer] VCS = git style = pep440 diff --git a/python/raft-dask/setup.cfg b/python/raft-dask/setup.cfg index c749a5a541..b005a7ab8f 100644 --- a/python/raft-dask/setup.cfg +++ b/python/raft-dask/setup.cfg @@ -1,11 +1,5 @@ # Copyright (c) 2020-2022, NVIDIA CORPORATION. -[flake8] -exclude = __init__.py,versioneer.py -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - [versioneer] VCS = git style = pep440 diff --git a/setup.cfg b/setup.cfg index 30b0ef0f7a..87cc11660c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,28 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +[flake8] +filename = *.py, *.pyx, *.pxd, *.pxi +exclude = __init__.py, *.egg, build, docs, .git, versioneer.py +force-check = True +ignore = + # line break before binary operator + W503, + # whitespace before : + E203 +per-file-ignores = + # Rules ignored only in Cython: + # E211: whitespace before '(' (used in multi-line imports) + # E225: Missing whitespace around operators (breaks cython casting syntax like ) + # E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) + # E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) + # E275: Missing whitespace after keyword (Doesn't work with Cython except?) + # E402: invalid syntax (works for Python, not Cython) + # E999: invalid syntax (works for Python, not Cython) + # W504: line break after binary operator (breaks lines that end with a pointer) + *.pyx: E211, E225, E226, E227, E275, E402, E999, W504 + *.pxd: E211, E225, E226, E227, E275, E402, E999, W504 + *.pxi: E211, E225, E226, E227, E275, E402, E999, W504 + [pydocstyle] # Due to https://github.com/PyCQA/pydocstyle/issues/363, we must exclude rather # than include using match-dir. Note that as discussed in From 16c16326512bc6722ad5258991eca1cf2ee9ad3f Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Wed, 9 Nov 2022 09:28:27 -0800 Subject: [PATCH 7/8] Re-apply cmake format to upstream changes --- cpp/bench/CMakeLists.txt | 127 +++++++++++++++++++-------------------- cpp/test/CMakeLists.txt | 62 ++++++++++--------- 2 files changed, 94 insertions(+), 95 deletions(-) diff --git a/cpp/bench/CMakeLists.txt b/cpp/bench/CMakeLists.txt index e1678a8d60..3c3d296b06 100644 --- a/cpp/bench/CMakeLists.txt +++ b/cpp/bench/CMakeLists.txt @@ -67,70 +67,65 @@ function(ConfigureBench) endfunction() if(BUILD_BENCH) - ConfigureBench(NAME CLUSTER_BENCH - PATH - bench/cluster/kmeans_balanced.cu - bench/cluster/kmeans.cu - bench/main.cpp - OPTIONAL DIST NN - ) - - ConfigureBench(NAME DISTANCE_BENCH - PATH - bench/distance/distance_cosine.cu - bench/distance/distance_exp_l2.cu - bench/distance/distance_l1.cu - bench/distance/distance_unexp_l2.cu - bench/distance/fused_l2_nn.cu - bench/distance/kernels.cu - bench/main.cpp - OPTIONAL DIST - ) - - ConfigureBench(NAME LINALG_BENCH - PATH - bench/linalg/add.cu - bench/linalg/map_then_reduce.cu - bench/linalg/matrix_vector_op.cu - bench/linalg/reduce_rows_by_key.cu - bench/linalg/reduce.cu - bench/main.cpp - ) - - ConfigureBench(NAME MATRIX_BENCH - PATH - bench/matrix/argmin.cu - bench/main.cpp - ) - - ConfigureBench(NAME RANDOM_BENCH - PATH - bench/random/make_blobs.cu - bench/random/permute.cu - bench/random/rng.cu - bench/main.cpp - ) - - ConfigureBench(NAME SPARSE_BENCH - PATH - bench/sparse/convert_csr.cu - bench/main.cpp - ) - - ConfigureBench(NAME NEIGHBORS_BENCH - PATH - bench/neighbors/knn/brute_force_float_int64_t.cu - bench/neighbors/knn/brute_force_float_uint32_t.cu - bench/neighbors/knn/ivf_flat_float_int64_t.cu - bench/neighbors/knn/ivf_flat_float_uint32_t.cu - bench/neighbors/knn/ivf_flat_int8_t_int64_t.cu - bench/neighbors/knn/ivf_flat_uint8_t_uint32_t.cu - bench/neighbors/knn/ivf_pq_float_int64_t.cu - bench/neighbors/knn/ivf_pq_float_uint32_t.cu - bench/neighbors/knn/ivf_pq_int8_t_int64_t.cu - bench/neighbors/knn/ivf_pq_uint8_t_uint32_t.cu - bench/neighbors/selection.cu - bench/main.cpp - OPTIONAL DIST NN - ) + ConfigureBench( + NAME CLUSTER_BENCH PATH bench/cluster/kmeans_balanced.cu bench/cluster/kmeans.cu bench/main.cpp + OPTIONAL DIST NN + ) + + ConfigureBench( + NAME + DISTANCE_BENCH + PATH + bench/distance/distance_cosine.cu + bench/distance/distance_exp_l2.cu + bench/distance/distance_l1.cu + bench/distance/distance_unexp_l2.cu + bench/distance/fused_l2_nn.cu + bench/distance/kernels.cu + bench/main.cpp + OPTIONAL + DIST + ) + + ConfigureBench( + NAME + LINALG_BENCH + PATH + bench/linalg/add.cu + bench/linalg/map_then_reduce.cu + bench/linalg/matrix_vector_op.cu + bench/linalg/reduce_rows_by_key.cu + bench/linalg/reduce.cu + bench/main.cpp + ) + + ConfigureBench(NAME MATRIX_BENCH PATH bench/matrix/argmin.cu bench/main.cpp) + + ConfigureBench( + NAME RANDOM_BENCH PATH bench/random/make_blobs.cu bench/random/permute.cu bench/random/rng.cu + bench/main.cpp + ) + + ConfigureBench(NAME SPARSE_BENCH PATH bench/sparse/convert_csr.cu bench/main.cpp) + + ConfigureBench( + NAME + NEIGHBORS_BENCH + PATH + bench/neighbors/knn/brute_force_float_int64_t.cu + bench/neighbors/knn/brute_force_float_uint32_t.cu + bench/neighbors/knn/ivf_flat_float_int64_t.cu + bench/neighbors/knn/ivf_flat_float_uint32_t.cu + bench/neighbors/knn/ivf_flat_int8_t_int64_t.cu + bench/neighbors/knn/ivf_flat_uint8_t_uint32_t.cu + bench/neighbors/knn/ivf_pq_float_int64_t.cu + bench/neighbors/knn/ivf_pq_float_uint32_t.cu + bench/neighbors/knn/ivf_pq_int8_t_int64_t.cu + bench/neighbors/knn/ivf_pq_uint8_t_uint32_t.cu + bench/neighbors/selection.cu + bench/main.cpp + OPTIONAL + DIST + NN + ) endif() diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 336d883c06..caebf6bd11 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -81,19 +81,21 @@ if(BUILD_TESTS) OPTIONAL DIST NN ) - ConfigureTest(NAME CORE_TEST - PATH - test/common/logger.cpp - test/handle.cpp - test/interruptible.cu - test/nvtx.cpp - test/mdarray.cu - test/mdspan_utils.cu - test/memory_type.cpp - test/span.cpp - test/span.cu - test/test.cpp - ) + ConfigureTest( + NAME + CORE_TEST + PATH + test/common/logger.cpp + test/handle.cpp + test/interruptible.cu + test/nvtx.cpp + test/mdarray.cu + test/mdspan_utils.cu + test/memory_type.cpp + test/span.cpp + test/span.cu + test/test.cpp + ) ConfigureTest( NAME @@ -156,22 +158,24 @@ if(BUILD_TESTS) test/linalg/unary_op.cu ) - ConfigureTest(NAME MATRIX_TEST - PATH - test/matrix/argmax.cu - test/matrix/argmin.cu - test/matrix/columnSort.cu - test/matrix/diagonal.cu - test/matrix/gather.cu - test/matrix/linewise_op.cu - test/matrix/math.cu - test/matrix/matrix.cu - test/matrix/norm.cu - test/matrix/reverse.cu - test/matrix/slice.cu - test/matrix/triangular.cu - test/spectral_matrix.cu - ) + ConfigureTest( + NAME + MATRIX_TEST + PATH + test/matrix/argmax.cu + test/matrix/argmin.cu + test/matrix/columnSort.cu + test/matrix/diagonal.cu + test/matrix/gather.cu + test/matrix/linewise_op.cu + test/matrix/math.cu + test/matrix/matrix.cu + test/matrix/norm.cu + test/matrix/reverse.cu + test/matrix/slice.cu + test/matrix/triangular.cu + test/spectral_matrix.cu + ) ConfigureTest( NAME From 1a6e0115b2c40ef4a66641af4f8a1d000e3dd2a0 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Thu, 10 Nov 2022 09:27:02 -0800 Subject: [PATCH 8/8] re-apply cmake format to cpp/test/CMakeLists.txt --- cpp/test/CMakeLists.txt | 68 +++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 18a9b3b820..66067e4dfd 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -123,39 +123,41 @@ if(BUILD_TESTS) ConfigureTest(NAME LABEL_TEST PATH test/label/label.cu test/label/merge_labels.cu) - ConfigureTest(NAME LINALG_TEST - PATH - test/linalg/add.cu - test/linalg/axpy.cu - test/linalg/binary_op.cu - test/linalg/cholesky_r1.cu - test/linalg/coalesced_reduction.cu - test/linalg/divide.cu - test/linalg/dot.cu - test/linalg/eig.cu - test/linalg/eig_sel.cu - test/linalg/gemm_layout.cu - test/linalg/gemv.cu - test/linalg/map.cu - test/linalg/map_then_reduce.cu - test/linalg/matrix_vector.cu - test/linalg/matrix_vector_op.cu - test/linalg/mean_squared_error.cu - test/linalg/multiply.cu - test/linalg/norm.cu - test/linalg/power.cu - test/linalg/reduce.cu - test/linalg/reduce_cols_by_key.cu - test/linalg/reduce_rows_by_key.cu - test/linalg/rsvd.cu - test/linalg/sqrt.cu - test/linalg/strided_reduction.cu - test/linalg/subtract.cu - test/linalg/svd.cu - test/linalg/ternary_op.cu - test/linalg/transpose.cu - test/linalg/unary_op.cu - ) + ConfigureTest( + NAME + LINALG_TEST + PATH + test/linalg/add.cu + test/linalg/axpy.cu + test/linalg/binary_op.cu + test/linalg/cholesky_r1.cu + test/linalg/coalesced_reduction.cu + test/linalg/divide.cu + test/linalg/dot.cu + test/linalg/eig.cu + test/linalg/eig_sel.cu + test/linalg/gemm_layout.cu + test/linalg/gemv.cu + test/linalg/map.cu + test/linalg/map_then_reduce.cu + test/linalg/matrix_vector.cu + test/linalg/matrix_vector_op.cu + test/linalg/mean_squared_error.cu + test/linalg/multiply.cu + test/linalg/norm.cu + test/linalg/power.cu + test/linalg/reduce.cu + test/linalg/reduce_cols_by_key.cu + test/linalg/reduce_rows_by_key.cu + test/linalg/rsvd.cu + test/linalg/sqrt.cu + test/linalg/strided_reduction.cu + test/linalg/subtract.cu + test/linalg/svd.cu + test/linalg/ternary_op.cu + test/linalg/transpose.cu + test/linalg/unary_op.cu + ) ConfigureTest( NAME