diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..336c218f1d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,98 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +repos: + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + # Use the config file specific to each subproject so that each + # project can specify its own first/third-party packages. + args: ["--config-root=python/", "--resolve-all-configs"] + files: python/.* + types_or: [python, cython, pyi] + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + files: python/.* + # Explicitly specify the pyproject.toml at the repo root, not per-project. + args: ["--config", "pyproject.toml"] + exclude: .*_version.py,.*versioneer.py + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + args: ["--config=setup.cfg"] + files: python/.*$ + types: [file] + types_or: [python, cython] + additional_dependencies: ["flake8-force"] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.971' + hooks: + - id: mypy + additional_dependencies: [types-cachetools] + args: ["--config-file=setup.cfg", + "python/pylibraft/pylibraft", + "python/raft-dask/raft_dask"] + pass_filenames: false + exclude: .*_version.py + - repo: https://github.com/PyCQA/pydocstyle + rev: 6.1.1 + hooks: + - id: pydocstyle + args: ["--config=setup.cfg"] + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v11.1.0 + hooks: + - id: clang-format + types_or: [c, c++, cuda] + args: ["-fallback-style=none", "-style=file", "-i"] + exclude: cpp/include/raft/thirdparty/.* + - repo: local + hooks: + - id: no-deprecationwarning + name: no-deprecationwarning + description: 'Enforce that DeprecationWarning is not introduced (use FutureWarning instead)' + entry: '(category=|\s)DeprecationWarning[,)]' + language: pygrep + types_or: [python, cython] + - id: cmake-format + name: cmake-format + entry: ./cpp/scripts/run-cmake-format.sh cmake-format + language: python + types: [cmake] + exclude: .*/thirdparty/.* + # Note that pre-commit autoupdate does not update the versions + # of dependencies, so we'll have to update this manually. + additional_dependencies: + - cmakelang==0.6.13 + verbose: true + require_serial: true + - id: cmake-lint + name: cmake-lint + entry: ./cpp/scripts/run-cmake-format.sh cmake-lint + language: python + types: [cmake] + # Note that pre-commit autoupdate does not update the versions + # of dependencies, so we'll have to update this manually. + additional_dependencies: + - cmakelang==0.6.13 + verbose: true + require_serial: true + exclude: .*/thirdparty/.* + - id: copyright-check + name: copyright-check + entry: python ./ci/checks/copyright.py --git-modified-only --update-current-year + language: python + pass_filenames: false + additional_dependencies: [gitpython] + - id: include-check + name: include-check + entry: python ./cpp/scripts/include_checker.py cpp/bench cpp/include cpp/test + pass_filenames: false + language: python + additional_dependencies: [gitpython] + +default_language_version: + python: python3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index faf777ba42..50fa034572 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,43 @@ into three categories: Remember, if you are unsure about anything, don't hesitate to comment on issues and ask for clarifications! + +### Python / Pre-commit hooks + +RAFT uses [pre-commit](https://pre-commit.com/) to execute code linters and formatters such as +[Black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/), and +[flake8](https://flake8.pycqa.org/en/latest/). These tools ensure a consistent code format +throughout the project. Using pre-commit ensures that linter versions and options are aligned for +all developers. Additionally, there is a CI check in place to enforce that committed code follows +our standards. + +To use `pre-commit`, install via `conda` or `pip`: + +```bash +conda install -c conda-forge pre-commit +``` + +```bash +pip install pre-commit +``` + +Then run pre-commit hooks before committing code: + +```bash +pre-commit run +``` + +Optionally, you may set up the pre-commit hooks to run automatically when you make a git commit. This can be done by running: + +```bash +pre-commit install +``` + +Now code linters and formatters will be run each time you commit changes. + +You can skip these checks with `git commit --no-verify` or with the short version `git commit -n`. + + ### Seasoned developers Once you have gotten your feet wet and are more comfortable with the code, you diff --git a/ci/checks/copyright.py b/ci/checks/copyright.py index 6b808cc051..ab382e3b94 100644 --- a/ci/checks/copyright.py +++ b/ci/checks/copyright.py @@ -35,7 +35,6 @@ re.compile(r"CMakeLists[.]txt$"), re.compile(r"CMakeLists_standalone[.]txt$"), re.compile(r"setup[.]cfg$"), - re.compile(r"[.]flake8[.]cython$"), re.compile(r"meta[.]yaml$") ] ExemptFiles = [] diff --git a/ci/checks/style.sh b/ci/checks/style.sh index fb5a64fdac..f8fcbe19f8 100644 --- a/ci/checks/style.sh +++ b/ci/checks/style.sh @@ -12,69 +12,12 @@ PATH=/opt/conda/bin:$PATH . /opt/conda/etc/profile.d/conda.sh conda activate rapids -# Run flake8 and get results/return code -FLAKE=`flake8 --exclude=cpp,thirdparty,__init__.py,versioneer.py && flake8 --config=python/.flake8.cython` -RETVAL=$? +FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.12/cmake-format-rapids-cmake.json +export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json +mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) +wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} -# Output results if failure otherwise show pass -if [ "$FLAKE" != "" ]; then - echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" - echo -e "$FLAKE" - echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: flake8 style check\n\n" -fi - -# Check for copyright headers in the files modified currently -COPYRIGHT=`python ci/checks/copyright.py --git-modified-only 2>&1` -CR_RETVAL=$? -if [ "$RETVAL" = "0" ]; then - RETVAL=$CR_RETVAL -fi - -# Output results if failure otherwise show pass -if [ "$CR_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: copyright check; begin output\n\n" - echo -e "$COPYRIGHT" - echo -e "\n\n>>>> FAILED: copyright check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: copyright check\n\n" -fi - -# Check for a consistent #include syntax -HASH_INCLUDE=`python cpp/scripts/include_checker.py \ - cpp/bench \ - cpp/include \ - cpp/test \ - 2>&1` -HASH_RETVAL=$? -if [ "$RETVAL" = "0" ]; then - RETVAL=$HASH_RETVAL -fi - -# Output results if failure otherwise show pass -if [ "$HASH_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: #include check; begin output\n\n" - echo -e "$HASH_INCLUDE" - echo -e "\n\n>>>> FAILED: #include check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: #include check\n\n" -fi - -# Check for a consistent code format -FORMAT=`python cpp/scripts/run-clang-format.py 2>&1` -FORMAT_RETVAL=$? -if [ "$RETVAL" = "0" ]; then - RETVAL=$FORMAT_RETVAL -fi - -# Output results if failure otherwise show pass -if [ "$FORMAT_RETVAL" != "0" ]; then - echo -e "\n\n>>>> FAILED: clang format check; begin output\n\n" - echo -e "$FORMAT" - echo -e "\n\n>>>> FAILED: clang format check; end output\n\n" -else - echo -e "\n\n>>>> PASSED: clang format check\n\n" -fi +# Run pre-commit checks +pre-commit run --hook-stage manual --all-files exit $RETVAL diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 552ee8db4c..6f0afa4eb0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,18 +1,16 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2020-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= set(RAPIDS_VERSION "22.12") set(RAFT_VERSION "22.12.00") @@ -26,7 +24,11 @@ include(rapids-find) rapids_cuda_init_architectures(RAFT) -project(RAFT VERSION ${RAFT_VERSION} LANGUAGES CXX CUDA) +project( + RAFT + VERSION ${RAFT_VERSION} + LANGUAGES CXX CUDA +) # Needed because GoogleBenchmark changes the state of FindThreads.cmake, causing subsequent runs to # have different values for the `Threads::Threads` target. Setting this flag ensures @@ -36,8 +38,8 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) # Write the version header rapids_cmake_write_version_file(include/raft/version_config.hpp) -############################################################################## -# - build type --------------------------------------------------------------- +# ################################################################################################## +# * build type --------------------------------------------------------------- # Set a default build type if none was specified rapids_cmake_build_type(Release) @@ -45,14 +47,16 @@ rapids_cmake_build_type(Release) # this is needed for clang-tidy runs set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -############################################################################## -# - User Options ------------------------------------------------------------ +# ################################################################################################## +# * User Options ------------------------------------------------------------ option(BUILD_SHARED_LIBS "Build raft shared libraries" ON) option(BUILD_TESTS "Build raft unit-tests" ON) option(BUILD_BENCH "Build raft C++ benchmark tests" OFF) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) -option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) +option(CUDA_ENABLE_LINEINFO + "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF +) option(CUDA_STATIC_RUNTIME "Statically link the CUDA toolkit runtime and libraries" OFF) option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) option(DISABLE_DEPRECATION_WARNINGS "Disable deprecaction warnings " ON) @@ -60,9 +64,16 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF) option(RAFT_NVTX "Enable nvtx markers" OFF) option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ${BUILD_TESTS}) -option(RAFT_COMPILE_NN_LIBRARY "Enable building raft nearest neighbors shared library instantiations" ${RAFT_COMPILE_LIBRARIES}) -option(RAFT_COMPILE_DIST_LIBRARY "Enable building raft distant shared library instantiations" ${RAFT_COMPILE_LIBRARIES}) -option(RAFT_ENABLE_NN_DEPENDENCIES "Search for raft::nn dependencies like faiss" ${RAFT_COMPILE_LIBRARIES}) +option( + RAFT_COMPILE_NN_LIBRARY "Enable building raft nearest neighbors shared library instantiations" + ${RAFT_COMPILE_LIBRARIES} +) +option(RAFT_COMPILE_DIST_LIBRARY "Enable building raft distant shared library instantiations" + ${RAFT_COMPILE_LIBRARIES} +) +option(RAFT_ENABLE_NN_DEPENDENCIES "Search for raft::nn dependencies like faiss" + ${RAFT_COMPILE_LIBRARIES} +) option(RAFT_ENABLE_thrust_DEPENDENCY "Enable Thrust dependency" ON) @@ -74,7 +85,13 @@ endif() option(RAFT_EXCLUDE_FAISS_FROM_ALL "Exclude FAISS targets from RAFT's 'all' target" ON) include(CMakeDependentOption) -cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_COMPILE_LIBRARIES OFF) +cmake_dependent_option( + RAFT_USE_FAISS_STATIC + "Build and statically link the FAISS library for nearest neighbors search on GPU" + ON + RAFT_COMPILE_LIBRARIES + OFF +) message(VERBOSE "RAFT: Building optional components: ${raft_FIND_COMPONENTS}") message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") @@ -85,35 +102,42 @@ message(VERBOSE "RAFT: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${RAFT_NVTX}") -message(VERBOSE "RAFT: Statically link the CUDA toolkit runtime and libraries: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE + "RAFT: Statically link the CUDA toolkit runtime and libraries: ${CUDA_STATIC_RUNTIME}" +) # Set RMM logging level -set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") -set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") +set(RMM_LOGGING_LEVEL + "INFO" + CACHE STRING "Choose the logging level." +) +set_property( + CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF" +) message(VERBOSE "RAFT: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") -############################################################################## -# - Conda environment detection ---------------------------------------------- +# ################################################################################################## +# * Conda environment detection ---------------------------------------------- if(DETECT_CONDA_ENV) - rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) - if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) - message(STATUS "RAFT: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH) + if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) + message( + STATUS "RAFT: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}" + ) set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") endif() endif() -############################################################################## -# - compiler options --------------------------------------------------------- +# ################################################################################################## +# * compiler options --------------------------------------------------------- set(_ctk_static_suffix "") if(CUDA_STATIC_RUNTIME) - # If we're statically linking CTK cuBLAS, - # we also want to statically link BLAS + # If we're statically linking CTK cuBLAS, we also want to statically link BLAS set(BLA_STATIC ON) set(_ctk_static_suffix "_static") - # Control legacy FindCUDA.cmake behavior too - # Remove this after we push it into rapids-cmake: + # Control legacy FindCUDA.cmake behavior too Remove this after we push it into rapids-cmake: # https://github.com/rapidsai/rapids-cmake/pull/259 set(CUDA_USE_STATIC_CUDA_RUNTIME ON) endif() @@ -121,7 +145,7 @@ endif() # CUDA runtime rapids_cuda_init_runtime(USE_STATIC ${CUDA_STATIC_RUNTIME}) -if (NOT DISABLE_OPENMP) +if(NOT DISABLE_OPENMP) find_package(OpenMP) if(OPENMP_FOUND) message(VERBOSE "RAFT: OpenMP found in ${OpenMP_CXX_INCLUDE_DIRS}") @@ -132,13 +156,15 @@ endif() # * determine GPU architectures # * enable the CMake CUDA language # * set other CUDA compilation flags -rapids_find_package(CUDAToolkit REQUIRED - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports) +rapids_find_package( + CUDAToolkit REQUIRED + BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports +) include(cmake/modules/ConfigureCUDA.cmake) -############################################################################## -# - Requirements ------------------------------------------------------------- +# ################################################################################################## +# * Requirements ------------------------------------------------------------- if(RAFT_COMPILE_LIBRARIES) set(RAFT_COMPILE_DIST_LIBRARY ON) @@ -159,7 +185,9 @@ include(cmake/thirdparty/get_faiss.cmake) if(RAFT_ENABLE_cuco_DEPENDENCY) include(${rapids-cmake-dir}/cpm/cuco.cmake) - rapids_cpm_cuco(BUILD_EXPORT_SET raft-distance-lib-exports INSTALL_EXPORT_SET raft-distance-lib-exports) + rapids_cpm_cuco( + BUILD_EXPORT_SET raft-distance-lib-exports INSTALL_EXPORT_SET raft-distance-lib-exports + ) endif() if(BUILD_TESTS) @@ -171,69 +199,77 @@ if(BUILD_BENCH) rapids_cpm_gbench() endif() -############################################################################## -# - raft --------------------------------------------------------------------- +# ################################################################################################## +# * raft --------------------------------------------------------------------- add_library(raft INTERFACE) add_library(raft::raft ALIAS raft) -target_include_directories(raft INTERFACE - "$" - "$") - -# Keep RAFT as lightweight as possible. -# Only CUDA libs and rmm should -# be used in global target. -target_link_libraries(raft INTERFACE - rmm::rmm - CUDA::cublas${_ctk_static_suffix} - CUDA::curand${_ctk_static_suffix} - CUDA::cusolver${_ctk_static_suffix} - CUDA::cusparse${_ctk_static_suffix} - $<$:raft::Thrust> +target_include_directories( + raft INTERFACE "$" "$" +) + +# Keep RAFT as lightweight as possible. Only CUDA libs and rmm should be used in global target. +target_link_libraries( + raft + INTERFACE rmm::rmm + CUDA::cublas${_ctk_static_suffix} + CUDA::curand${_ctk_static_suffix} + CUDA::cusolver${_ctk_static_suffix} + CUDA::cusparse${_ctk_static_suffix} + $<$:raft::Thrust> ) target_compile_features(raft INTERFACE cxx_std_17 $) if(RAFT_COMPILE_DIST_LIBRARY OR RAFT_COMPILE_NN_LIBRARY) - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld" - [=[ + file( + WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld" + [=[ SECTIONS { .nvFatBinSegment : { *(.nvFatBinSegment) } .nv_fatbin : { *(.nv_fatbin) } } -]=]) +]=] + ) endif() -############################################################################## -# - NVTX support in raft ----------------------------------------------------- +# ################################################################################################## +# * NVTX support in raft ----------------------------------------------------- -if (RAFT_NVTX) +if(RAFT_NVTX) # This enables NVTX within the project with no option to disable it downstream. target_link_libraries(raft INTERFACE CUDA::nvToolsExt) target_compile_definitions(raft INTERFACE NVTX_ENABLED) else() - # Allow enable NVTX downstream if not set here. - # This creates a new option at build/install time, which is set by default to OFF, - # but can be enabled in the dependent project. - get_property(nvtx_option_help_string CACHE RAFT_NVTX PROPERTY HELPSTRING) - string(CONCAT nvtx_export_string - "option(RAFT_NVTX \"" ${nvtx_option_help_string} "\" OFF)" - [=[ + # Allow enable NVTX downstream if not set here. This creates a new option at build/install time, + # which is set by default to OFF, but can be enabled in the dependent project. + get_property( + nvtx_option_help_string + CACHE RAFT_NVTX + PROPERTY HELPSTRING + ) + string( + CONCAT + nvtx_export_string + "option(RAFT_NVTX \"" + ${nvtx_option_help_string} + "\" OFF)" + [=[ target_link_libraries(raft::raft INTERFACE $<$:CUDA::nvToolsExt>) target_compile_definitions(raft::raft INTERFACE $<$:NVTX_ENABLED>) - ]=]) + ]=] + ) endif() -############################################################################## -# - raft_distance ------------------------------------------------------------ -# TODO: -# Currently, this package also contains the 'random' namespace (for rmat logic) -# We couldn't get this to work properly due to strange CI failures as noticed -# in the PR#778. In the long term, we should rename this package to `raft_compiled` -# in order to have a single pre-compiled raft package for those who need it. +# ################################################################################################## +# * raft_distance ------------------------------------------------------------ TODO: Currently, this +# package also contains the 'random' namespace (for rmat logic) We couldn't get this to work +# properly due to strange CI failures as noticed in the PR#778. In the long term, we should rename +# this package to `raft_compiled` in order to have a single pre-compiled raft package for those +# who need it. add_library(raft_distance INTERFACE) if(TARGET raft_distance AND (NOT TARGET raft::distance)) @@ -243,7 +279,8 @@ endif() set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance) if(RAFT_COMPILE_DIST_LIBRARY) - add_library(raft_distance_lib + add_library( + raft_distance_lib src/distance/pairwise_distance.cu src/distance/fused_l2_min_arg.cu src/distance/update_centroids_float.cu @@ -262,9 +299,9 @@ if(RAFT_COMPILE_DIST_LIBRARY) src/distance/specializations/detail/kernels/gram_matrix_base_float.cu src/distance/specializations/detail/kernels/polynomial_kernel_double_int.cu src/distance/specializations/detail/kernels/polynomial_kernel_float_int.cu -# These are somehow missing a kernel definition which is causing a compile error. -# src/distance/specializations/detail/kernels/rbf_kernel_double.cu -# src/distance/specializations/detail/kernels/rbf_kernel_float.cu + # These are somehow missing a kernel definition which is causing a compile error. + # src/distance/specializations/detail/kernels/rbf_kernel_double.cu + # src/distance/specializations/detail/kernels/rbf_kernel_float.cu src/distance/specializations/detail/kernels/tanh_kernel_double.cu src/distance/specializations/detail/kernels/tanh_kernel_float.cu src/distance/specializations/detail/kl_divergence_float_float_float_int.cu @@ -302,26 +339,23 @@ if(RAFT_COMPILE_DIST_LIBRARY) ) set_target_properties( raft_distance_lib - PROPERTIES OUTPUT_NAME raft_distance - BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON) - - target_link_libraries(raft_distance_lib - PUBLIC raft::raft - cuco::cuco - ) - target_compile_options(raft_distance_lib - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - target_compile_definitions(raft_distance_lib - INTERFACE "RAFT_DISTANCE_COMPILED") + PROPERTIES OUTPUT_NAME raft_distance + BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) + + target_link_libraries(raft_distance_lib PUBLIC raft::raft cuco::cuco) + target_compile_options( + raft_distance_lib PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) + target_compile_definitions(raft_distance_lib INTERFACE "RAFT_DISTANCE_COMPILED") # ensure CUDA symbols aren't relocated to the middle of the debug build binaries target_link_options(raft_distance_lib PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") @@ -332,13 +366,12 @@ if(TARGET raft_distance_lib AND (NOT TARGET raft::raft_distance_lib)) add_library(raft::raft_distance_lib ALIAS raft_distance_lib) endif() -target_link_libraries(raft_distance INTERFACE - raft::raft - $ +target_link_libraries( + raft_distance INTERFACE raft::raft $ ) -############################################################################## -# - raft_nn ------------------------------------------------------------------ +# ################################################################################################## +# * raft_nn ------------------------------------------------------------------ add_library(raft_nn INTERFACE) if(TARGET raft_nn AND (NOT TARGET raft::nn)) @@ -348,121 +381,135 @@ endif() set_target_properties(raft_nn PROPERTIES EXPORT_NAME nn) if(RAFT_COMPILE_NN_LIBRARY) - add_library(raft_nn_lib - src/nn/specializations/ball_cover.cu - src/nn/specializations/detail/ball_cover_lowdim_pass_one_2d.cu - src/nn/specializations/detail/ball_cover_lowdim_pass_two_2d.cu - src/nn/specializations/detail/ball_cover_lowdim_pass_one_3d.cu - src/nn/specializations/detail/ball_cover_lowdim_pass_two_3d.cu - src/nn/specializations/detail/ivfpq_compute_similarity_float_fast.cu - src/nn/specializations/detail/ivfpq_compute_similarity_float_no_basediff.cu - src/nn/specializations/detail/ivfpq_compute_similarity_float_no_smem_lut.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_fast.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_no_basediff.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_no_smem_lut.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_fast.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_no_basediff.cu - src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_no_smem_lut.cu - src/nn/specializations/detail/ivfpq_compute_similarity_half_fast.cu - src/nn/specializations/detail/ivfpq_compute_similarity_half_no_basediff.cu - src/nn/specializations/detail/ivfpq_compute_similarity_half_no_smem_lut.cu - src/nn/specializations/detail/ivfpq_search_float_int64_t.cu - src/nn/specializations/detail/ivfpq_search_float_uint32_t.cu - src/nn/specializations/detail/ivfpq_search_float_uint64_t.cu - src/nn/specializations/fused_l2_knn_long_float_true.cu - src/nn/specializations/fused_l2_knn_long_float_false.cu - src/nn/specializations/fused_l2_knn_int_float_true.cu - src/nn/specializations/fused_l2_knn_int_float_false.cu - src/nn/specializations/knn.cu - ) + add_library( + raft_nn_lib + src/nn/specializations/ball_cover.cu + src/nn/specializations/detail/ball_cover_lowdim_pass_one_2d.cu + src/nn/specializations/detail/ball_cover_lowdim_pass_two_2d.cu + src/nn/specializations/detail/ball_cover_lowdim_pass_one_3d.cu + src/nn/specializations/detail/ball_cover_lowdim_pass_two_3d.cu + src/nn/specializations/detail/ivfpq_compute_similarity_float_fast.cu + src/nn/specializations/detail/ivfpq_compute_similarity_float_no_basediff.cu + src/nn/specializations/detail/ivfpq_compute_similarity_float_no_smem_lut.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_fast.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_no_basediff.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8s_no_smem_lut.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_fast.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_no_basediff.cu + src/nn/specializations/detail/ivfpq_compute_similarity_fp8u_no_smem_lut.cu + src/nn/specializations/detail/ivfpq_compute_similarity_half_fast.cu + src/nn/specializations/detail/ivfpq_compute_similarity_half_no_basediff.cu + src/nn/specializations/detail/ivfpq_compute_similarity_half_no_smem_lut.cu + src/nn/specializations/detail/ivfpq_search_float_int64_t.cu + src/nn/specializations/detail/ivfpq_search_float_uint32_t.cu + src/nn/specializations/detail/ivfpq_search_float_uint64_t.cu + src/nn/specializations/fused_l2_knn_long_float_true.cu + src/nn/specializations/fused_l2_knn_long_float_false.cu + src/nn/specializations/fused_l2_knn_int_float_true.cu + src/nn/specializations/fused_l2_knn_int_float_false.cu + src/nn/specializations/knn.cu + ) set_target_properties( raft_nn_lib - PROPERTIES OUTPUT_NAME raft_nn - BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON) - - target_link_libraries(raft_nn_lib - PUBLIC faiss::faiss - raft::raft) - target_compile_options(raft_nn_lib - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) + PROPERTIES OUTPUT_NAME raft_nn + BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) + + target_link_libraries(raft_nn_lib PUBLIC faiss::faiss raft::raft) + target_compile_options( + raft_nn_lib PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) # ensure CUDA symbols aren't relocated to the middle of the debug build binaries target_link_options(raft_nn_lib PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") - target_compile_definitions(raft_nn_lib - INTERFACE "RAFT_NN_COMPILED") + target_compile_definitions(raft_nn_lib INTERFACE "RAFT_NN_COMPILED") endif() if(TARGET raft_nn_lib AND (NOT TARGET raft::raft_nn_lib)) add_library(raft::raft_nn_lib ALIAS raft_nn_lib) endif() -target_link_libraries(raft_nn INTERFACE - raft::raft - $) +target_link_libraries(raft_nn INTERFACE raft::raft $) -############################################################################## -# - install targets----------------------------------------------------------- -rapids_cmake_install_lib_dir( lib_dir ) +# ################################################################################################## +# * install targets----------------------------------------------------------- +rapids_cmake_install_lib_dir(lib_dir) include(GNUInstallDirs) include(CPack) -install(TARGETS raft - DESTINATION ${lib_dir} - COMPONENT raft - EXPORT raft-exports) +install( + TARGETS raft + DESTINATION ${lib_dir} + COMPONENT raft + EXPORT raft-exports +) -install(TARGETS raft_distance - DESTINATION ${lib_dir} - COMPONENT raft - EXPORT raft-distance-exports) +install( + TARGETS raft_distance + DESTINATION ${lib_dir} + COMPONENT raft + EXPORT raft-distance-exports +) -install(TARGETS raft_nn - DESTINATION ${lib_dir} - COMPONENT raft - EXPORT raft-nn-exports) +install( + TARGETS raft_nn + DESTINATION ${lib_dir} + COMPONENT raft + EXPORT raft-nn-exports +) if(TARGET raft_distance_lib) - install(TARGETS raft_distance_lib - DESTINATION ${lib_dir} - COMPONENT distance - EXPORT raft-distance-lib-exports) - install(DIRECTORY include/raft_distance - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - COMPONENT distance) + install( + TARGETS raft_distance_lib + DESTINATION ${lib_dir} + COMPONENT distance + EXPORT raft-distance-lib-exports + ) + install( + DIRECTORY include/raft_distance + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT distance + ) endif() if(TARGET raft_nn_lib) - install(TARGETS raft_nn_lib - DESTINATION ${lib_dir} - COMPONENT nn - EXPORT raft-nn-lib-exports) + install( + TARGETS raft_nn_lib + DESTINATION ${lib_dir} + COMPONENT nn + EXPORT raft-nn-lib-exports + ) endif() - -install(DIRECTORY include/raft - COMPONENT raft - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install( + DIRECTORY include/raft + COMPONENT raft + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) # Temporary install of raft.hpp while the file is removed -install(FILES include/raft.hpp - COMPONENT raft - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft) +install( + FILES include/raft.hpp + COMPONENT raft + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft +) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/include/raft/version_config.hpp - COMPONENT raft - DESTINATION include/raft) +install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/include/raft/version_config.hpp + COMPONENT raft + DESTINATION include/raft +) -############################################################################## -# - export/install optional components -------------------------------------- +# ################################################################################################## +# * export/install optional components -------------------------------------- include("${rapids-cmake-dir}/export/write_dependencies.cmake") @@ -479,30 +526,31 @@ endif() foreach(comp install_comp IN ZIP_LISTS raft_components raft_install_comp) install( - EXPORT raft-${comp}-exports - FILE raft-${comp}-targets.cmake - NAMESPACE raft:: - DESTINATION "${lib_dir}/cmake/raft" - COMPONENT ${install_comp} + EXPORT raft-${comp}-exports + FILE raft-${comp}-targets.cmake + NAMESPACE raft:: + DESTINATION "${lib_dir}/cmake/raft" + COMPONENT ${install_comp} ) export( - EXPORT raft-${comp}-exports - FILE ${RAFT_BINARY_DIR}/raft-${comp}-targets.cmake - NAMESPACE raft:: + EXPORT raft-${comp}-exports + FILE ${RAFT_BINARY_DIR}/raft-${comp}-targets.cmake + NAMESPACE raft:: ) rapids_export_write_dependencies( - BUILD raft-${comp}-exports "${PROJECT_BINARY_DIR}/raft-${comp}-dependencies.cmake" + BUILD raft-${comp}-exports "${PROJECT_BINARY_DIR}/raft-${comp}-dependencies.cmake" ) rapids_export_write_dependencies( - INSTALL raft-${comp}-exports "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/${install_comp}/raft-${comp}-dependencies.cmake" + INSTALL raft-${comp}-exports + "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/${install_comp}/raft-${comp}-dependencies.cmake" ) endforeach() -############################################################################## -# - install export ----------------------------------------------------------- +# ################################################################################################## +# * install export ----------------------------------------------------------- set(doc_string - [=[ + [=[ Provide targets for the RAFT: Reusable Accelerated Functions and Tools RAFT contains fundamental widely-used algorithms and primitives @@ -517,21 +565,27 @@ Imported Targets: - raft::nn brought in by the `nn` optional component - raft::distance brought in by the `distance` optional component -]=]) +]=] +) set(code_string ${nvtx_export_string}) if(RAFT_ENABLE_thrust_DEPENDENCY) - string(APPEND code_string - [=[ + string( + APPEND + code_string + [=[ if(NOT TARGET raft::Thrust) thrust_create_target(raft::Thrust FROM_OPTIONS) endif() - ]=]) + ]=] + ) endif() -string(APPEND code_string -[=[ +string( + APPEND + code_string + [=[ if(distance IN_LIST raft_FIND_COMPONENTS) enable_language(CUDA) endif() @@ -545,45 +599,41 @@ if(nn IN_LIST raft_FIND_COMPONENTS) add_library(faiss ALIAS faiss::faiss) endif() endif() -]=]) +]=] +) # Use `rapids_export` for 22.04 as it will have COMPONENT support include(cmake/modules/raft_export.cmake) -raft_export(INSTALL raft - COMPONENTS nn distance - EXPORT_SET raft-exports - GLOBAL_TARGETS raft nn distance - NAMESPACE raft:: - DOCUMENTATION doc_string - FINAL_CODE_BLOCK code_string) - -############################################################################## -# - build export ------------------------------------------------------------- -raft_export(BUILD raft - EXPORT_SET raft-exports - COMPONENTS nn distance - GLOBAL_TARGETS raft raft_distance raft_nn - DOCUMENTATION doc_string - NAMESPACE raft:: - FINAL_CODE_BLOCK code_string) - -############################################################################## -# - build test executable ---------------------------------------------------- +raft_export( + INSTALL raft COMPONENTS nn distance EXPORT_SET raft-exports GLOBAL_TARGETS raft nn distance + NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string +) + +# ################################################################################################## +# * build export ------------------------------------------------------------- +raft_export( + BUILD raft EXPORT_SET raft-exports COMPONENTS nn distance GLOBAL_TARGETS raft raft_distance + raft_nn DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string +) + +# ################################################################################################## +# * build test executable ---------------------------------------------------- if(BUILD_TESTS) include(test/CMakeLists.txt) endif() -############################################################################## -# - build benchmark executable ----------------------------------------------- +# ################################################################################################## +# * build benchmark executable ----------------------------------------------- if(BUILD_BENCH) include(bench/CMakeLists.txt) endif() -############################################################################## -# - doxygen targets ---------------------------------------------------------- +# ################################################################################################## +# * doxygen targets ---------------------------------------------------------- include(cmake/doxygen.cmake) -add_doxygen_target(IN_DOXYFILE doxygen/Doxyfile.in - OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile - CWD ${CMAKE_CURRENT_BINARY_DIR}) +add_doxygen_target( + IN_DOXYFILE doxygen/Doxyfile.in OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile CWD + ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/cpp/bench/CMakeLists.txt b/cpp/bench/CMakeLists.txt index 81e894fbbc..3c3d296b06 100644 --- a/cpp/bench/CMakeLists.txt +++ b/cpp/bench/CMakeLists.txt @@ -1,141 +1,131 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= -################################################################################################### -# - compiler function ----------------------------------------------------------------------------- +# ################################################################################################## +# * compiler function ----------------------------------------------------------------------------- function(ConfigureBench) -set(options OPTIONAL DIST NN) -set(oneValueArgs NAME ) -set(multiValueArgs PATH TARGETS CONFIGURATIONS) - -cmake_parse_arguments(ConfigureBench "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - -set(BENCH_NAME ${ConfigureBench_NAME}) - -add_executable(${BENCH_NAME} ${ConfigureBench_PATH}) - -target_link_libraries(${BENCH_NAME} - PRIVATE - raft::raft - $<$:raft::distance> - $<$:raft::nn> - benchmark::benchmark - Threads::Threads - $ - $ - ) - -set_target_properties(${BENCH_NAME} - PROPERTIES - # set target compile options - INSTALL_RPATH "\$ORIGIN/../../../lib" - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - ) - -target_compile_options(${BENCH_NAME} - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - -target_include_directories(${BENCH_NAME} - PUBLIC "$" - ) - -install( - TARGETS ${BENCH_NAME} - COMPONENT testing - DESTINATION bin/gbench/libraft - EXCLUDE_FROM_ALL) + set(options OPTIONAL DIST NN) + set(oneValueArgs NAME) + set(multiValueArgs PATH TARGETS CONFIGURATIONS) + + cmake_parse_arguments(ConfigureBench "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + set(BENCH_NAME ${ConfigureBench_NAME}) + + add_executable(${BENCH_NAME} ${ConfigureBench_PATH}) + + target_link_libraries( + ${BENCH_NAME} + PRIVATE raft::raft + $<$:raft::distance> + $<$:raft::nn> + benchmark::benchmark + Threads::Threads + $ + $ + ) + + set_target_properties( + ${BENCH_NAME} + PROPERTIES # set target compile options + INSTALL_RPATH "\$ORIGIN/../../../lib" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) + + target_compile_options( + ${BENCH_NAME} PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) + + target_include_directories(${BENCH_NAME} PUBLIC "$") + + install( + TARGETS ${BENCH_NAME} + COMPONENT testing + DESTINATION bin/gbench/libraft + EXCLUDE_FROM_ALL + ) endfunction() if(BUILD_BENCH) - ConfigureBench(NAME CLUSTER_BENCH - PATH - bench/cluster/kmeans_balanced.cu - bench/cluster/kmeans.cu - bench/main.cpp - OPTIONAL DIST NN - ) - - ConfigureBench(NAME DISTANCE_BENCH - PATH - bench/distance/distance_cosine.cu - bench/distance/distance_exp_l2.cu - bench/distance/distance_l1.cu - bench/distance/distance_unexp_l2.cu - bench/distance/fused_l2_nn.cu - bench/distance/kernels.cu - bench/main.cpp - OPTIONAL DIST - ) - - ConfigureBench(NAME LINALG_BENCH - PATH - bench/linalg/add.cu - bench/linalg/map_then_reduce.cu - bench/linalg/matrix_vector_op.cu - bench/linalg/reduce_rows_by_key.cu - bench/linalg/reduce.cu - bench/main.cpp - ) - - ConfigureBench(NAME MATRIX_BENCH - PATH - bench/matrix/argmin.cu - bench/main.cpp - ) - - ConfigureBench(NAME RANDOM_BENCH - PATH - bench/random/make_blobs.cu - bench/random/permute.cu - bench/random/rng.cu - bench/main.cpp - ) - - ConfigureBench(NAME SPARSE_BENCH - PATH - bench/sparse/convert_csr.cu - bench/main.cpp - ) - - ConfigureBench(NAME NEIGHBORS_BENCH - PATH - bench/neighbors/knn/brute_force_float_int64_t.cu - bench/neighbors/knn/brute_force_float_uint32_t.cu - bench/neighbors/knn/ivf_flat_float_int64_t.cu - bench/neighbors/knn/ivf_flat_float_uint32_t.cu - bench/neighbors/knn/ivf_flat_int8_t_int64_t.cu - bench/neighbors/knn/ivf_flat_uint8_t_uint32_t.cu - bench/neighbors/knn/ivf_pq_float_int64_t.cu - bench/neighbors/knn/ivf_pq_float_uint32_t.cu - bench/neighbors/knn/ivf_pq_int8_t_int64_t.cu - bench/neighbors/knn/ivf_pq_uint8_t_uint32_t.cu - bench/neighbors/selection.cu - bench/main.cpp - OPTIONAL DIST NN - ) + ConfigureBench( + NAME CLUSTER_BENCH PATH bench/cluster/kmeans_balanced.cu bench/cluster/kmeans.cu bench/main.cpp + OPTIONAL DIST NN + ) + + ConfigureBench( + NAME + DISTANCE_BENCH + PATH + bench/distance/distance_cosine.cu + bench/distance/distance_exp_l2.cu + bench/distance/distance_l1.cu + bench/distance/distance_unexp_l2.cu + bench/distance/fused_l2_nn.cu + bench/distance/kernels.cu + bench/main.cpp + OPTIONAL + DIST + ) + + ConfigureBench( + NAME + LINALG_BENCH + PATH + bench/linalg/add.cu + bench/linalg/map_then_reduce.cu + bench/linalg/matrix_vector_op.cu + bench/linalg/reduce_rows_by_key.cu + bench/linalg/reduce.cu + bench/main.cpp + ) + + ConfigureBench(NAME MATRIX_BENCH PATH bench/matrix/argmin.cu bench/main.cpp) + + ConfigureBench( + NAME RANDOM_BENCH PATH bench/random/make_blobs.cu bench/random/permute.cu bench/random/rng.cu + bench/main.cpp + ) + + ConfigureBench(NAME SPARSE_BENCH PATH bench/sparse/convert_csr.cu bench/main.cpp) + + ConfigureBench( + NAME + NEIGHBORS_BENCH + PATH + bench/neighbors/knn/brute_force_float_int64_t.cu + bench/neighbors/knn/brute_force_float_uint32_t.cu + bench/neighbors/knn/ivf_flat_float_int64_t.cu + bench/neighbors/knn/ivf_flat_float_uint32_t.cu + bench/neighbors/knn/ivf_flat_int8_t_int64_t.cu + bench/neighbors/knn/ivf_flat_uint8_t_uint32_t.cu + bench/neighbors/knn/ivf_pq_float_int64_t.cu + bench/neighbors/knn/ivf_pq_float_uint32_t.cu + bench/neighbors/knn/ivf_pq_int8_t_int64_t.cu + bench/neighbors/knn/ivf_pq_uint8_t_uint32_t.cu + bench/neighbors/selection.cu + bench/main.cpp + OPTIONAL + DIST + NN + ) endif() - diff --git a/cpp/cmake/config.json b/cpp/cmake/config.json new file mode 100644 index 0000000000..f7cc50e513 --- /dev/null +++ b/cpp/cmake/config.json @@ -0,0 +1,43 @@ +{ + "parse": { + "additional_commands": { + "CPMFindPackage": { + "kwargs": { + "NAME": 1, + "GITHUB_REPOSITORY": "?", + "GIT_TAG": "?", + "VERSION": "?", + "GIT_SHALLOW": "?", + "OPTIONS": "*", + "FIND_PACKAGE_ARGUMENTS": "*" + } + }, + "ConfigureTest": { + "flags": ["TEST_NAME", "TEST_SRC"] + }, + "ConfigureBench": { + "flags": ["BENCH_NAME", "BENCH_SRC"] + } + } + }, + "format": { + "line_width": 100, + "tab_size": 2, + "command_case": "unchanged", + "max_lines_hwrap": 1, + "max_pargs_hwrap": 999, + "dangle_parens": true + }, + "lint": { + "disabled_codes": ["C0301", "C0111", "C0113"], + "function_pattern": "[0-9A-z_]+", + "macro_pattern": "[0-9A-z_]+", + "global_var_pattern": "[A-z][0-9A-z_]+", + "internal_var_pattern": "_[A-z][0-9A-z_]+", + "local_var_pattern": "[A-z][A-z0-9_]+", + "private_var_pattern": "_[0-9A-z_]+", + "public_var_pattern": "[A-z][0-9A-z_]+", + "argument_var_pattern": "[A-z][A-z0-9_]+", + "keyword_pattern": "[A-z][0-9A-z_]+" + } +} diff --git a/cpp/cmake/doxygen.cmake b/cpp/cmake/doxygen.cmake index 7d06ec194c..a6fddbacd6 100644 --- a/cpp/cmake/doxygen.cmake +++ b/cpp/cmake/doxygen.cmake @@ -1,16 +1,14 @@ # Copyright (c) 2020-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. # find_package(Doxygen 1.8.11) @@ -24,11 +22,13 @@ function(add_doxygen_target) configure_file(${dox_IN_DOXYFILE} ${dox_OUT_DOXYFILE} @ONLY) message("Command: ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE}") - add_custom_target(docs_raft + add_custom_target( + docs_raft ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE} WORKING_DIRECTORY ${dox_CWD} VERBATIM - COMMENT "Generate doxygen docs") + COMMENT "Generate doxygen docs" + ) else() message("add_doxygen_target: doxygen exe not found") endif() diff --git a/cpp/cmake/modules/ConfigureCUDA.cmake b/cpp/cmake/modules/ConfigureCUDA.cmake index 440c8c4f3a..9b029a5242 100644 --- a/cpp/cmake/modules/ConfigureCUDA.cmake +++ b/cpp/cmake/modules/ConfigureCUDA.cmake @@ -1,26 +1,24 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2018-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= if(DISABLE_DEPRECATION_WARNINGS) - list(APPEND RAFT_CXX_FLAGS -Wno-deprecated-declarations) - list(APPEND RAFT_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) + list(APPEND RAFT_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND RAFT_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) endif() if(CMAKE_COMPILER_IS_GNUCXX) - list(APPEND RAFT_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) + list(APPEND RAFT_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) endif() list(APPEND RAFT_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) @@ -31,21 +29,22 @@ list(APPEND RAFT_CUDA_FLAGS -Xfatbin=-compress-all) # set warnings as errors if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) - list(APPEND RAFT_CUDA_FLAGS -Werror=all-warnings) + list(APPEND RAFT_CUDA_FLAGS -Werror=all-warnings) endif() list(APPEND RAFT_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) -# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking +# Option to enable line info in CUDA device compilation to allow introspection when profiling / +# memchecking if(CUDA_ENABLE_LINEINFO) - list(APPEND RAFT_CUDA_FLAGS -lineinfo) + list(APPEND RAFT_CUDA_FLAGS -lineinfo) endif() if(OpenMP_FOUND) - list(APPEND RAFT_CUDA_FLAGS -Xcompiler=${OpenMP_CXX_FLAGS}) + list(APPEND RAFT_CUDA_FLAGS -Xcompiler=${OpenMP_CXX_FLAGS}) endif() # Debug options if(CMAKE_BUILD_TYPE MATCHES Debug) - message(VERBOSE "RAFT: Building with debugging flags") - list(APPEND RAFT_CUDA_FLAGS -G -Xcompiler=-rdynamic) + message(VERBOSE "RAFT: Building with debugging flags") + list(APPEND RAFT_CUDA_FLAGS -G -Xcompiler=-rdynamic) endif() diff --git a/cpp/cmake/modules/raft_export.cmake b/cpp/cmake/modules/raft_export.cmake index 748fa8ad26..41fe6d73da 100644 --- a/cpp/cmake/modules/raft_export.cmake +++ b/cpp/cmake/modules/raft_export.cmake @@ -1,18 +1,16 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2021-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= include_guard(GLOBAL) #[=======================================================================[.rst: @@ -174,19 +172,26 @@ function(raft_export type project_name) set(scratch_dir "${PROJECT_BINARY_DIR}/rapids-cmake/${project_name}/export/raft/") - configure_package_config_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" - "${scratch_dir}/${project_name}-config.cmake" - INSTALL_DESTINATION "${install_location}") + configure_package_config_file( + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" + "${scratch_dir}/${project_name}-config.cmake" INSTALL_DESTINATION "${install_location}" + ) if(rapids_version_set) write_basic_package_version_file( - "${scratch_dir}/${project_name}-config-version.cmake" VERSION ${rapids_project_version} - COMPATIBILITY ${rapids_project_version_compat}) + "${scratch_dir}/${project_name}-config-version.cmake" + VERSION ${rapids_project_version} + COMPATIBILITY ${rapids_project_version_compat} + ) endif() - install(EXPORT ${RAPIDS_EXPORT_SET} FILE ${project_name}-targets.cmake - NAMESPACE ${RAPIDS_PROJECT_VERSION} DESTINATION "${install_location}" - COMPONENT raft) + install( + EXPORT ${RAPIDS_EXPORT_SET} + FILE ${project_name}-targets.cmake + NAMESPACE ${RAPIDS_PROJECT_VERSION} + DESTINATION "${install_location}" + COMPONENT raft + ) if(TARGET rapids_export_install_${RAPIDS_EXPORT_SET}) include("${rapids-cmake-dir}/export/write_dependencies.cmake") @@ -203,41 +208,55 @@ function(raft_export type project_name) endif() # Install everything we have generated - install(DIRECTORY "${scratch_dir}/" DESTINATION "${install_location}" - COMPONENT raft) + install( + DIRECTORY "${scratch_dir}/" + DESTINATION "${install_location}" + COMPONENT raft + ) foreach(comp nn distance) set(scratch_dir "${PROJECT_BINARY_DIR}/rapids-cmake/${project_name}/export/${comp}/") file(MAKE_DIRECTORY "${scratch_dir}") - install(DIRECTORY "${scratch_dir}" DESTINATION "${install_location}" - COMPONENT ${comp}) + install( + DIRECTORY "${scratch_dir}" + DESTINATION "${install_location}" + COMPONENT ${comp} + ) endforeach() else() set(install_location "${PROJECT_BINARY_DIR}") - configure_package_config_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" - "${install_location}/${project_name}-config.cmake" - INSTALL_DESTINATION "${install_location}") + configure_package_config_file( + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" + "${install_location}/${project_name}-config.cmake" INSTALL_DESTINATION "${install_location}" + ) if(rapids_version_set) write_basic_package_version_file( - "${install_location}/${project_name}-config-version.cmake" VERSION ${rapids_project_version} - COMPATIBILITY ${rapids_project_version_compat}) + "${install_location}/${project_name}-config-version.cmake" + VERSION ${rapids_project_version} + COMPATIBILITY ${rapids_project_version_compat} + ) endif() - export(EXPORT ${RAPIDS_EXPORT_SET} NAMESPACE ${RAPIDS_PROJECT_VERSION} - FILE "${install_location}/${project_name}-targets.cmake") + export( + EXPORT ${RAPIDS_EXPORT_SET} + NAMESPACE ${RAPIDS_PROJECT_VERSION} + FILE "${install_location}/${project_name}-targets.cmake" + ) if(TARGET rapids_export_build_${RAPIDS_EXPORT_SET}) include("${rapids-cmake-dir}/export/write_dependencies.cmake") - rapids_export_write_dependencies(BUILD ${RAPIDS_EXPORT_SET} - "${install_location}/${project_name}-dependencies.cmake") + rapids_export_write_dependencies( + BUILD ${RAPIDS_EXPORT_SET} "${install_location}/${project_name}-dependencies.cmake" + ) endif() if(DEFINED RAPIDS_LANGUAGES) include("${rapids-cmake-dir}/export/write_language.cmake") foreach(lang IN LISTS RAPIDS_LANGUAGES) - rapids_export_write_language(BUILD ${lang} - "${install_location}/${project_name}-${lang}-language.cmake") + rapids_export_write_language( + BUILD ${lang} "${install_location}/${project_name}-${lang}-language.cmake" + ) endforeach() endif() diff --git a/cpp/scripts/__clang_cuda_additional_intrinsics.h b/cpp/scripts/__clang_cuda_additional_intrinsics.h index 8964d210bf..b9c032dc45 100644 --- a/cpp/scripts/__clang_cuda_additional_intrinsics.h +++ b/cpp/scripts/__clang_cuda_additional_intrinsics.h @@ -1,3 +1,4 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. #ifndef __CLANG_CUDA_ADDITIONAL_INTRINSICS_H__ #define __CLANG_CUDA_ADDITIONAL_INTRINSICS_H__ #ifndef __CUDA__ @@ -7,46 +8,49 @@ // for some of these macros, see cuda_fp16.hpp #if defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320)) #if (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__) -#define __LDG_PTR "l" -#define __LBITS "64" +#define __LDG_PTR "l" +#define __LBITS "64" #else -#define __LDG_PTR "r" -#define __LBITS "32" -#endif // (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__) +#define __LDG_PTR "r" +#define __LBITS "32" +#endif // (defined(_MSC_VER) && defined(_WIN64)) || defined(__LP64__) || defined(__CUDACC_RTC__) #define __NOARG -#define __MAKE_LD(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ - __device__ __forceinline__ c_typ __ld ## cop (const c_typ* addr) { \ - int_typ out; \ - asm("ld." #cop "." ptx_typ " %0, [%1];" \ - : "=" inl_typ(out) : __LDG_PTR(addr)mem); \ - return (c_typ)out; \ +#define __MAKE_LD(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ + __device__ __forceinline__ c_typ __ld##cop(const c_typ* addr) \ + { \ + int_typ out; \ + asm("ld." #cop "." ptx_typ " %0, [%1];" : "=" inl_typ(out) : __LDG_PTR(addr) mem); \ + return (c_typ)out; \ } -#define __MAKE_LD2(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ - __device__ __forceinline__ c_typ __ld ## cop (const c_typ* addr) { \ - int_typ out1, out2; \ - asm("ld." #cop ".v2." ptx_typ " {%0, %1}, [%2];" \ - : "=" inl_typ(out1), "=" inl_typ(out2) : __LDG_PTR(addr)mem); \ - c_typ out; \ - out.x = out1; \ - out.y = out2; \ - return out; \ +#define __MAKE_LD2(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ + __device__ __forceinline__ c_typ __ld##cop(const c_typ* addr) \ + { \ + int_typ out1, out2; \ + asm("ld." #cop ".v2." ptx_typ " {%0, %1}, [%2];" \ + : "=" inl_typ(out1), "=" inl_typ(out2) \ + : __LDG_PTR(addr) mem); \ + c_typ out; \ + out.x = out1; \ + out.y = out2; \ + return out; \ } -#define __MAKE_LD4(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ - __device__ __forceinline__ c_typ __ld ## cop (const c_typ* addr) { \ - int_typ out1, out2, out3, out4; \ - asm("ld." #cop".v4." ptx_typ " {%0, %1, %2, %3}, [%4];" \ - : "=" inl_typ(out1), "=" inl_typ(out2), \ - "=" inl_typ(out3), "=" inl_typ(out4) : __LDG_PTR(addr)mem); \ - c_typ out; \ - out.x = out1; \ - out.y = out2; \ - out.z = out3; \ - out.w = out4; \ - return out; \ +#define __MAKE_LD4(cop, c_typ, int_typ, ptx_typ, inl_typ, mem) \ + __device__ __forceinline__ c_typ __ld##cop(const c_typ* addr) \ + { \ + int_typ out1, out2, out3, out4; \ + asm("ld." #cop ".v4." ptx_typ " {%0, %1, %2, %3}, [%4];" \ + : "=" inl_typ(out1), "=" inl_typ(out2), "=" inl_typ(out3), "=" inl_typ(out4) \ + : __LDG_PTR(addr) mem); \ + c_typ out; \ + out.x = out1; \ + out.y = out2; \ + out.z = out3; \ + out.w = out4; \ + return out; \ } __MAKE_LD(cg, char, short, "s8", "h", __NOARG) @@ -82,7 +86,6 @@ __MAKE_LD4(cg, int4, int, "s32", "r", __NOARG) __MAKE_LD4(cg, uint4, unsigned int, "u32", "r", __NOARG) __MAKE_LD4(cg, float4, float, "f32", "f", __NOARG) - __MAKE_LD(ca, char, short, "s8", "h", __NOARG) __MAKE_LD(ca, signed char, short, "s8", "h", __NOARG) __MAKE_LD(ca, unsigned char, short, "u8", "h", __NOARG) @@ -116,7 +119,6 @@ __MAKE_LD4(ca, int4, int, "s32", "r", __NOARG) __MAKE_LD4(ca, uint4, unsigned int, "u32", "r", __NOARG) __MAKE_LD4(ca, float4, float, "f32", "f", __NOARG) - __MAKE_LD(cs, char, short, "s8", "h", __NOARG) __MAKE_LD(cs, signed char, short, "s8", "h", __NOARG) __MAKE_LD(cs, unsigned char, short, "u8", "h", __NOARG) @@ -150,7 +152,6 @@ __MAKE_LD4(cs, int4, int, "s32", "r", __NOARG) __MAKE_LD4(cs, uint4, unsigned int, "u32", "r", __NOARG) __MAKE_LD4(cs, float4, float, "f32", "f", __NOARG) - __MAKE_LD(lu, char, short, "s8", "h", : "memory") __MAKE_LD(lu, signed char, short, "s8", "h", : "memory") __MAKE_LD(lu, unsigned char, short, "u8", "h", : "memory") @@ -184,7 +185,6 @@ __MAKE_LD4(lu, int4, int, "s32", "r", : "memory") __MAKE_LD4(lu, uint4, unsigned int, "u32", "r", : "memory") __MAKE_LD4(lu, float4, float, "f32", "f", : "memory") - __MAKE_LD(cv, char, short, "s8", "h", : "memory") __MAKE_LD(cv, signed char, short, "s8", "h", : "memory") __MAKE_LD(cv, unsigned char, short, "u8", "h", : "memory") @@ -218,26 +218,30 @@ __MAKE_LD4(cv, int4, int, "s32", "r", : "memory") __MAKE_LD4(cv, uint4, unsigned int, "u32", "r", : "memory") __MAKE_LD4(cv, float4, float, "f32", "f", : "memory") - -#define __MAKE_ST(cop, c_typ, int_typ, ptx_typ, inl_typ) \ - __device__ __forceinline__ void __st ## cop (c_typ* addr, c_typ v) { \ - asm("st." #cop "." ptx_typ " [%0], %1;" \ - :: __LDG_PTR(addr), inl_typ((int_typ)v) : "memory"); \ +#define __MAKE_ST(cop, c_typ, int_typ, ptx_typ, inl_typ) \ + __device__ __forceinline__ void __st##cop(c_typ* addr, c_typ v) \ + { \ + asm("st." #cop "." ptx_typ " [%0], %1;" ::__LDG_PTR(addr), inl_typ((int_typ)v) : "memory"); \ } -#define __MAKE_ST2(cop, c_typ, int_typ, ptx_typ, inl_typ) \ - __device__ __forceinline__ void __st ## cop (c_typ* addr, c_typ v) { \ - int_typ v1 = v.x, v2 = v.y; \ - asm("st." #cop ".v2." ptx_typ " [%0], {%1, %2};" \ - :: __LDG_PTR(addr), inl_typ(v1), inl_typ(v2) : "memory"); \ +#define __MAKE_ST2(cop, c_typ, int_typ, ptx_typ, inl_typ) \ + __device__ __forceinline__ void __st##cop(c_typ* addr, c_typ v) \ + { \ + int_typ v1 = v.x, v2 = v.y; \ + asm("st." #cop ".v2." ptx_typ " [%0], {%1, %2};" ::__LDG_PTR(addr), inl_typ(v1), inl_typ(v2) \ + : "memory"); \ } -#define __MAKE_ST4(cop, c_typ, int_typ, ptx_typ, inl_typ) \ - __device__ __forceinline__ c_typ __st ## cop (c_typ* addr, c_typ v) { \ - int_typ v1 = v.x, v2 = v.y, v3 = v.z, v4 = v.w; \ - asm("st." #cop ".v4." ptx_typ " [%0], {%1, %2, %3, %4};" \ - :: __LDG_PTR(addr), inl_typ(v1), inl_typ(v2), \ - inl_typ(v3), inl_typ(v4) : "memory"); \ +#define __MAKE_ST4(cop, c_typ, int_typ, ptx_typ, inl_typ) \ + __device__ __forceinline__ c_typ __st##cop(c_typ* addr, c_typ v) \ + { \ + int_typ v1 = v.x, v2 = v.y, v3 = v.z, v4 = v.w; \ + asm("st." #cop ".v4." ptx_typ " [%0], {%1, %2, %3, %4};" ::__LDG_PTR(addr), \ + inl_typ(v1), \ + inl_typ(v2), \ + inl_typ(v3), \ + inl_typ(v4) \ + : "memory"); \ } __MAKE_ST(wb, char, short, "s8", "h") @@ -273,7 +277,6 @@ __MAKE_ST4(wb, int4, int, "s32", "r") __MAKE_ST4(wb, uint4, unsigned int, "u32", "r") __MAKE_ST4(wb, float4, float, "f32", "f") - __MAKE_ST(cg, char, short, "s8", "h") __MAKE_ST(cg, signed char, short, "s8", "h") __MAKE_ST(cg, unsigned char, short, "u8", "h") @@ -307,7 +310,6 @@ __MAKE_ST4(cg, int4, int, "s32", "r") __MAKE_ST4(cg, uint4, unsigned int, "u32", "r") __MAKE_ST4(cg, float4, float, "f32", "f") - __MAKE_ST(cs, char, short, "s8", "h") __MAKE_ST(cs, signed char, short, "s8", "h") __MAKE_ST(cs, unsigned char, short, "u8", "h") @@ -341,7 +343,6 @@ __MAKE_ST4(cs, int4, int, "s32", "r") __MAKE_ST4(cs, uint4, unsigned int, "u32", "r") __MAKE_ST4(cs, float4, float, "f32", "f") - __MAKE_ST(wt, char, short, "s8", "h") __MAKE_ST(wt, signed char, short, "s8", "h") __MAKE_ST(wt, unsigned char, short, "u8", "h") @@ -375,7 +376,6 @@ __MAKE_ST4(wt, int4, int, "s32", "r") __MAKE_ST4(wt, uint4, unsigned int, "u32", "r") __MAKE_ST4(wt, float4, float, "f32", "f") - #undef __MAKE_ST4 #undef __MAKE_ST2 #undef __MAKE_ST @@ -386,6 +386,6 @@ __MAKE_ST4(wt, float4, float, "f32", "f") #undef __LBITS #undef __LDG_PTR -#endif // defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320)) +#endif // defined(__cplusplus) && (!defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 320)) -#endif // defined(__CLANG_CUDA_ADDITIONAL_INTRINSICS_H__) +#endif // defined(__CLANG_CUDA_ADDITIONAL_INTRINSICS_H__) diff --git a/cpp/scripts/run-cmake-format.sh b/cpp/scripts/run-cmake-format.sh new file mode 100755 index 0000000000..db5a8b5804 --- /dev/null +++ b/cpp/scripts/run-cmake-format.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Copyright (c) 2022, NVIDIA CORPORATION. + +# This script is a wrapper for cmakelang that may be used with pre-commit. The +# wrapping is necessary because RAPIDS libraries split configuration for +# cmakelang linters between a local config file and a second config file that's +# shared across all of RAPIDS via rapids-cmake. In order to keep it up to date +# this file is only maintained in one place (the rapids-cmake repo) and +# pulled down during builds. We need a way to invoke CMake linting commands +# without causing pre-commit failures (which could block local commits or CI), +# while also being sufficiently flexible to allow users to maintain the config +# file independently of a build directory. +# +# This script provides the minimal functionality to enable those use cases. It +# searches in a number of predefined locations for the rapids-cmake config file +# and exits gracefully if the file is not found. If a user wishes to specify a +# config file at a nonstandard location, they may do so by setting the +# environment variable RAPIDS_CMAKE_FORMAT_FILE. +# +# This script can be invoked directly anywhere within the project repository. +# Alternatively, it may be invoked as a pre-commit hook via +# `pre-commit run (cmake-format)|(cmake-lint)`. +# +# Usage: +# bash run-cmake-format.sh {cmake-format,cmake-lint} infile [infile ...] + +status=0 +if [ -z ${RAFT_ROOT:+PLACEHOLDER} ]; then + RAFT_BUILD_DIR=$(git rev-parse --show-toplevel 2>&1)/cpp/build + status=$? +else + RAFT_BUILD_DIR=${RAFT_ROOT} +fi + +if ! [ ${status} -eq 0 ]; then + if [[ ${RAFT_BUILD_DIR} == *"not a git repository"* ]]; then + echo "This script must be run inside the raft repository, or the RAFT_ROOT environment variable must be set." + else + echo "Script failed with unknown error attempting to determine project root:" + echo ${RAFT_BUILD_DIR} + fi + exit 1 +fi + +DEFAULT_FORMAT_FILE_LOCATIONS=( + "${RAFT_BUILD_DIR:-${HOME}}/_deps/rapids-cmake-src/cmake-format-rapids-cmake.json" +) + +if [ -z ${RAPIDS_CMAKE_FORMAT_FILE:+PLACEHOLDER} ]; then + for file_path in ${DEFAULT_FORMAT_FILE_LOCATIONS[@]}; do + if [ -f ${file_path} ]; then + RAPIDS_CMAKE_FORMAT_FILE=${file_path} + break + fi + done +fi + +if [ -z ${RAPIDS_CMAKE_FORMAT_FILE:+PLACEHOLDER} ]; then + echo "The rapids-cmake cmake-format configuration file was not found at any of the default search locations: " + echo "" + ( IFS=$'\n'; echo "${DEFAULT_FORMAT_FILE_LOCATIONS[*]}" ) + echo "" + echo "Try setting the environment variable RAPIDS_CMAKE_FORMAT_FILE to the path to the config file." + exit 0 +else + echo "Using format file ${RAPIDS_CMAKE_FORMAT_FILE}" +fi + +if [[ $1 == "cmake-format" ]]; then + cmake-format -i --config-files cpp/cmake/config.json ${RAPIDS_CMAKE_FORMAT_FILE} -- ${@:2} +elif [[ $1 == "cmake-lint" ]]; then + # Since the pre-commit hook is verbose, we have to be careful to only + # present cmake-lint's output (which is quite verbose) if we actually + # observe a failure. + OUTPUT=$(cmake-lint --config-files cpp/cmake/config.json ${RAPIDS_CMAKE_FORMAT_FILE} -- ${@:2}) + status=$? + + if ! [ ${status} -eq 0 ]; then + echo "${OUTPUT}" + fi + exit ${status} +fi diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 0f5ebabcb9..66067e4dfd 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -1,41 +1,37 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2021-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= - -################################################################################################### -# - compiler function ----------------------------------------------------------------------------- +# ################################################################################################## +# * compiler function ----------------------------------------------------------------------------- function(ConfigureTest) - set(options OPTIONAL DIST NN) - set(oneValueArgs NAME ) - set(multiValueArgs PATH TARGETS CONFIGURATIONS) + set(options OPTIONAL DIST NN) + set(oneValueArgs NAME) + set(multiValueArgs PATH TARGETS CONFIGURATIONS) - cmake_parse_arguments(ConfigureTest "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) + cmake_parse_arguments(ConfigureTest "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(TEST_NAME ${ConfigureTest_NAME}) + set(TEST_NAME ${ConfigureTest_NAME}) - add_executable(${TEST_NAME} ${ConfigureTest_PATH}) + add_executable(${TEST_NAME} ${ConfigureTest_PATH}) - message("TEST PATH: ${ConfigureTest_PATH}") + message("TEST PATH: ${ConfigureTest_PATH}") - target_link_libraries(${TEST_NAME} - PRIVATE - raft::raft + target_link_libraries( + ${TEST_NAME} + PRIVATE raft::raft $<$:raft::distance> $<$:raft::nn> GTest::gtest @@ -43,251 +39,251 @@ function(ConfigureTest) Threads::Threads $ $ - ) + ) - add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) - set_target_properties(${TEST_NAME} - PROPERTIES - # set target compile options - INSTALL_RPATH "\$ORIGIN/../../../lib" - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - ) + set_target_properties( + ${TEST_NAME} + PROPERTIES # set target compile options + INSTALL_RPATH "\$ORIGIN/../../../lib" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + ) - target_compile_options(${TEST_NAME} - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) + target_compile_options( + ${TEST_NAME} PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) - target_include_directories(${TEST_NAME} - PUBLIC "$" - ) + target_include_directories(${TEST_NAME} PUBLIC "$") - install( - TARGETS ${TEST_NAME} - COMPONENT testing - DESTINATION bin/gtests/libraft - EXCLUDE_FROM_ALL) + install( + TARGETS ${TEST_NAME} + COMPONENT testing + DESTINATION bin/gtests/libraft + EXCLUDE_FROM_ALL + ) endfunction() +# ################################################################################################## +# test sources ################################################################################## +# ################################################################################################## -################################################################################################### -### test sources ################################################################################## -################################################################################################### - -################################################################################################### -# - distance tests ------------------------------------------------------------------------- +# ################################################################################################## +# * distance tests ------------------------------------------------------------------------- if(BUILD_TESTS) - ConfigureTest(NAME CLUSTER_TEST - PATH - test/cluster/kmeans.cu - test/cluster_solvers.cu - test/cluster/linkage.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME CLUSTER_TEST PATH test/cluster/kmeans.cu test/cluster_solvers.cu test/cluster/linkage.cu + OPTIONAL DIST NN + ) - ConfigureTest(NAME CORE_TEST - PATH - test/common/logger.cpp - test/handle.cpp - test/interruptible.cu - test/nvtx.cpp - test/mdarray.cu - test/mdspan_utils.cu - test/memory_type.cpp - test/span.cpp - test/span.cu - test/test.cpp - ) + ConfigureTest( + NAME + CORE_TEST + PATH + test/common/logger.cpp + test/handle.cpp + test/interruptible.cu + test/nvtx.cpp + test/mdarray.cu + test/mdspan_utils.cu + test/memory_type.cpp + test/span.cpp + test/span.cu + test/test.cpp + ) - ConfigureTest(NAME DISTANCE_TEST - PATH - test/distance/dist_adj.cu - test/distance/dist_canberra.cu - test/distance/dist_chebyshev.cu - test/distance/dist_correlation.cu - test/distance/dist_cos.cu - test/distance/dist_euc_exp.cu - test/distance/dist_euc_unexp.cu - test/distance/dist_hamming.cu - test/distance/dist_hellinger.cu - test/distance/dist_jensen_shannon.cu - test/distance/dist_kl_divergence.cu - test/distance/dist_l1.cu - test/distance/dist_minkowski.cu - test/distance/dist_russell_rao.cu - test/distance/fused_l2_nn.cu - test/distance/gram.cu - OPTIONAL DIST - ) + ConfigureTest( + NAME + DISTANCE_TEST + PATH + test/distance/dist_adj.cu + test/distance/dist_canberra.cu + test/distance/dist_chebyshev.cu + test/distance/dist_correlation.cu + test/distance/dist_cos.cu + test/distance/dist_euc_exp.cu + test/distance/dist_euc_unexp.cu + test/distance/dist_hamming.cu + test/distance/dist_hellinger.cu + test/distance/dist_jensen_shannon.cu + test/distance/dist_kl_divergence.cu + test/distance/dist_l1.cu + test/distance/dist_minkowski.cu + test/distance/dist_russell_rao.cu + test/distance/fused_l2_nn.cu + test/distance/gram.cu + OPTIONAL + DIST + ) - ConfigureTest(NAME LABEL_TEST - PATH - test/label/label.cu - test/label/merge_labels.cu - ) + ConfigureTest(NAME LABEL_TEST PATH test/label/label.cu test/label/merge_labels.cu) - ConfigureTest(NAME LINALG_TEST - PATH - test/linalg/add.cu - test/linalg/axpy.cu - test/linalg/binary_op.cu - test/linalg/cholesky_r1.cu - test/linalg/coalesced_reduction.cu - test/linalg/divide.cu - test/linalg/dot.cu - test/linalg/eig.cu - test/linalg/eig_sel.cu - test/linalg/gemm_layout.cu - test/linalg/gemv.cu - test/linalg/map.cu - test/linalg/map_then_reduce.cu - test/linalg/matrix_vector.cu - test/linalg/matrix_vector_op.cu - test/linalg/mean_squared_error.cu - test/linalg/multiply.cu - test/linalg/norm.cu - test/linalg/power.cu - test/linalg/reduce.cu - test/linalg/reduce_cols_by_key.cu - test/linalg/reduce_rows_by_key.cu - test/linalg/rsvd.cu - test/linalg/sqrt.cu - test/linalg/strided_reduction.cu - test/linalg/subtract.cu - test/linalg/svd.cu - test/linalg/ternary_op.cu - test/linalg/transpose.cu - test/linalg/unary_op.cu - ) + ConfigureTest( + NAME + LINALG_TEST + PATH + test/linalg/add.cu + test/linalg/axpy.cu + test/linalg/binary_op.cu + test/linalg/cholesky_r1.cu + test/linalg/coalesced_reduction.cu + test/linalg/divide.cu + test/linalg/dot.cu + test/linalg/eig.cu + test/linalg/eig_sel.cu + test/linalg/gemm_layout.cu + test/linalg/gemv.cu + test/linalg/map.cu + test/linalg/map_then_reduce.cu + test/linalg/matrix_vector.cu + test/linalg/matrix_vector_op.cu + test/linalg/mean_squared_error.cu + test/linalg/multiply.cu + test/linalg/norm.cu + test/linalg/power.cu + test/linalg/reduce.cu + test/linalg/reduce_cols_by_key.cu + test/linalg/reduce_rows_by_key.cu + test/linalg/rsvd.cu + test/linalg/sqrt.cu + test/linalg/strided_reduction.cu + test/linalg/subtract.cu + test/linalg/svd.cu + test/linalg/ternary_op.cu + test/linalg/transpose.cu + test/linalg/unary_op.cu + ) - ConfigureTest(NAME MATRIX_TEST - PATH - test/matrix/argmax.cu - test/matrix/argmin.cu - test/matrix/columnSort.cu - test/matrix/diagonal.cu - test/matrix/gather.cu - test/matrix/linewise_op.cu - test/matrix/math.cu - test/matrix/matrix.cu - test/matrix/norm.cu - test/matrix/reverse.cu - test/matrix/slice.cu - test/matrix/triangular.cu - test/spectral_matrix.cu - ) + ConfigureTest( + NAME + MATRIX_TEST + PATH + test/matrix/argmax.cu + test/matrix/argmin.cu + test/matrix/columnSort.cu + test/matrix/diagonal.cu + test/matrix/gather.cu + test/matrix/linewise_op.cu + test/matrix/math.cu + test/matrix/matrix.cu + test/matrix/norm.cu + test/matrix/reverse.cu + test/matrix/slice.cu + test/matrix/triangular.cu + test/spectral_matrix.cu + ) - ConfigureTest(NAME RANDOM_TEST - PATH - test/random/make_blobs.cu - test/random/make_regression.cu - test/random/multi_variable_gaussian.cu - test/random/permute.cu - test/random/rng.cu - test/random/rng_int.cu - test/random/rmat_rectangular_generator.cu - test/random/sample_without_replacement.cu - ) + ConfigureTest( + NAME + RANDOM_TEST + PATH + test/random/make_blobs.cu + test/random/make_regression.cu + test/random/multi_variable_gaussian.cu + test/random/permute.cu + test/random/rng.cu + test/random/rng_int.cu + test/random/rmat_rectangular_generator.cu + test/random/sample_without_replacement.cu + ) - ConfigureTest(NAME SOLVERS_TEST - PATH - test/cluster_solvers_deprecated.cu - test/eigen_solvers.cu - test/lap/lap.cu - test/mst.cu - ) + ConfigureTest( + NAME SOLVERS_TEST PATH test/cluster_solvers_deprecated.cu test/eigen_solvers.cu test/lap/lap.cu + test/mst.cu + ) - ConfigureTest(NAME SPARSE_TEST - PATH - test/sparse/add.cu - test/sparse/convert_coo.cu - test/sparse/convert_csr.cu - test/sparse/csr_row_slice.cu - test/sparse/csr_to_dense.cu - test/sparse/csr_transpose.cu - test/sparse/degree.cu - test/sparse/filter.cu - test/sparse/norm.cu - test/sparse/reduce.cu - test/sparse/row_op.cu - test/sparse/sort.cu - test/sparse/symmetrize.cu - ) + ConfigureTest( + NAME + SPARSE_TEST + PATH + test/sparse/add.cu + test/sparse/convert_coo.cu + test/sparse/convert_csr.cu + test/sparse/csr_row_slice.cu + test/sparse/csr_to_dense.cu + test/sparse/csr_transpose.cu + test/sparse/degree.cu + test/sparse/filter.cu + test/sparse/norm.cu + test/sparse/reduce.cu + test/sparse/row_op.cu + test/sparse/sort.cu + test/sparse/symmetrize.cu + ) - ConfigureTest(NAME SPARSE_DIST_TEST - PATH - test/sparse/dist_coo_spmv.cu - test/sparse/distance.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME SPARSE_DIST_TEST PATH test/sparse/dist_coo_spmv.cu test/sparse/distance.cu OPTIONAL DIST + NN + ) - ConfigureTest(NAME SPARSE_NEIGHBORS_TEST - PATH - test/sparse/neighbors/connect_components.cu - test/sparse/neighbors/brute_force.cu - test/sparse/neighbors/knn_graph.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME SPARSE_NEIGHBORS_TEST PATH test/sparse/neighbors/connect_components.cu + test/sparse/neighbors/brute_force.cu test/sparse/neighbors/knn_graph.cu OPTIONAL DIST NN + ) - ConfigureTest(NAME NEIGHBORS_TEST - PATH - test/neighbors/ann_ivf_flat.cu - test/neighbors/ann_ivf_pq/test_float_int64_t.cu - test/neighbors/ann_ivf_pq/test_float_uint32_t.cu - test/neighbors/ann_ivf_pq/test_float_uint64_t.cu - test/neighbors/ann_ivf_pq/test_int8_t_uint64_t.cu - test/neighbors/ann_ivf_pq/test_uint8_t_uint64_t.cu - test/neighbors/knn.cu - test/neighbors/fused_l2_knn.cu - test/neighbors/haversine.cu - test/neighbors/ball_cover.cu - test/neighbors/epsilon_neighborhood.cu - test/neighbors/faiss_mr.cu - test/neighbors/selection.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME + NEIGHBORS_TEST + PATH + test/neighbors/ann_ivf_flat.cu + test/neighbors/ann_ivf_pq/test_float_int64_t.cu + test/neighbors/ann_ivf_pq/test_float_uint32_t.cu + test/neighbors/ann_ivf_pq/test_float_uint64_t.cu + test/neighbors/ann_ivf_pq/test_int8_t_uint64_t.cu + test/neighbors/ann_ivf_pq/test_uint8_t_uint64_t.cu + test/neighbors/knn.cu + test/neighbors/fused_l2_knn.cu + test/neighbors/haversine.cu + test/neighbors/ball_cover.cu + test/neighbors/epsilon_neighborhood.cu + test/neighbors/faiss_mr.cu + test/neighbors/selection.cu + OPTIONAL + DIST + NN + ) - ConfigureTest(NAME STATS_TEST - PATH - test/stats/accuracy.cu - test/stats/adjusted_rand_index.cu - test/stats/completeness_score.cu - test/stats/contingencyMatrix.cu - test/stats/cov.cu - test/stats/dispersion.cu - test/stats/entropy.cu - test/stats/histogram.cu - test/stats/homogeneity_score.cu - test/stats/information_criterion.cu - test/stats/kl_divergence.cu - test/stats/mean.cu - test/stats/meanvar.cu - test/stats/mean_center.cu - test/stats/minmax.cu - test/stats/mutual_info_score.cu - test/stats/r2_score.cu - test/stats/rand_index.cu - test/stats/regression_metrics.cu - test/stats/silhouette_score.cu - test/stats/stddev.cu - test/stats/sum.cu - test/stats/trustworthiness.cu - test/stats/weighted_mean.cu - test/stats/v_measure.cu - OPTIONAL DIST NN - ) + ConfigureTest( + NAME + STATS_TEST + PATH + test/stats/accuracy.cu + test/stats/adjusted_rand_index.cu + test/stats/completeness_score.cu + test/stats/contingencyMatrix.cu + test/stats/cov.cu + test/stats/dispersion.cu + test/stats/entropy.cu + test/stats/histogram.cu + test/stats/homogeneity_score.cu + test/stats/information_criterion.cu + test/stats/kl_divergence.cu + test/stats/mean.cu + test/stats/meanvar.cu + test/stats/mean_center.cu + test/stats/minmax.cu + test/stats/mutual_info_score.cu + test/stats/r2_score.cu + test/stats/rand_index.cu + test/stats/regression_metrics.cu + test/stats/silhouette_score.cu + test/stats/stddev.cu + test/stats/sum.cu + test/stats/trustworthiness.cu + test/stats/weighted_mean.cu + test/stats/v_measure.cu + OPTIONAL + DIST + NN + ) - ConfigureTest(NAME UTILS_TEST - PATH - test/common/seive.cu - test/cudart_utils.cpp - test/device_atomics.cu - test/integer_utils.cpp - test/pow2_utils.cu - ) + ConfigureTest( + NAME UTILS_TEST PATH test/common/seive.cu test/cudart_utils.cpp test/device_atomics.cu + test/integer_utils.cpp test/pow2_utils.cu + ) endif() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..dfd22f3378 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[tool.black] +line-length = 79 +target-version = ["py38"] +include = '\.py?$' +force-exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/.flake8 b/python/.flake8 deleted file mode 100644 index ef2e5a8495..0000000000 --- a/python/.flake8 +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -[flake8] -exclude = __init__.py -ignore = - # line break before binary operator - W503 - # whitespace before : - E203 \ No newline at end of file diff --git a/python/.flake8.cython b/python/.flake8.cython deleted file mode 100644 index 3cd436d3f3..0000000000 --- a/python/.flake8.cython +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -[flake8] -filename = *.pyx, *.pxd -exclude = *.egg, build, docs, .git -ignore = E999, E225, E226, E227, W503, W504 - -# Rules ignored: -# E999: invalid syntax (works for Python, not Cython) -# E225: Missing whitespace around operators (breaks cython casting syntax like ) -# E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) -# E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) -# W503: line break before binary operator (breaks lines that start with a pointer) -# W504: line break after binary operator (breaks lines that end with a pointer) diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 7432b98e30..faea60bdb1 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -25,18 +25,23 @@ project( # language to be enabled here. The test project that is built in scikit-build to verify # various linking options for the python library is hardcoded to build with C, so until # that is fixed we need to keep C. - C - CXX) + C CXX +) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" ON) +option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" + ON +) # If the user requested it we attempt to find RAFT. if(FIND_RAFT_CPP) - find_package(raft ${pylibraft_version} REQUIRED COMPONENTS distance) - if(NOT TARGET raft::raft_distance_lib) - message(FATAL_ERROR "Building against a preexisting libraft library requires the distance components of that library to have been built!") + find_package(raft ${pylibraft_version} REQUIRED COMPONENTS distance) + if(NOT TARGET raft::raft_distance_lib) + message( + FATAL_ERROR + "Building against a preexisting libraft library requires the distance components of that library to have been built!" + ) - endif() + endif() else() set(raft_FOUND OFF) endif() @@ -44,9 +49,8 @@ endif() include(rapids-cython) if(NOT raft_FOUND) - # TODO: This will not be necessary once we upgrade to CMake 3.22, which will - # pull in the required languages for the C++ project even if this project - # does not require those languges. + # TODO: This will not be necessary once we upgrade to CMake 3.22, which will pull in the required + # languages for the C++ project even if this project does not require those languges. include(rapids-cuda) rapids_cuda_init_architectures(pylibraft) enable_language(CUDA) @@ -60,9 +64,9 @@ if(NOT raft_FOUND) set(RAFT_COMPILE_DIST_LIBRARY ON) add_subdirectory(../../cpp raft-cpp) - # When building the C++ libraries from source we must copy - # libraft_distance.so alongside the pairwise_distance and random Cython libraries - # TODO: when we have a single 'compiled' raft library, we shouldn't need this + # When building the C++ libraries from source we must copy libraft_distance.so alongside the + # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft + # library, we shouldn't need this set(cython_lib_dir pylibraft) install(TARGETS raft_distance_lib DESTINATION ${cython_lib_dir}) endif() diff --git a/python/pylibraft/pylibraft/_version.py b/python/pylibraft/pylibraft/_version.py index 58cd44da3b..81de84e96e 100644 --- a/python/pylibraft/pylibraft/_version.py +++ b/python/pylibraft/pylibraft/_version.py @@ -70,7 +70,7 @@ def decorate(f): def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None ): """Call the given command(s).""" assert isinstance(commands, list) @@ -85,7 +85,7 @@ def run_command( env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), - ) + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -123,7 +123,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return { - "version": dirname[len(parentdir_prefix):], + "version": dirname[len(parentdir_prefix) :], "full-revisionid": None, "dirty": False, "error": None, @@ -193,7 +193,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -210,7 +210,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) return { @@ -264,7 +264,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): "--long", "--match", "%s*" % tag_prefix, - ], + ], cwd=root, ) # --long was added in git-1.5.5 @@ -299,7 +299,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if not mo: # unparseable. Maybe git-describe is misbehaving? pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out + "unable to parse git-describe output: '%s'" % describe_out ) return pieces @@ -314,7 +314,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): tag_prefix, ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index 44e34e0213..497fb52735 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -18,11 +18,11 @@ set(linked_libraries raft::raft raft::distance) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - MODULE_PREFIX cluster_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX cluster_ +) foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) - set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") endforeach() diff --git a/python/pylibraft/pylibraft/cluster/kmeans.pyx b/python/pylibraft/pylibraft/cluster/kmeans.pyx index 940d118c3b..732a78585d 100644 --- a/python/pylibraft/pylibraft/cluster/kmeans.pyx +++ b/python/pylibraft/pylibraft/cluster/kmeans.pyx @@ -20,15 +20,14 @@ import numpy as np -from libc.stdint cimport uintptr_t from cython.operator cimport dereference as deref - -from libcpp cimport bool -from libcpp cimport nullptr +from libc.stdint cimport uintptr_t +from libcpp cimport bool, nullptr from pylibraft.common import Handle from pylibraft.common.handle import auto_sync_handle from pylibraft.common.handle cimport handle_t + from pylibraft.common.input_validation import * from pylibraft.distance import DISTANCE_TYPES diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index f8d86e0386..3b49cef429 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -18,8 +18,7 @@ set(linked_libraries raft::raft) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - ASSOCIATED_TARGETS raft - MODULE_PREFIX common_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ +) diff --git a/python/pylibraft/pylibraft/common/cuda.pxd b/python/pylibraft/pylibraft/common/cuda.pxd index ae6246dee1..a44d9aeb63 100644 --- a/python/pylibraft/pylibraft/common/cuda.pxd +++ b/python/pylibraft/pylibraft/common/cuda.pxd @@ -16,6 +16,7 @@ from cuda.ccudart cimport cudaStream_t + cdef class Stream: cdef cudaStream_t s diff --git a/python/pylibraft/pylibraft/common/cuda.pyx b/python/pylibraft/pylibraft/common/cuda.pyx index 9b35aebdba..606860dbe9 100644 --- a/python/pylibraft/pylibraft/common/cuda.pyx +++ b/python/pylibraft/pylibraft/common/cuda.pyx @@ -19,16 +19,16 @@ # cython: embedsignature = True # cython: language_level = 3 -from cuda.ccudart cimport( - cudaStream_t, +from cuda.ccudart cimport ( cudaError_t, - cudaSuccess, + cudaGetErrorName, + cudaGetErrorString, + cudaGetLastError, + cudaStream_t, cudaStreamCreate, cudaStreamDestroy, cudaStreamSynchronize, - cudaGetLastError, - cudaGetErrorString, - cudaGetErrorName + cudaSuccess, ) diff --git a/python/pylibraft/pylibraft/common/handle.pxd b/python/pylibraft/pylibraft/common/handle.pxd index d7abe236ba..e763768eac 100644 --- a/python/pylibraft/pylibraft/common/handle.pxd +++ b/python/pylibraft/pylibraft/common/handle.pxd @@ -20,11 +20,11 @@ # cython: language_level = 3 -from libcpp.memory cimport shared_ptr -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from libcpp.memory cimport shared_ptr, unique_ptr + from rmm._lib.cuda_stream_pool cimport cuda_stream_pool -from libcpp.memory cimport shared_ptr -from libcpp.memory cimport unique_ptr +from rmm._lib.cuda_stream_view cimport cuda_stream_view + cdef extern from "raft/core/handle.hpp" namespace "raft" nogil: cdef cppclass handle_t: diff --git a/python/pylibraft/pylibraft/common/handle.pyx b/python/pylibraft/pylibraft/common/handle.pyx index 3e9ed569ad..c14c22f5aa 100644 --- a/python/pylibraft/pylibraft/common/handle.pyx +++ b/python/pylibraft/pylibraft/common/handle.pyx @@ -21,10 +21,10 @@ import functools -from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread -from rmm._lib.cuda_stream_view cimport cuda_stream_view +from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread, cuda_stream_view from .cuda cimport Stream + from .cuda import CudaRuntimeError diff --git a/python/pylibraft/pylibraft/common/interruptible.pxd b/python/pylibraft/pylibraft/common/interruptible.pxd index a4c7d90ac2..aaccf8aeab 100644 --- a/python/pylibraft/pylibraft/common/interruptible.pxd +++ b/python/pylibraft/pylibraft/common/interruptible.pxd @@ -20,8 +20,10 @@ # cython: language_level = 3 from libcpp.memory cimport shared_ptr + from rmm._lib.cuda_stream_view cimport cuda_stream_view + cdef extern from "raft/core/interruptible.hpp" namespace "raft" nogil: cdef cppclass interruptible: void cancel() diff --git a/python/pylibraft/pylibraft/common/interruptible.pyx b/python/pylibraft/pylibraft/common/interruptible.pyx index dfc95490ed..fc2e6d9e1f 100644 --- a/python/pylibraft/pylibraft/common/interruptible.pyx +++ b/python/pylibraft/pylibraft/common/interruptible.pyx @@ -21,10 +21,12 @@ import contextlib import signal + +from cuda.ccudart cimport cudaStream_t from cython.operator cimport dereference from rmm._lib.cuda_stream_view cimport cuda_stream_view -from cuda.ccudart cimport cudaStream_t + from .cuda cimport Stream diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index 01cdbd0876..cae00007d6 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -13,14 +13,12 @@ # ============================================================================= # Set the list of Cython files to build -set(cython_sources pairwise_distance.pyx - fused_l2_nn.pyx) +set(cython_sources pairwise_distance.pyx fused_l2_nn.pyx) set(linked_libraries raft::raft raft::distance) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - ASSOCIATED_TARGETS raft - MODULE_PREFIX distance_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ +) diff --git a/python/pylibraft/pylibraft/distance/__init__.py b/python/pylibraft/pylibraft/distance/__init__.py index 7d68f02936..b251e71ba3 100644 --- a/python/pylibraft/pylibraft/distance/__init__.py +++ b/python/pylibraft/pylibraft/distance/__init__.py @@ -14,5 +14,4 @@ # from .fused_l2_nn import fused_l2_nn_argmin -from .pairwise_distance import distance as pairwise_distance -from .pairwise_distance import DISTANCE_TYPES \ No newline at end of file +from .pairwise_distance import DISTANCE_TYPES, distance as pairwise_distance diff --git a/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx b/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx index 75b9670005..73cd60058f 100644 --- a/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx +++ b/python/pylibraft/pylibraft/distance/fused_l2_nn.pyx @@ -20,11 +20,12 @@ import numpy as np -from libc.stdint cimport uintptr_t from cython.operator cimport dereference as deref - +from libc.stdint cimport uintptr_t from libcpp cimport bool + from .distance_type cimport DistanceType + from pylibraft.common import Handle from pylibraft.common.handle import auto_sync_handle from pylibraft.common.handle cimport handle_t diff --git a/python/pylibraft/pylibraft/distance/pairwise_distance.pyx b/python/pylibraft/pylibraft/distance/pairwise_distance.pyx index b6b73a86a1..76cdf0b2d3 100644 --- a/python/pylibraft/pylibraft/distance/pairwise_distance.pyx +++ b/python/pylibraft/pylibraft/distance/pairwise_distance.pyx @@ -20,10 +20,10 @@ import numpy as np -from libc.stdint cimport uintptr_t from cython.operator cimport dereference as deref - +from libc.stdint cimport uintptr_t from libcpp cimport bool + from .distance_type cimport DistanceType from pylibraft.common import Handle diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index c44b0cea67..f6789cc894 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -14,14 +14,13 @@ # Set the list of Cython files to build set(cython_sources rmat_rectangular_generator.pyx) -# TODO: should finally be replaced with 'compiled' library to be more generic, -# when that is available +# TODO: should finally be replaced with 'compiled' library to be more generic, when that is +# available set(linked_libraries raft::raft raft::distance) # Build all of the Cython targets rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - ASSOCIATED_TARGETS raft - MODULE_PREFIX random_) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ +) diff --git a/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx b/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx index 56359ff8a3..8698ab6539 100644 --- a/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx +++ b/python/pylibraft/pylibraft/random/rmat_rectangular_generator.pyx @@ -20,15 +20,18 @@ import numpy as np -from libc.stdint cimport uintptr_t, int64_t from cython.operator cimport dereference as deref +from libc.stdint cimport int64_t, uintptr_t + from pylibraft.common import Handle from pylibraft.common.handle import auto_sync_handle -from pylibraft.common.handle cimport handle_t -from .rng_state cimport RngState from libcpp cimport bool +from pylibraft.common.handle cimport handle_t + +from .rng_state cimport RngState + cdef extern from "raft_distance/random/rmat_rectangular_generator.hpp" \ namespace "raft::random::runtime": diff --git a/python/pylibraft/pylibraft/random/rng_state.pxd b/python/pylibraft/pylibraft/random/rng_state.pxd index 9d1e2d17e3..8c89db3618 100644 --- a/python/pylibraft/pylibraft/random/rng_state.pxd +++ b/python/pylibraft/pylibraft/random/rng_state.pxd @@ -20,6 +20,7 @@ from libc.stdint cimport uint64_t + cdef extern from "raft/random/rng_state.hpp" namespace "raft::random" nogil: ctypedef enum GeneratorType: diff --git a/python/pylibraft/pylibraft/test/test_distance.py b/python/pylibraft/pylibraft/test/test_distance.py index 7f35a25493..670beb156e 100644 --- a/python/pylibraft/pylibraft/test/test_distance.py +++ b/python/pylibraft/pylibraft/test/test_distance.py @@ -13,22 +13,32 @@ # limitations under the License. # -from scipy.spatial.distance import cdist -import pytest import numpy as np +import pytest +from scipy.spatial.distance import cdist from pylibraft.common import Handle from pylibraft.distance import pairwise_distance - from pylibraft.testing.utils import TestDeviceBuffer @pytest.mark.parametrize("n_rows", [100]) @pytest.mark.parametrize("n_cols", [100]) -@pytest.mark.parametrize("metric", ["euclidean", "cityblock", "chebyshev", - "canberra", "correlation", "hamming", - "jensenshannon", "russellrao", "cosine", - "sqeuclidean"]) +@pytest.mark.parametrize( + "metric", + [ + "euclidean", + "cityblock", + "chebyshev", + "canberra", + "correlation", + "hamming", + "jensenshannon", + "russellrao", + "cosine", + "sqeuclidean", + ], +) @pytest.mark.parametrize("order", ["F", "C"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_distance(n_rows, n_cols, metric, order, dtype): diff --git a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py b/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py index 1ce1ee2d1f..abe56f2b04 100644 --- a/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py +++ b/python/pylibraft/pylibraft/test/test_fused_l2_argmin.py @@ -13,9 +13,9 @@ # limitations under the License. # -from scipy.spatial.distance import cdist -import pytest import numpy as np +import pytest +from scipy.spatial.distance import cdist from pylibraft.common import Handle from pylibraft.distance import fused_l2_nn_argmin @@ -43,8 +43,9 @@ def test_fused_l2_nn_minarg(n_rows, n_cols, n_clusters, dtype): output_device = TestDeviceBuffer(output, "C") handle = Handle() - fused_l2_nn_argmin(input1_device, input2_device, output_device, - True, handle=handle) + fused_l2_nn_argmin( + input1_device, input2_device, output_device, True, handle=handle + ) handle.sync() actual = output_device.copy_to_host() diff --git a/python/pylibraft/pylibraft/test/test_kmeans.py b/python/pylibraft/pylibraft/test/test_kmeans.py index 1d65470a82..d198ac2f8f 100644 --- a/python/pylibraft/pylibraft/test/test_kmeans.py +++ b/python/pylibraft/pylibraft/test/test_kmeans.py @@ -13,13 +13,12 @@ # limitations under the License. # -import pytest import numpy as np +import pytest -from pylibraft.common import Handle from pylibraft.cluster.kmeans import compute_new_centroids +from pylibraft.common import Handle from pylibraft.distance import pairwise_distance - from pylibraft.testing.utils import TestDeviceBuffer @@ -29,8 +28,9 @@ @pytest.mark.parametrize("metric", ["euclidean", "sqeuclidean"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) @pytest.mark.parametrize("additional_args", [True, False]) -def test_compute_new_centroids(n_rows, n_cols, metric, n_clusters, dtype, - additional_args): +def test_compute_new_centroids( + n_rows, n_cols, metric, n_clusters, dtype, additional_args +): order = "C" @@ -44,16 +44,20 @@ def test_compute_new_centroids(n_rows, n_cols, metric, n_clusters, dtype, centroids = X[:n_clusters] centroids_device = TestDeviceBuffer(centroids, order) - weight_per_cluster = np.zeros((n_clusters, ), dtype=dtype) - weight_per_cluster_device = TestDeviceBuffer(weight_per_cluster, order) \ - if additional_args else None + weight_per_cluster = np.zeros((n_clusters,), dtype=dtype) + weight_per_cluster_device = ( + TestDeviceBuffer(weight_per_cluster, order) + if additional_args + else None + ) new_centroids = np.zeros((n_clusters, n_cols), dtype=dtype) new_centroids_device = TestDeviceBuffer(new_centroids, order) sample_weights = np.ones((n_rows,)).astype(dtype) / n_rows - sample_weights_device = TestDeviceBuffer(sample_weights, order) \ - if additional_args else None + sample_weights_device = ( + TestDeviceBuffer(sample_weights, order) if additional_args else None + ) # Compute new centroids naively dists = np.zeros((n_rows, n_clusters), dtype=dtype) @@ -72,13 +76,15 @@ def test_compute_new_centroids(n_rows, n_cols, metric, n_clusters, dtype, g = sample_weights[labels == i].sum() expected_centers[i, :] = j / g - compute_new_centroids(X_device, - centroids_device, - labels_device, - new_centroids_device, - sample_weights=sample_weights_device, - weight_per_cluster=weight_per_cluster_device, - handle=handle) + compute_new_centroids( + X_device, + centroids_device, + labels_device, + new_centroids_device, + sample_weights=sample_weights_device, + weight_per_cluster=weight_per_cluster_device, + handle=handle, + ) # pylibraft functions are often asynchronous so the # handle needs to be explicitly synchronized diff --git a/python/pylibraft/pylibraft/test/test_random.py b/python/pylibraft/pylibraft/test/test_random.py index e0b7140f1c..77494ea277 100644 --- a/python/pylibraft/pylibraft/test/test_random.py +++ b/python/pylibraft/pylibraft/test/test_random.py @@ -13,12 +13,11 @@ # limitations under the License. # -import pytest import numpy as np +import pytest from pylibraft.common import Handle from pylibraft.random import rmat - from pylibraft.testing.utils import TestDeviceBuffer diff --git a/python/pylibraft/pylibraft/test/test_z_interruptible.py b/python/pylibraft/pylibraft/test/test_z_interruptible.py index bda98edd13..aac428b2d5 100644 --- a/python/pylibraft/pylibraft/test/test_z_interruptible.py +++ b/python/pylibraft/pylibraft/test/test_z_interruptible.py @@ -1,9 +1,11 @@ # Copyright (c) 2022, NVIDIA CORPORATION. import os -import pytest import signal import time + +import pytest + from pylibraft.common.interruptible import cuda_interruptible, cuda_yield @@ -14,14 +16,15 @@ def send_ctrl_c(): def test_should_cancel_via_interruptible(): start_time = time.monotonic() - with pytest.raises(RuntimeError, match='this thread was cancelled'): + with pytest.raises(RuntimeError, match="this thread was cancelled"): with cuda_interruptible(): send_ctrl_c() cuda_yield() time.sleep(1.0) end_time = time.monotonic() - assert end_time < start_time + 0.5, \ - "The process seems to have waited, while it shouldn't have." + assert ( + end_time < start_time + 0.5 + ), "The process seems to have waited, while it shouldn't have." def test_should_cancel_via_python(): @@ -31,8 +34,9 @@ def test_should_cancel_via_python(): cuda_yield() time.sleep(1.0) end_time = time.monotonic() - assert end_time < start_time + 0.5, \ - "The process seems to have waited, while it shouldn't have." + assert ( + end_time < start_time + 0.5 + ), "The process seems to have waited, while it shouldn't have." def test_should_wait_no_interrupt(): @@ -41,8 +45,9 @@ def test_should_wait_no_interrupt(): cuda_yield() time.sleep(1.0) end_time = time.monotonic() - assert end_time > start_time + 0.5, \ - "The process seems to be cancelled, while it shouldn't be." + assert ( + end_time > start_time + 0.5 + ), "The process seems to be cancelled, while it shouldn't be." def test_should_wait_no_yield(): @@ -51,5 +56,6 @@ def test_should_wait_no_yield(): send_ctrl_c() time.sleep(1.0) end_time = time.monotonic() - assert end_time > start_time + 0.5, \ - "The process seems to be cancelled, while it shouldn't be." + assert ( + end_time > start_time + 0.5 + ), "The process seems to be cancelled, while it shouldn't be." diff --git a/python/pylibraft/pylibraft/testing/utils.py b/python/pylibraft/pylibraft/testing/utils.py index 979fbb5672..86cf4558db 100644 --- a/python/pylibraft/pylibraft/testing/utils.py +++ b/python/pylibraft/pylibraft/testing/utils.py @@ -19,12 +19,12 @@ class TestDeviceBuffer: - def __init__(self, ndarray, order): self.ndarray_ = ndarray - self.device_buffer_ = \ - rmm.DeviceBuffer.to_device(ndarray.ravel(order=order).tobytes()) + self.device_buffer_ = rmm.DeviceBuffer.to_device( + ndarray.ravel(order=order).tobytes() + ) @property def __cuda_array_interface__(self): @@ -35,8 +35,12 @@ def __cuda_array_interface__(self): return host_cai def copy_to_host(self): - return np.frombuffer(self.device_buffer_.tobytes(), - dtype=self.ndarray_.dtype, - like=self.ndarray_)\ - .astype(self.ndarray_.dtype)\ + return ( + np.frombuffer( + self.device_buffer_.tobytes(), + dtype=self.ndarray_.dtype, + like=self.ndarray_, + ) + .astype(self.ndarray_.dtype) .reshape(self.ndarray_.shape) + ) diff --git a/python/pylibraft/setup.cfg b/python/pylibraft/setup.cfg index e1f4865ac9..67cc148155 100644 --- a/python/pylibraft/setup.cfg +++ b/python/pylibraft/setup.cfg @@ -1,11 +1,5 @@ # Copyright (c) 2022, NVIDIA CORPORATION. -[flake8] -exclude = __init__.py,versioneer.py -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - [versioneer] VCS = git style = pep440 @@ -34,6 +28,7 @@ known_rapids= rmm known_first_party= raft + pylibraft default_section=THIRDPARTY sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER skip= diff --git a/python/pylibraft/setup.py b/python/pylibraft/setup.py index 561c105d36..289214e856 100644 --- a/python/pylibraft/setup.py +++ b/python/pylibraft/setup.py @@ -14,40 +14,40 @@ # limitations under the License. # +import versioneer from setuptools import find_packages from skbuild import setup -import versioneer - -setup(name='pylibraft', - description="RAFT: Reusable Algorithms Functions and other Tools", - version=versioneer.get_version(), - classifiers=[ +setup( + name="pylibraft", + description="RAFT: Reusable Algorithms Functions and other Tools", + version=versioneer.get_version(), + classifiers=[ "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9" - ], - author="NVIDIA Corporation", - package_data={ - # Note: A dict comprehension with an explicit copy is necessary - # (rather than something simpler like a dict.fromkeys) because - # otherwise every package will refer to the same list and skbuild - # modifies it in place. - key: ["*.hpp", "*.pxd"] - for key in find_packages( - include=[ - "pylibraft.distance", - "pylibraft.distance.includes", - "pylibraft.common", - "pylibraft.common.includes", - "pylibraft.random", - "pylibraft.random.includes" - ] - ) - }, - packages=find_packages(include=['pylibraft', 'pylibraft.*']), - license="Apache", - cmdclass=versioneer.get_cmdclass(), - zip_safe=False - ) + "Programming Language :: Python :: 3.9", + ], + author="NVIDIA Corporation", + package_data={ + # Note: A dict comprehension with an explicit copy is necessary + # (rather than something simpler like a dict.fromkeys) because + # otherwise every package will refer to the same list and skbuild + # modifies it in place. + key: ["*.hpp", "*.pxd"] + for key in find_packages( + include=[ + "pylibraft.distance", + "pylibraft.distance.includes", + "pylibraft.common", + "pylibraft.common.includes", + "pylibraft.random", + "pylibraft.random.includes", + ] + ) + }, + packages=find_packages(include=["pylibraft", "pylibraft.*"]), + license="Apache", + cmdclass=versioneer.get_cmdclass(), + zip_safe=False, +) diff --git a/python/pylibraft/setuputils.py b/python/pylibraft/setuputils.py index d93e4b06a4..0a3f421856 100755 --- a/python/pylibraft/setuputils.py +++ b/python/pylibraft/setuputils.py @@ -51,15 +51,15 @@ def clean_folder(path): path : String Path to the folder to be cleaned. """ - shutil.rmtree(path + '/__pycache__', ignore_errors=True) + shutil.rmtree(path + "/__pycache__", ignore_errors=True) - folders = glob.glob(path + '/*/') + folders = glob.glob(path + "/*/") for folder in folders: - shutil.rmtree(folder + '/__pycache__', ignore_errors=True) + shutil.rmtree(folder + "/__pycache__", ignore_errors=True) clean_folder(folder) - cython_exts = glob.glob(folder + '/*.cpp') - cython_exts.extend(glob.glob(folder + '/*.cpython*')) + cython_exts = glob.glob(folder + "/*.cpp") + cython_exts.extend(glob.glob(folder + "/*.cpython*")) for file in cython_exts: os.remove(file) diff --git a/python/pylibraft/versioneer.py b/python/pylibraft/versioneer.py index b8c4bc423b..73e3259bf1 100644 --- a/python/pylibraft/versioneer.py +++ b/python/pylibraft/versioneer.py @@ -278,10 +278,12 @@ """ from __future__ import print_function + try: import configparser except ImportError: import ConfigParser as configparser + import errno import json import os @@ -309,11 +311,13 @@ def get_root(): setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -326,8 +330,10 @@ def get_root(): me_dir = os.path.normcase(os.path.splitext(me)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py) + ) except NameError: pass return root @@ -349,6 +355,7 @@ def get(parser, name): if parser.has_option("versioneer", name): return parser.get("versioneer", name) return None + cfg = VersioneerConfig() cfg.VCS = VCS cfg.style = get(parser, "style") or "" @@ -373,17 +380,20 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None +): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -391,10 +401,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -419,7 +432,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, p.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY[ + "git" +] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -994,7 +1009,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1003,7 +1018,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1011,19 +1026,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -1038,8 +1060,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command( + GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True + ) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1047,10 +1070,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1073,17 +1105,18 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = ( + "unable to parse git-describe output: '%s'" % describe_out + ) return pieces # tag @@ -1092,10 +1125,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1106,13 +1141,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command( + GITS, ["rev-list", "HEAD", "--count"], cwd=root + ) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1168,16 +1205,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1206,11 +1249,17 @@ def versions_from_file(filename): contents = f.read() except EnvironmentError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1219,8 +1268,9 @@ def versions_from_file(filename): def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps( + versions, sort_keys=True, indent=1, separators=(",", ": ") + ) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1252,8 +1302,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1367,11 +1416,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1391,9 +1442,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1416,8 +1471,9 @@ def get_versions(verbose=False): handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1471,9 +1527,13 @@ def get_versions(verbose=False): if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version(): @@ -1522,6 +1582,7 @@ def run(self): print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in both distutils and setuptools @@ -1554,14 +1615,17 @@ def run(self): # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join( + self.build_lib, cfg.versionfile_build + ) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -1582,17 +1646,21 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.distutils_buildexe import py2exe as _py2exe # py3 except ImportError: @@ -1611,13 +1679,17 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments @@ -1644,8 +1716,10 @@ def make_release_tree(self, base_dir, files): # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -1700,11 +1774,15 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except ( + EnvironmentError, + configparser.NoSectionError, + configparser.NoOptionError, + ) as e: if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print( + "Adding sample versioneer config to setup.cfg", file=sys.stderr + ) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -1713,15 +1791,18 @@ def do_setup(): print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: @@ -1763,8 +1844,10 @@ def do_setup(): else: print(" 'versioneer.py' already in MANIFEST.in") if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) + print( + " appending versionfile_source ('%s') to MANIFEST.in" + % cfg.versionfile_source + ) with open(manifest_in, "a") as f: f.write("include %s\n" % cfg.versionfile_source) else: diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 886371f222..a25b1b5b4a 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -25,23 +25,23 @@ project( # language to be enabled here. The test project that is built in scikit-build to verify # various linking options for the python library is hardcoded to build with C, so until # that is fixed we need to keep C. - C - CXX) + C CXX +) option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - OFF) + OFF +) # If the user requested it we attempt to find RAFT. if(FIND_RAFT_CPP) - find_package(raft ${raft_dask_version} REQUIRED) + find_package(raft ${raft_dask_version} REQUIRED) else() set(raft_FOUND OFF) endif() if(NOT raft_FOUND) - # TODO: This will not be necessary once we upgrade to CMake 3.22, which will - # pull in the required languages for the C++ project even if this project - # does not require those languges. + # TODO: This will not be necessary once we upgrade to CMake 3.22, which will pull in the required + # languages for the C++ project even if this project does not require those languges. include(rapids-cuda) rapids_cuda_init_architectures(raft-dask) enable_language(CUDA) @@ -49,9 +49,8 @@ if(NOT raft_FOUND) # rapids_cuda_init_architectures relies on `project` including. include("${CMAKE_PROJECT_raft_dask_INCLUDE}") - # raft-dask doesn't actually use raft libraries, it just needs the headers, so - # we can turn off all library compilation and we don't need to install - # anything here. + # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all + # library compilation and we don't need to install anything here. set(BUILD_TESTS OFF) set(BUILD_BENCH OFF) set(RAFT_COMPILE_LIBRARIES OFF) diff --git a/python/raft-dask/cmake/thirdparty/get_nccl.cmake b/python/raft-dask/cmake/thirdparty/get_nccl.cmake index c2cc17b399..bb4b0e4dae 100644 --- a/python/raft-dask/cmake/thirdparty/get_nccl.cmake +++ b/python/raft-dask/cmake/thirdparty/get_nccl.cmake @@ -1,39 +1,34 @@ -#============================================================================= +# ============================================================================= # Copyright (c) 2021-2022, NVIDIA CORPORATION. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= include(rapids-find) function(find_and_configure_nccl) - if(TARGET NCCL::NCCL) - return() - endif() + if(TARGET NCCL::NCCL) + return() + endif() - rapids_find_generate_module(NCCL - HEADER_NAMES nccl.h - LIBRARY_NAMES nccl - ) + rapids_find_generate_module( + NCCL + HEADER_NAMES nccl.h + LIBRARY_NAMES nccl + ) - # Currently NCCL has no CMake build-system so we require - # it built and installed on the machine already - rapids_find_package(NCCL REQUIRED) + # Currently NCCL has no CMake build-system so we require it built and installed on the machine + # already + rapids_find_package(NCCL REQUIRED) endfunction() find_and_configure_nccl() - - - - diff --git a/python/raft-dask/raft_dask/_version.py b/python/raft-dask/raft_dask/_version.py index 454b0fe7aa..4eb6056f17 100644 --- a/python/raft-dask/raft_dask/_version.py +++ b/python/raft-dask/raft_dask/_version.py @@ -70,7 +70,7 @@ def decorate(f): def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None ): """Call the given command(s).""" assert isinstance(commands, list) @@ -85,7 +85,7 @@ def run_command( env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), - ) + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -123,7 +123,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return { - "version": dirname[len(parentdir_prefix):], + "version": dirname[len(parentdir_prefix) :], "full-revisionid": None, "dirty": False, "error": None, @@ -193,7 +193,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -210,7 +210,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) return { @@ -264,7 +264,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): "--long", "--match", "%s*" % tag_prefix, - ], + ], cwd=root, ) # --long was added in git-1.5.5 @@ -299,7 +299,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if not mo: # unparseable. Maybe git-describe is misbehaving? pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out + "unable to parse git-describe output: '%s'" % describe_out ) return pieces @@ -314,7 +314,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): tag_prefix, ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 0e1da015fc..77b6695118 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -18,6 +18,5 @@ find_package(ucx REQUIRED) set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft NCCL::NCCL ucx::ucp) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - CXX) + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX +) diff --git a/python/raft-dask/raft_dask/common/__init__.py b/python/raft-dask/raft_dask/common/__init__.py index 8c25cdde90..c8ce695def 100644 --- a/python/raft-dask/raft_dask/common/__init__.py +++ b/python/raft-dask/raft_dask/common/__init__.py @@ -13,22 +13,21 @@ # limitations under the License. # -from .comms import Comms -from .comms import local_handle - -from .comms_utils import inject_comms_on_handle -from .comms_utils import inject_comms_on_handle_coll_only -from .comms_utils import perform_test_comms_allreduce -from .comms_utils import perform_test_comms_send_recv -from .comms_utils import perform_test_comms_device_send_or_recv -from .comms_utils import perform_test_comms_device_sendrecv -from .comms_utils import perform_test_comms_device_multicast_sendrecv -from .comms_utils import perform_test_comms_allgather -from .comms_utils import perform_test_comms_gather -from .comms_utils import perform_test_comms_gatherv -from .comms_utils import perform_test_comms_bcast -from .comms_utils import perform_test_comms_reduce -from .comms_utils import perform_test_comms_reducescatter -from .comms_utils import perform_test_comm_split - +from .comms import Comms, local_handle +from .comms_utils import ( + inject_comms_on_handle, + inject_comms_on_handle_coll_only, + perform_test_comm_split, + perform_test_comms_allgather, + perform_test_comms_allreduce, + perform_test_comms_bcast, + perform_test_comms_device_multicast_sendrecv, + perform_test_comms_device_send_or_recv, + perform_test_comms_device_sendrecv, + perform_test_comms_gather, + perform_test_comms_gatherv, + perform_test_comms_reduce, + perform_test_comms_reducescatter, + perform_test_comms_send_recv, +) from .ucx import UCX diff --git a/python/raft-dask/raft_dask/common/comms.py b/python/raft-dask/raft_dask/common/comms.py index d7260ece9b..0a7fb02da7 100644 --- a/python/raft-dask/raft_dask/common/comms.py +++ b/python/raft-dask/raft_dask/common/comms.py @@ -13,29 +13,28 @@ # limitations under the License. # -from .nccl import nccl -from .ucx import UCX - -from .comms_utils import inject_comms_on_handle -from .comms_utils import inject_comms_on_handle_coll_only - -from .utils import parse_host_port -from pylibraft.common.handle import Handle - -from dask.distributed import get_worker, default_client - -import warnings - import logging import time import uuid +import warnings from collections import OrderedDict +from pylibraft.common.handle import Handle + +from dask.distributed import default_client, get_worker + +from .comms_utils import ( + inject_comms_on_handle, + inject_comms_on_handle_coll_only, +) +from .nccl import nccl +from .ucx import UCX +from .utils import parse_host_port + logger = logging.getLogger(__name__) class Comms: - """ Initializes and manages underlying NCCL and UCX comms handles across the workers of a Dask cluster. It is expected that `init()` will be @@ -45,7 +44,7 @@ class Comms: Examples -------- - .. code-block:: python + .. code-block:: python # The following code block assumes we have wrapped a C++ # function in a Python function called `run_algorithm`, @@ -175,7 +174,6 @@ def init(self, workers=None): Parameters ---------- - workers : Sequence Unique collection of workers for initializing comms. """ @@ -256,7 +254,6 @@ def local_handle(sessionId): Returns ------- - handle : raft.Handle or None """ state = get_raft_comm_state(sessionId, get_worker()) @@ -277,7 +274,6 @@ def get_raft_comm_state(sessionId, state_object=None): Returns ------- - session state : str session state associated with sessionId """ diff --git a/python/raft-dask/raft_dask/common/comms_utils.pyx b/python/raft-dask/raft_dask/common/comms_utils.pyx index 2014284af1..7db04ef455 100644 --- a/python/raft-dask/raft_dask/common/comms_utils.pyx +++ b/python/raft-dask/raft_dask/common/comms_utils.pyx @@ -17,14 +17,12 @@ # cython: embedsignature = True # cython: language_level = 3 -from libc.stdlib cimport malloc, free -from cython.operator cimport dereference as deref - from cpython.long cimport PyLong_AsVoidPtr - +from cython.operator cimport dereference as deref +from libc.stdint cimport uintptr_t +from libc.stdlib cimport free, malloc from libcpp cimport bool -from libc.stdint cimport uintptr_t cdef extern from "nccl.h": diff --git a/python/raft-dask/raft_dask/common/nccl.pyx b/python/raft-dask/raft_dask/common/nccl.pyx index fd113e2222..a4d59610d3 100644 --- a/python/raft-dask/raft_dask/common/nccl.pyx +++ b/python/raft-dask/raft_dask/common/nccl.pyx @@ -19,11 +19,11 @@ # cython: embedsignature = True # cython: language_level = 3 -from libc.stdint cimport uintptr_t from cython.operator cimport dereference as deref - +from libc.stdint cimport uintptr_t +from libc.stdlib cimport free, malloc from libcpp cimport bool -from libc.stdlib cimport malloc, free + cdef extern from "raft/comms/std_comms.hpp" namespace "raft::comms": void get_unique_id(char *uid, int size) except + diff --git a/python/raft-dask/raft_dask/common/utils.py b/python/raft-dask/raft_dask/common/utils.py index daf51530be..78a899aa50 100644 --- a/python/raft-dask/raft_dask/common/utils.py +++ b/python/raft-dask/raft_dask/common/utils.py @@ -32,8 +32,8 @@ def parse_host_port(address): ------- tuple with host and port info : tuple(host, port) """ - if '://' in address: - address = address.rsplit('://', 1)[1] - host, port = address.split(':') + if "://" in address: + address = address.rsplit("://", 1)[1] + host, port = address.split(":") port = int(port) return host, port diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index 5e417f42ee..e0add1e55b 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -15,6 +15,5 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - CXX) + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX +) diff --git a/python/raft-dask/raft_dask/test/conftest.py b/python/raft-dask/raft_dask/test/conftest.py index f5cdc49700..39ee21cbaa 100644 --- a/python/raft-dask/raft_dask/test/conftest.py +++ b/python/raft-dask/raft_dask/test/conftest.py @@ -1,13 +1,12 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +import os + import pytest from dask.distributed import Client +from dask_cuda import LocalCUDACluster, initialize -from dask_cuda import initialize -from dask_cuda import LocalCUDACluster - -import os os.environ["UCX_LOG_LEVEL"] = "error" @@ -25,14 +24,18 @@ def cluster(): @pytest.fixture(scope="session") def ucx_cluster(): - initialize.initialize(create_cuda_context=True, - enable_tcp_over_ucx=enable_tcp_over_ucx, - enable_nvlink=enable_nvlink, - enable_infiniband=enable_infiniband) - cluster = LocalCUDACluster(protocol="ucx", - enable_tcp_over_ucx=enable_tcp_over_ucx, - enable_nvlink=enable_nvlink, - enable_infiniband=enable_infiniband) + initialize.initialize( + create_cuda_context=True, + enable_tcp_over_ucx=enable_tcp_over_ucx, + enable_nvlink=enable_nvlink, + enable_infiniband=enable_infiniband, + ) + cluster = LocalCUDACluster( + protocol="ucx", + enable_tcp_over_ucx=enable_tcp_over_ucx, + enable_nvlink=enable_nvlink, + enable_infiniband=enable_infiniband, + ) yield cluster cluster.close() diff --git a/python/raft-dask/raft_dask/test/test_comms.py b/python/raft-dask/raft_dask/test/test_comms.py index 29b4d963d9..74ec446e94 100644 --- a/python/raft-dask/raft_dask/test/test_comms.py +++ b/python/raft-dask/raft_dask/test/test_comms.py @@ -13,28 +13,29 @@ # limitations under the License. # -import pytest - from collections import OrderedDict -from dask.distributed import Client -from dask.distributed import wait +import pytest + +from dask.distributed import Client, wait try: - from raft_dask import Comms - from raft_dask.common import local_handle - from raft_dask.common import perform_test_comms_send_recv - from raft_dask.common import perform_test_comms_device_send_or_recv - from raft_dask.common import perform_test_comms_device_sendrecv - from raft_dask.common import perform_test_comms_device_multicast_sendrecv - from raft_dask.common import perform_test_comms_allreduce - from raft_dask.common import perform_test_comms_bcast - from raft_dask.common import perform_test_comms_reduce - from raft_dask.common import perform_test_comms_allgather - from raft_dask.common import perform_test_comms_gather - from raft_dask.common import perform_test_comms_gatherv - from raft_dask.common import perform_test_comms_reducescatter - from raft_dask.common import perform_test_comm_split + from raft_dask.common import ( + Comms, + local_handle, + perform_test_comm_split, + perform_test_comms_allgather, + perform_test_comms_allreduce, + perform_test_comms_bcast, + perform_test_comms_device_multicast_sendrecv, + perform_test_comms_device_send_or_recv, + perform_test_comms_device_sendrecv, + perform_test_comms_gather, + perform_test_comms_gatherv, + perform_test_comms_reduce, + perform_test_comms_reducescatter, + perform_test_comms_send_recv, + ) pytestmark = pytest.mark.mg except ImportError: diff --git a/python/raft-dask/raft_dask/test/test_raft.py b/python/raft-dask/raft_dask/test/test_raft.py index e1370dccc9..e1e1358f58 100644 --- a/python/raft-dask/raft_dask/test/test_raft.py +++ b/python/raft-dask/raft_dask/test/test_raft.py @@ -13,9 +13,10 @@ # limitations under the License. # -import pytest import sys +import pytest + try: import raft_dask except ImportError: @@ -23,7 +24,7 @@ pytestmart = pytest.mark.skip pytestmark = pytest.mark.skipif( - 'raft_dask' not in sys.argv, reason="marker to allow integration of RAFT" + "raft_dask" not in sys.argv, reason="marker to allow integration of RAFT" ) diff --git a/python/raft-dask/setup.cfg b/python/raft-dask/setup.cfg index c749a5a541..b005a7ab8f 100644 --- a/python/raft-dask/setup.cfg +++ b/python/raft-dask/setup.cfg @@ -1,11 +1,5 @@ # Copyright (c) 2020-2022, NVIDIA CORPORATION. -[flake8] -exclude = __init__.py,versioneer.py -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - [versioneer] VCS = git style = pep440 diff --git a/python/raft-dask/setup.py b/python/raft-dask/setup.py index 59bcc4a2dc..f5408e1f9e 100644 --- a/python/raft-dask/setup.py +++ b/python/raft-dask/setup.py @@ -14,37 +14,36 @@ # limitations under the License. # +import versioneer from setuptools import find_packages from skbuild import setup -import versioneer - - -setup(name='raft-dask', - description="Reusable Accelerated Functions & Tools Dask Infrastructure", - version=versioneer.get_version(), - classifiers=[ +setup( + name="raft-dask", + description="Reusable Accelerated Functions & Tools Dask Infrastructure", + version=versioneer.get_version(), + classifiers=[ "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9" - ], - author="NVIDIA Corporation", - package_data={ - # Note: A dict comprehension with an explicit copy is necessary - # (rather than something simpler like a dict.fromkeys) because - # otherwise every package will refer to the same list and skbuild - # modifies it in place. - key: ["*.hpp", "*.pxd"] - for key in find_packages( - include=[ - "raft_dask.common", - "raft_dask.common.includes", - ] - ) - }, - packages=find_packages(include=['raft_dask', 'raft_dask.*']), - license="Apache", - cmdclass=versioneer.get_cmdclass(), - zip_safe=False - ) + "Programming Language :: Python :: 3.9", + ], + author="NVIDIA Corporation", + package_data={ + # Note: A dict comprehension with an explicit copy is necessary + # (rather than something simpler like a dict.fromkeys) because + # otherwise every package will refer to the same list and skbuild + # modifies it in place. + key: ["*.hpp", "*.pxd"] + for key in find_packages( + include=[ + "raft_dask.common", + "raft_dask.common.includes", + ] + ) + }, + packages=find_packages(include=["raft_dask", "raft_dask.*"]), + license="Apache", + cmdclass=versioneer.get_cmdclass(), + zip_safe=False, +) diff --git a/python/raft-dask/setuputils.py b/python/raft-dask/setuputils.py index 8893e09fd3..9370d29876 100755 --- a/python/raft-dask/setuputils.py +++ b/python/raft-dask/setuputils.py @@ -51,15 +51,15 @@ def clean_folder(path): path : String Path to the folder to be cleaned. """ - shutil.rmtree(path + '/__pycache__', ignore_errors=True) + shutil.rmtree(path + "/__pycache__", ignore_errors=True) - folders = glob.glob(path + '/*/') + folders = glob.glob(path + "/*/") for folder in folders: - shutil.rmtree(folder + '/__pycache__', ignore_errors=True) + shutil.rmtree(folder + "/__pycache__", ignore_errors=True) clean_folder(folder) - cython_exts = glob.glob(folder + '/*.cpp') - cython_exts.extend(glob.glob(folder + '/*.cpython*')) + cython_exts = glob.glob(folder + "/*.cpp") + cython_exts.extend(glob.glob(folder + "/*.cpython*")) for file in cython_exts: os.remove(file) diff --git a/python/raft-dask/versioneer.py b/python/raft-dask/versioneer.py index b8c4bc423b..73e3259bf1 100644 --- a/python/raft-dask/versioneer.py +++ b/python/raft-dask/versioneer.py @@ -278,10 +278,12 @@ """ from __future__ import print_function + try: import configparser except ImportError: import ConfigParser as configparser + import errno import json import os @@ -309,11 +311,13 @@ def get_root(): setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -326,8 +330,10 @@ def get_root(): me_dir = os.path.normcase(os.path.splitext(me)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py) + ) except NameError: pass return root @@ -349,6 +355,7 @@ def get(parser, name): if parser.has_option("versioneer", name): return parser.get("versioneer", name) return None + cfg = VersioneerConfig() cfg.VCS = VCS cfg.style = get(parser, "style") or "" @@ -373,17 +380,20 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None +): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -391,10 +401,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -419,7 +432,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, p.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY[ + "git" +] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -994,7 +1009,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1003,7 +1018,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1011,19 +1026,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -1038,8 +1060,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command( + GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True + ) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1047,10 +1070,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1073,17 +1105,18 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = ( + "unable to parse git-describe output: '%s'" % describe_out + ) return pieces # tag @@ -1092,10 +1125,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1106,13 +1141,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command( + GITS, ["rev-list", "HEAD", "--count"], cwd=root + ) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1168,16 +1205,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1206,11 +1249,17 @@ def versions_from_file(filename): contents = f.read() except EnvironmentError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1219,8 +1268,9 @@ def versions_from_file(filename): def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps( + versions, sort_keys=True, indent=1, separators=(",", ": ") + ) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1252,8 +1302,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1367,11 +1416,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1391,9 +1442,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1416,8 +1471,9 @@ def get_versions(verbose=False): handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1471,9 +1527,13 @@ def get_versions(verbose=False): if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version(): @@ -1522,6 +1582,7 @@ def run(self): print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in both distutils and setuptools @@ -1554,14 +1615,17 @@ def run(self): # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join( + self.build_lib, cfg.versionfile_build + ) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -1582,17 +1646,21 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.distutils_buildexe import py2exe as _py2exe # py3 except ImportError: @@ -1611,13 +1679,17 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments @@ -1644,8 +1716,10 @@ def make_release_tree(self, base_dir, files): # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -1700,11 +1774,15 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except ( + EnvironmentError, + configparser.NoSectionError, + configparser.NoOptionError, + ) as e: if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print( + "Adding sample versioneer config to setup.cfg", file=sys.stderr + ) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -1713,15 +1791,18 @@ def do_setup(): print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: @@ -1763,8 +1844,10 @@ def do_setup(): else: print(" 'versioneer.py' already in MANIFEST.in") if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) + print( + " appending versionfile_source ('%s') to MANIFEST.in" + % cfg.versionfile_source + ) with open(manifest_in, "a") as f: f.write("include %s\n" % cfg.versionfile_source) else: diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..87cc11660c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,46 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +[flake8] +filename = *.py, *.pyx, *.pxd, *.pxi +exclude = __init__.py, *.egg, build, docs, .git, versioneer.py +force-check = True +ignore = + # line break before binary operator + W503, + # whitespace before : + E203 +per-file-ignores = + # Rules ignored only in Cython: + # E211: whitespace before '(' (used in multi-line imports) + # E225: Missing whitespace around operators (breaks cython casting syntax like ) + # E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) + # E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) + # E275: Missing whitespace after keyword (Doesn't work with Cython except?) + # E402: invalid syntax (works for Python, not Cython) + # E999: invalid syntax (works for Python, not Cython) + # W504: line break after binary operator (breaks lines that end with a pointer) + *.pyx: E211, E225, E226, E227, E275, E402, E999, W504 + *.pxd: E211, E225, E226, E227, E275, E402, E999, W504 + *.pxi: E211, E225, E226, E227, E275, E402, E999, W504 + +[pydocstyle] +# Due to https://github.com/PyCQA/pydocstyle/issues/363, we must exclude rather +# than include using match-dir. Note that as discussed in +# https://stackoverflow.com/questions/65478393/how-to-filter-directories-using-the-match-dir-flag-for-pydocstyle, +# unlike the match option above this match-dir will have no effect when +# pydocstyle is invoked from pre-commit. Therefore this exclusion list must +# also be maintained in the pre-commit config file. +match-dir = ^(?!(ci|cpp|conda|docs|java|notebooks)).*$ +# Allow missing docstrings for docutils +ignore-decorators = .*(docutils|doc_apply|copy_docstring).* +select = + D201, D204, D206, D207, D208, D209, D210, D211, D214, D215, D300, D301, D302, D403, D405, D406, D407, D408, D409, D410, D411, D412, D414, D418 + # Would like to enable the following rules in the future: + # D200, D202, D205, D400 + +[mypy] +ignore_missing_imports = True +# If we don't specify this, then mypy will check excluded files if +# they are imported by a checked file. +follow_imports = skip +exclude=_version.py