Breaking apart benchmarks into individual binaries (rapidsai#883)

Authors: - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Divye Gala (https://github.com/divyegala) URL: rapidsai#883
cjnolet · Oct 4, 2022 · 844a919 · 844a919
1 parent 830cb89
commit 844a919
Show file tree

Hide file tree

Showing 3 changed files with 142 additions and 76 deletions.
diff --git a/BUILD.md b/BUILD.md
@@ -7,6 +7,7 @@
     - [C++ Shared Libraries](#shared_cpp_libs)
     - [Improving Rebuild Times](#ccache)
     - [Googletests](#gtests)
+    - [Googlebench](#gbench)
     - [C++ Using Cmake](#cpp_using_cmake)
     - [Python](#python)
 - [Using RAFT in downstream projects](#use_raft)
@@ -102,17 +103,17 @@ It can take sometime to compile all of the tests. You can build individual tests
 ./build.sh libraft tests --limit-tests=SPATIAL_TEST;DISTANCE_TEST;MATRIX_TEST
 ```
 
-### <a id="benchmarks"></a>Benchmarks
+### <a id="gbench"></a>Benchmarks
 
-Compile the benchmarks using the `bench` target in `build.sh`:
+The benchmarks are broken apart by algorithm category, so you will find several binaries in `cpp/build/` named `*_BENCH`.
 ```bash
 ./build.sh libraft bench
 ```
 
-To run the benchmarks:
+It can take sometime to compile all of the tests. You can build individual tests by providing a semicolon-separated list to the `--limit-tests` option in `build.sh`:
 
 ```bash
-./cpp/build/bench_raft
+./build.sh libraft bench --limit-bench=SPATIAL_BENCH;DISTANCE_BENCH;LINALG_BENCH
 ```
 
 ### <a id="cpp_using_cmake"></a>C++ Using Cmake

diff --git a/build.sh b/build.sh
@@ -19,7 +19,7 @@ ARGS=$*
 REPODIR=$(cd $(dirname $0); pwd)
 
 VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean -v -g --install --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps"
-HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<tool>] [--limit-tests=<targets>]
+HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<tool>] [--limit-tests=<targets>] [--limit-bench=<targets>]
  where <target> is:
    clean            - remove all existing build artifacts and configuration (start over)
    libraft          - build the raft C++ code only. Also builds the C-wrapper library
@@ -41,6 +41,7 @@ HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<to
    --minimal-deps              - disables dependencies like thrust so they can be overridden.
                                  can be useful for a pure header-only install
    --limit-tests               - semicolon-separated list of test executables to compile (e.g. SPATIAL_TEST;CLUSTER_TEST)
+   --limit-bench               - semicolon-separated list of benchmark executables to compute (e.g. SPATIAL_BENCH;CLUSTER_BENCH)
    --allgpuarch                - build for all supported GPU architectures
    --buildfaiss                - build faiss statically into raft
    --install                   - install cmake targets
@@ -72,7 +73,7 @@ COMPILE_DIST_LIBRARY=OFF
 ENABLE_NN_DEPENDENCIES=OFF
 
 TEST_TARGETS="CLUSTER_TEST;CORE_TEST;DISTANCE_TEST;LABEL_TEST;LINALG_TEST;MATRIX_TEST;RANDOM_TEST;SOLVERS_TEST;SPARSE_TEST;SPARSE_DIST_TEST;SPARSE_NN_TEST;SPATIAL_TEST;STATS_TEST;UTILS_TEST"
-
+BENCH_TARGETS="CLUSTER_BENCH;SPATIAL_BENCH;DISTANCE_BENCH;LINALG_BENCH;SPARSE_BENCH;RANDOM_BENCH"
 ENABLE_thrust_DEPENDENCY=ON
 
 CACHE_ARGS=""
@@ -154,6 +155,21 @@ function limitTests {
     fi
 }
 
+function limitBench {
+    # Check for option to limit the set of test binaries to build
+    if [[ -n $(echo $ARGS | { grep -E "\-\-limit\-bench" || true; } ) ]]; then
+        # There are possible weird edge cases that may cause this regex filter to output nothing and fail silently
+        # the true pipe will catch any weird edge cases that may happen and will cause the program to fall back
+        # on the invalid option error
+        LIMIT_BENCH_TARGETS=$(echo $ARGS | sed -e 's/.*--limit-bench=//' -e 's/ .*//')
+        if [[ -n ${LIMIT_BENCH_TARGETS} ]]; then
+            # Remove the full LIMIT_TEST_TARGETS argument from list of args so that it passes validArgs function
+            ARGS=${ARGS//--limit-bench=$LIMIT_BENCH_TARGETS/}
+            BENCH_TARGETS=${LIMIT_BENCH_TARGETS}
+        fi
+    fi
+}
+
 if hasArg -h || hasArg --help; then
     echo "${HELP}"
     exit 0
@@ -164,6 +180,7 @@ if (( ${NUMARGS} != 0 )); then
     cmakeArgs
     cacheTool
     limitTests
+    limitBench
     for a in ${ARGS}; do
         if ! (echo " ${VALIDARGS} " | grep -q " ${a} "); then
             echo "Invalid option: ${a}"
@@ -221,7 +238,7 @@ if hasArg bench || (( ${NUMARGS} == 0 )); then
     COMPILE_DIST_LIBRARY=ON
     ENABLE_NN_DEPENDENCIES=ON
     COMPILE_NN_LIBRARY=ON
-    CMAKE_TARGET="${CMAKE_TARGET};bench_raft"
+    CMAKE_TARGET="${CMAKE_TARGET};${BENCH_TARGETS}"
 fi
 
 if hasArg --buildfaiss; then

diff --git a/cpp/bench/CMakeLists.txt b/cpp/bench/CMakeLists.txt
@@ -14,74 +14,122 @@
 # limitations under the License.
 #=============================================================================
 
-set(RAFT_CPP_BENCH_TARGET "bench_raft")
-
-# (please keep the filenames in alphabetical order)
-add_executable(${RAFT_CPP_BENCH_TARGET}
-  bench/cluster/kmeans_balanced.cu
-  bench/cluster/kmeans.cu
-  bench/distance/distance_cosine.cu
-  bench/distance/distance_exp_l2.cu
-  bench/distance/distance_l1.cu
-  bench/distance/distance_unexp_l2.cu
-  bench/linalg/add.cu
-  bench/linalg/map_then_reduce.cu
-  bench/linalg/matrix_vector_op.cu
-  bench/linalg/reduce.cu
-  bench/random/make_blobs.cu
-  bench/random/permute.cu
-  bench/random/rng.cu
-  bench/sparse/convert_csr.cu
-  bench/spatial/fused_l2_nn.cu
-  bench/spatial/knn/brute_force_float_int64_t.cu
-  bench/spatial/knn/brute_force_float_uint32_t.cu
-  bench/spatial/knn/ivf_flat_float_int64_t.cu
-  bench/spatial/knn/ivf_flat_float_uint32_t.cu
-  bench/spatial/knn/ivf_flat_int8_t_int64_t.cu
-  bench/spatial/knn/ivf_flat_uint8_t_uint32_t.cu
-  bench/spatial/knn/ivf_pq_float_int64_t.cu
-  bench/spatial/knn/ivf_pq_float_uint32_t.cu
-  bench/spatial/knn/ivf_pq_int8_t_int64_t.cu
-  bench/spatial/knn/ivf_pq_uint8_t_uint32_t.cu
-  bench/spatial/selection.cu
-  bench/main.cpp
-)
-
-set_target_properties(${RAFT_CPP_BENCH_TARGET}
-  PROPERTIES BUILD_RPATH "\$ORIGIN"
-          # set target compile options
-          CXX_STANDARD                        17
-          CXX_STANDARD_REQUIRED               ON
-          CUDA_STANDARD                       17
-          CUDA_STANDARD_REQUIRED              ON
-          POSITION_INDEPENDENT_CODE           ON
-          INTERFACE_POSITION_INDEPENDENT_CODE ON
-          INSTALL_RPATH "\$ORIGIN/../../../lib"
-)
-
-target_compile_options(${RAFT_CPP_BENCH_TARGET}
-  PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:${RAFT_CXX_FLAGS}>"
-          "$<$<COMPILE_LANGUAGE:CUDA>:${RAFT_CUDA_FLAGS}>"
-)
-
-target_include_directories(${RAFT_CPP_BENCH_TARGET}
-  PUBLIC  "$<BUILD_INTERFACE:${RAFT_SOURCE_DIR}/bench>"
-)
-
-target_link_libraries(${RAFT_CPP_BENCH_TARGET}
-  PRIVATE
-    raft::raft
-    raft::distance
-    raft::nn
-    faiss::faiss
-    benchmark::benchmark
-    $<TARGET_NAME_IF_EXISTS:OpenMP::OpenMP_CXX>
-    $<TARGET_NAME_IF_EXISTS:conda_env>
-)
+###################################################################################################
+# - compiler function -----------------------------------------------------------------------------
+
+function(ConfigureBench)
+
+set(options OPTIONAL DIST NN)
+set(oneValueArgs NAME )
+set(multiValueArgs PATH TARGETS CONFIGURATIONS)
+
+cmake_parse_arguments(ConfigureBench "${options}" "${oneValueArgs}"
+        "${multiValueArgs}" ${ARGN} )
+
+set(BENCH_NAME ${ConfigureBench_NAME})
+
+add_executable(${BENCH_NAME} ${ConfigureBench_PATH})
+
+message("BENCH PATH: ${ConfigureBench_PATH}")
+
+target_link_libraries(${BENCH_NAME}
+        PRIVATE
+        raft::raft
+        $<$<BOOL:${ConfigureBench_DIST}>:raft::distance>
+        $<$<BOOL:${ConfigureBench_NN}>:raft::nn>
+        benchmark::benchmark
+        Threads::Threads
+        $<TARGET_NAME_IF_EXISTS:OpenMP::OpenMP_CXX>
+        $<TARGET_NAME_IF_EXISTS:conda_env>
+        )
+
+set_target_properties(${BENCH_NAME}
+        PROPERTIES
+        # set target compile options
+        INSTALL_RPATH "\$ORIGIN/../../../lib"
+        CXX_STANDARD                        17
+        CXX_STANDARD_REQUIRED               ON
+        CUDA_STANDARD                       17
+        CUDA_STANDARD_REQUIRED              ON
+        POSITION_INDEPENDENT_CODE           ON
+        INTERFACE_POSITION_INDEPENDENT_CODE ON
+        )
+
+target_compile_options(${BENCH_NAME}
+        PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:${RAFT_CXX_FLAGS}>"
+        "$<$<COMPILE_LANGUAGE:CUDA>:${RAFT_CUDA_FLAGS}>"
+        )
+
+target_include_directories(${BENCH_NAME}
+        PUBLIC  "$<BUILD_INTERFACE:${RAFT_SOURCE_DIR}/bench>"
+        )
 
 install(
-  TARGETS ${RAFT_CPP_BENCH_TARGET}
-  COMPONENT testing
-  DESTINATION bin/libraft/gbench
-  EXCLUDE_FROM_ALL
-)
+        TARGETS ${BENCH_NAME}
+        COMPONENT testing
+        DESTINATION bin/gbench/libraft
+        EXCLUDE_FROM_ALL)
+
+endfunction()
+
+if(BUILD_BENCH)
+    ConfigureBench(NAME CLUSTER_BENCH
+            PATH
+            bench/cluster/kmeans_balanced.cu
+            bench/cluster/kmeans.cu
+            bench/main.cpp
+            OPTIONAL DIST NN
+            )
+
+    ConfigureBench(NAME DISTANCE_BENCH
+            PATH
+            bench/distance/distance_cosine.cu
+            bench/distance/distance_exp_l2.cu
+            bench/distance/distance_l1.cu
+            bench/distance/distance_unexp_l2.cu
+            bench/main.cpp
+            OPTIONAL DIST
+            )
+
+    ConfigureBench(NAME LINALG_BENCH
+            PATH
+            bench/linalg/add.cu
+            bench/linalg/map_then_reduce.cu
+            bench/linalg/matrix_vector_op.cu
+            bench/linalg/reduce.cu
+            bench/main.cpp
+            )
+
+    ConfigureBench(NAME RANDOM_BENCH
+            PATH
+            bench/random/make_blobs.cu
+            bench/random/permute.cu
+            bench/random/rng.cu
+            bench/main.cpp
+            )
+
+    ConfigureBench(NAME SPARSE_BENCH
+            PATH
+            bench/sparse/convert_csr.cu
+            bench/main.cpp
+            )
+
+    ConfigureBench(NAME SPATIAL_BENCH
+            PATH
+            bench/spatial/fused_l2_nn.cu
+            bench/spatial/knn/brute_force_float_int64_t.cu
+            bench/spatial/knn/brute_force_float_uint32_t.cu
+            bench/spatial/knn/ivf_flat_float_int64_t.cu
+            bench/spatial/knn/ivf_flat_float_uint32_t.cu
+            bench/spatial/knn/ivf_flat_int8_t_int64_t.cu
+            bench/spatial/knn/ivf_flat_uint8_t_uint32_t.cu
+            bench/spatial/knn/ivf_pq_float_int64_t.cu
+            bench/spatial/knn/ivf_pq_float_uint32_t.cu
+            bench/spatial/knn/ivf_pq_int8_t_int64_t.cu
+            bench/spatial/knn/ivf_pq_uint8_t_uint32_t.cu
+            bench/spatial/selection.cu
+            bench/main.cpp
+            OPTIONAL DIST NN
+            )
+endif()
+