From a8430ae3e3503a46073ee93d314f531924ada438 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 11 Nov 2020 00:34:19 -0800 Subject: [PATCH 1/5] Add CMake option to enable sanitizer --- CMakeLists.txt | 13 +++++++ cmake/Sanitizer.cmake | 69 +++++++++++++++++++++++++++++++++++ cmake/modules/FindASan.cmake | 13 +++++++ cmake/modules/FindLSan.cmake | 13 +++++++ cmake/modules/FindTSan.cmake | 13 +++++++ cmake/modules/FindUBSan.cmake | 13 +++++++ 6 files changed, 134 insertions(+) create mode 100644 cmake/Sanitizer.cmake create mode 100644 cmake/modules/FindASan.cmake create mode 100644 cmake/modules/FindLSan.cmake create mode 100644 cmake/modules/FindTSan.cmake create mode 100644 cmake/modules/FindUBSan.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index a4baa8c5361f..f57c55b5b61e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,11 @@ OPTION(USE_HDFS "Enable HDFS support (EXPERIMENTAL)" OFF) OPTION(USE_TIMETAG "Set to ON to output time costs" OFF) OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF) OPTION(USE_DEBUG "Set to ON for Debug mode" OFF) +OPTION(USE_SANITIZER "Use santizer flags" OFF) +option(SANITIZER_PATH "Path to sanitizer libs") +set(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING + "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are +address, leak, undefined and thread.") OPTION(BUILD_STATIC_LIB "Build static library" OFF) OPTION(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF) OPTION(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF) @@ -37,6 +42,14 @@ else() PROJECT(lightgbm LANGUAGES C CXX) endif() +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules") + +#-- Sanitizer +if (USE_SANITIZER) + include(cmake/Sanitizer.cmake) + enable_sanitizers("${ENABLED_SANITIZERS}") +endif (USE_SANITIZER) + if(__INTEGRATE_OPENCL) set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE) set(USE_GPU OFF CACHE BOOL "" FORCE) diff --git a/cmake/Sanitizer.cmake b/cmake/Sanitizer.cmake new file mode 100644 index 000000000000..5405ef497241 --- /dev/null +++ b/cmake/Sanitizer.cmake @@ -0,0 +1,69 @@ +# Set appropriate compiler and linker flags for sanitizers. +# +# Usage of this module: +# enable_sanitizers("address;leak") + +# Add flags +macro(enable_sanitizer sanitizer) + if(${sanitizer} MATCHES "address") + find_package(ASan) + set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address") + if (ASan_FOUND) + link_libraries(${ASan_LIBRARY}) + endif (ASan_FOUND) + + elseif(${sanitizer} MATCHES "thread") + find_package(TSan) + set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread") + if (TSan_FOUND) + link_libraries(${TSan_LIBRARY}) + endif (TSan_FOUND) + + elseif(${sanitizer} MATCHES "leak") + find_package(LSan) + set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak") + if (LSan_FOUND) + link_libraries(${LSan_LIBRARY}) + endif (LSan_FOUND) + + elseif(${sanitizer} MATCHES "undefined") + find_package(UBSan) + set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=undefined -fno-sanitize-recover=undefined") + if (UBSan_FOUND) + link_libraries(${UBSan_LIBRARY}) + endif (UBSan_FOUND) + + else() + message(FATAL_ERROR "Santizer ${sanitizer} not supported.") + endif() +endmacro() + +macro(enable_sanitizers SANITIZERS) + # Check sanitizers compatibility. + foreach ( _san ${SANITIZERS} ) + string(TOLOWER ${_san} _san) + if (_san MATCHES "thread") + if (${_use_other_sanitizers}) + message(FATAL_ERROR + "thread sanitizer is not compatible with ${_san} sanitizer.") + endif() + set(_use_thread_sanitizer 1) + else () + if (${_use_thread_sanitizer}) + message(FATAL_ERROR + "${_san} sanitizer is not compatible with thread sanitizer.") + endif() + set(_use_other_sanitizers 1) + endif() + endforeach() + + message("Sanitizers: ${SANITIZERS}") + + foreach( _san ${SANITIZERS} ) + string(TOLOWER ${_san} _san) + enable_sanitizer(${_san}) + endforeach() + message("Sanitizers compile flags: ${SAN_COMPILE_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_COMPILE_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_COMPILE_FLAGS}") +endmacro() diff --git a/cmake/modules/FindASan.cmake b/cmake/modules/FindASan.cmake new file mode 100644 index 000000000000..e7b273853337 --- /dev/null +++ b/cmake/modules/FindASan.cmake @@ -0,0 +1,13 @@ +set(ASan_LIB_NAME ASan) + +find_library(ASan_LIBRARY + NAMES libasan.so libasan.so.5 libasan.so.4 libasan.so.3 libasan.so.2 libasan.so.1 libasan.so.0 + PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(ASan DEFAULT_MSG + ASan_LIBRARY) + +mark_as_advanced( + ASan_LIBRARY + ASan_LIB_NAME) diff --git a/cmake/modules/FindLSan.cmake b/cmake/modules/FindLSan.cmake new file mode 100644 index 000000000000..3f68fb05bf08 --- /dev/null +++ b/cmake/modules/FindLSan.cmake @@ -0,0 +1,13 @@ +set(LSan_LIB_NAME lsan) + +find_library(LSan_LIBRARY + NAMES liblsan.so liblsan.so.0 liblsan.so.0.0.0 + PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LSan DEFAULT_MSG + LSan_LIBRARY) + +mark_as_advanced( + LSan_LIBRARY + LSan_LIB_NAME) diff --git a/cmake/modules/FindTSan.cmake b/cmake/modules/FindTSan.cmake new file mode 100644 index 000000000000..aa01802f8063 --- /dev/null +++ b/cmake/modules/FindTSan.cmake @@ -0,0 +1,13 @@ +set(TSan_LIB_NAME tsan) + +find_library(TSan_LIBRARY + NAMES libtsan.so libtsan.so.0 libtsan.so.0.0.0 + PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(TSan DEFAULT_MSG + TSan_LIBRARY) + +mark_as_advanced( + TSan_LIBRARY + TSan_LIB_NAME) diff --git a/cmake/modules/FindUBSan.cmake b/cmake/modules/FindUBSan.cmake new file mode 100644 index 000000000000..e1b72eb6df1b --- /dev/null +++ b/cmake/modules/FindUBSan.cmake @@ -0,0 +1,13 @@ +set(UBSan_LIB_NAME UBSan) + +find_library(UBSan_LIBRARY + NAMES libubsan.so libubsan.so.5 libubsan.so.4 libubsan.so.3 libubsan.so.2 libubsan.so.1 libubsan.so.0 + PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(UBSan DEFAULT_MSG + UBSan_LIBRARY) + +mark_as_advanced( + UBSan_LIBRARY + UBSan_LIB_NAME) From 77422b2165b349f4c0a1de8ece59786bdd5dcab0 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 11 Nov 2020 01:43:15 -0800 Subject: [PATCH 2/5] Set up gtest --- .gitignore | 1 + CMakeLists.txt | 24 ++++++++++++++++++++++-- tests/cpp_test/test_main.cpp | 11 +++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 tests/cpp_test/test_main.cpp diff --git a/.gitignore b/.gitignore index 9cbb2e872f03..f65739c4ea7a 100644 --- a/.gitignore +++ b/.gitignore @@ -269,6 +269,7 @@ _Pvt_Extensions *.app /windows/LightGBM.VC.db lightgbm +/testlightgbm # Created by https://www.gitignore.io/api/python diff --git a/CMakeLists.txt b/CMakeLists.txt index f57c55b5b61e..2a53760c1fd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,10 +7,11 @@ OPTION(USE_TIMETAG "Set to ON to output time costs" OFF) OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF) OPTION(USE_DEBUG "Set to ON for Debug mode" OFF) OPTION(USE_SANITIZER "Use santizer flags" OFF) -option(SANITIZER_PATH "Path to sanitizer libs") -set(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING +OPTION(SANITIZER_PATH "Path to sanitizer libs") +SET(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are address, leak, undefined and thread.") +OPTION(BUILD_CPP_TEST "Build C++ tests with Google Test" OFF) OPTION(BUILD_STATIC_LIB "Build static library" OFF) OPTION(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF) OPTION(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF) @@ -464,6 +465,25 @@ if(__BUILD_FOR_R) endif(MSVC) endif(__BUILD_FOR_R) +#-- Google C++ tests +if(BUILD_CPP_TEST) + find_package(GTest CONFIG) + if(NOT GTEST_FOUND) + message(STATUS "Did not find Google Test in the system root. Fetching Google Test now...") + include(FetchContent) + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-1.10.0 + ) + FetchContent_MakeAvailable(googletest) + add_library(GTest::GTest ALIAS gtest) + endif() + file(GLOB CPP_TEST_SOURCES tests/cpp_test/*.cpp) + add_executable(testlightgbm ${CPP_TEST_SOURCES} ${SOURCES}) + target_link_libraries(testlightgbm PRIVATE GTest::GTest) +endif() + install(TARGETS lightgbm _lightgbm RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib diff --git a/tests/cpp_test/test_main.cpp b/tests/cpp_test/test_main.cpp new file mode 100644 index 000000000000..b881e09283c4 --- /dev/null +++ b/tests/cpp_test/test_main.cpp @@ -0,0 +1,11 @@ +/*! + * Copyright (c) 2020 Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE file in the project root for license information. + */ +#include + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + testing::FLAGS_gtest_death_test_style = "threadsafe"; + return RUN_ALL_TESTS(); +} From eb3eea085a0e2ee922926a744e4d629b206d9575 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Fri, 5 Mar 2021 18:18:00 -0800 Subject: [PATCH 3/5] Address reviewer's feedback --- cmake/Sanitizer.cmake | 12 ++++++------ cmake/modules/FindASan.cmake | 6 +----- cmake/modules/FindLSan.cmake | 4 ---- cmake/modules/FindTSan.cmake | 4 ---- cmake/modules/FindUBSan.cmake | 6 +----- tests/cpp_test/test_main.cpp | 2 +- 6 files changed, 9 insertions(+), 25 deletions(-) diff --git a/cmake/Sanitizer.cmake b/cmake/Sanitizer.cmake index 5405ef497241..30e644ad2886 100644 --- a/cmake/Sanitizer.cmake +++ b/cmake/Sanitizer.cmake @@ -6,28 +6,28 @@ # Add flags macro(enable_sanitizer sanitizer) if(${sanitizer} MATCHES "address") - find_package(ASan) + find_package(ASan REQUIRED) set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address") if (ASan_FOUND) link_libraries(${ASan_LIBRARY}) endif (ASan_FOUND) elseif(${sanitizer} MATCHES "thread") - find_package(TSan) + find_package(TSan REQUIRED) set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread") if (TSan_FOUND) link_libraries(${TSan_LIBRARY}) endif (TSan_FOUND) elseif(${sanitizer} MATCHES "leak") - find_package(LSan) + find_package(LSan REQUIRED) set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak") if (LSan_FOUND) link_libraries(${LSan_LIBRARY}) endif (LSan_FOUND) elseif(${sanitizer} MATCHES "undefined") - find_package(UBSan) + find_package(UBSan REQUIRED) set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=undefined -fno-sanitize-recover=undefined") if (UBSan_FOUND) link_libraries(${UBSan_LIBRARY}) @@ -57,13 +57,13 @@ macro(enable_sanitizers SANITIZERS) endif() endforeach() - message("Sanitizers: ${SANITIZERS}") + message(STATUS "Sanitizers: ${SANITIZERS}") foreach( _san ${SANITIZERS} ) string(TOLOWER ${_san} _san) enable_sanitizer(${_san}) endforeach() - message("Sanitizers compile flags: ${SAN_COMPILE_FLAGS}") + message(STATUS "Sanitizers compile flags: ${SAN_COMPILE_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_COMPILE_FLAGS}") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_COMPILE_FLAGS}") endmacro() diff --git a/cmake/modules/FindASan.cmake b/cmake/modules/FindASan.cmake index e7b273853337..660dfa3a15b2 100644 --- a/cmake/modules/FindASan.cmake +++ b/cmake/modules/FindASan.cmake @@ -1,13 +1,9 @@ set(ASan_LIB_NAME ASan) find_library(ASan_LIBRARY - NAMES libasan.so libasan.so.5 libasan.so.4 libasan.so.3 libasan.so.2 libasan.so.1 libasan.so.0 + NAMES libasan.so libasan.so.5 libasan.so.4 libasan.so.3 libasan.so.2 libasan.so.1 libasan.so.0 libasan.so.0.0.0 PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(ASan DEFAULT_MSG ASan_LIBRARY) - -mark_as_advanced( - ASan_LIBRARY - ASan_LIB_NAME) diff --git a/cmake/modules/FindLSan.cmake b/cmake/modules/FindLSan.cmake index 3f68fb05bf08..1b69fb7aa74a 100644 --- a/cmake/modules/FindLSan.cmake +++ b/cmake/modules/FindLSan.cmake @@ -7,7 +7,3 @@ find_library(LSan_LIBRARY include(FindPackageHandleStandardArgs) find_package_handle_standard_args(LSan DEFAULT_MSG LSan_LIBRARY) - -mark_as_advanced( - LSan_LIBRARY - LSan_LIB_NAME) diff --git a/cmake/modules/FindTSan.cmake b/cmake/modules/FindTSan.cmake index aa01802f8063..0fd26ace0781 100644 --- a/cmake/modules/FindTSan.cmake +++ b/cmake/modules/FindTSan.cmake @@ -7,7 +7,3 @@ find_library(TSan_LIBRARY include(FindPackageHandleStandardArgs) find_package_handle_standard_args(TSan DEFAULT_MSG TSan_LIBRARY) - -mark_as_advanced( - TSan_LIBRARY - TSan_LIB_NAME) diff --git a/cmake/modules/FindUBSan.cmake b/cmake/modules/FindUBSan.cmake index e1b72eb6df1b..400007a86ae2 100644 --- a/cmake/modules/FindUBSan.cmake +++ b/cmake/modules/FindUBSan.cmake @@ -1,13 +1,9 @@ set(UBSan_LIB_NAME UBSan) find_library(UBSan_LIBRARY - NAMES libubsan.so libubsan.so.5 libubsan.so.4 libubsan.so.3 libubsan.so.2 libubsan.so.1 libubsan.so.0 + NAMES libubsan.so libubsan.so.1 libubsan.so.0 libubsan.so.0.0.0 PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(UBSan DEFAULT_MSG UBSan_LIBRARY) - -mark_as_advanced( - UBSan_LIBRARY - UBSan_LIB_NAME) diff --git a/tests/cpp_test/test_main.cpp b/tests/cpp_test/test_main.cpp index b881e09283c4..e84c8142b52c 100644 --- a/tests/cpp_test/test_main.cpp +++ b/tests/cpp_test/test_main.cpp @@ -1,5 +1,5 @@ /*! - * Copyright (c) 2020 Microsoft Corporation. All rights reserved. + * Copyright (c) 2021 Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE file in the project root for license information. */ #include From dfaa89e941100ed867e967d54bb75792e2f704a2 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Mon, 8 Mar 2021 15:57:25 -0800 Subject: [PATCH 4/5] Address reviewer's feedback --- CMakeLists.txt | 2 +- cmake/Sanitizer.cmake | 16 ++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 492bd453e427..c5a3f834f3b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ OPTION(USE_TIMETAG "Set to ON to output time costs" OFF) OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF) OPTION(USE_DEBUG "Set to ON for Debug mode" OFF) OPTION(USE_SANITIZER "Use santizer flags" OFF) -OPTION(SANITIZER_PATH "Path to sanitizer libs") +SET(SANITIZER_PATH "Path to sanitizer libs") SET(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are address, leak, undefined and thread.") diff --git a/cmake/Sanitizer.cmake b/cmake/Sanitizer.cmake index 30e644ad2886..9e1a94bc1569 100644 --- a/cmake/Sanitizer.cmake +++ b/cmake/Sanitizer.cmake @@ -8,30 +8,22 @@ macro(enable_sanitizer sanitizer) if(${sanitizer} MATCHES "address") find_package(ASan REQUIRED) set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address") - if (ASan_FOUND) - link_libraries(${ASan_LIBRARY}) - endif (ASan_FOUND) + link_libraries(${ASan_LIBRARY}) elseif(${sanitizer} MATCHES "thread") find_package(TSan REQUIRED) set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread") - if (TSan_FOUND) - link_libraries(${TSan_LIBRARY}) - endif (TSan_FOUND) + link_libraries(${TSan_LIBRARY}) elseif(${sanitizer} MATCHES "leak") find_package(LSan REQUIRED) set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak") - if (LSan_FOUND) - link_libraries(${LSan_LIBRARY}) - endif (LSan_FOUND) + link_libraries(${LSan_LIBRARY}) elseif(${sanitizer} MATCHES "undefined") find_package(UBSan REQUIRED) set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=undefined -fno-sanitize-recover=undefined") - if (UBSan_FOUND) - link_libraries(${UBSan_LIBRARY}) - endif (UBSan_FOUND) + link_libraries(${UBSan_LIBRARY}) else() message(FATAL_ERROR "Santizer ${sanitizer} not supported.") From 433ea4746b4cefa3207492e19cd2d79a7aef9a6c Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Tue, 9 Mar 2021 05:14:44 -0800 Subject: [PATCH 5/5] Update CMakeLists.txt Co-authored-by: Nikita Titov --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c5a3f834f3b5..6c8ce6877a07 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ OPTION(USE_TIMETAG "Set to ON to output time costs" OFF) OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF) OPTION(USE_DEBUG "Set to ON for Debug mode" OFF) OPTION(USE_SANITIZER "Use santizer flags" OFF) -SET(SANITIZER_PATH "Path to sanitizer libs") +SET(SANITIZER_PATH "" CACHE STRING "Path to sanitizer libs") SET(ENABLED_SANITIZERS "address" "leak" "undefined" CACHE STRING "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are address, leak, undefined and thread.")