diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 8810e7b0e..ff156f5ac 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -1,6 +1,6 @@ name: sanitizers -on: [workflow_dispatch] +on: [push, pull_request] jobs: build-and-test: runs-on: ubuntu-latest @@ -13,10 +13,14 @@ jobs: sanitizer: [Thread, Address, Undefined] - include: - # Memory sanitizer is not available for gcc - - compiler: clang10 - sanitizer: MemoryWithOrigin + # Memory sanitizer triggers on almost anything and some of the things + # are outside our control. Additionally, there seem to be + # inconsistencies between local and CI runs, so disabling this for now + # + # include: + # # Memory sanitizer is not available for gcc + # - compiler: clang10 + # sanitizer: MemoryWithOrigin steps: - uses: actions/checkout@v2 - uses: cvmfs-contrib/github-action-cvmfs@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index ac4cc0f44..859a8a50f 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ podio_set_rpath() set(USE_SANITIZER "" CACHE STRING "Compile with a sanitizer. Options are Address, Memory, MemoryWithOrigin, Undefined, Thread, Leak, Address;Undefined") ADD_SANITIZER_FLAGS() +option(FORCE_RUN_ALL_TESTS "Run all the tests even those with known problems" OFF) #--- Declare options ----------------------------------------------------------- option(CREATE_DOC "Whether or not to create doxygen doc target." OFF) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e589b3199..9b4716ba5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,14 +62,6 @@ else() set(CMAKE_CXX_FLAGS ${CXX_FLAGS_CMAKE_USED}) endif() -CREATE_PODIO_TEST(unittest.cpp "Catch2::Catch2;Catch2::Catch2WithMain") - -if(NOT USE_SANITIZER MATCHES "Memory(WithOrigin)?") - # This can fail with memory sanitizer because it actually runs an executable. - # Hence, we enable this only for non memory-sanitizer runs - include(Catch) - catch_discover_tests(unittest WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) -endif() if (TARGET TestDataModelSioBlocks) set(sio_dependent_tests write_sio.cpp read_sio.cpp read_and_write_sio.cpp write_timed_sio.cpp read_timed_sio.cpp) @@ -85,7 +77,6 @@ if (TARGET TestDataModelSioBlocks) target_link_libraries(read_timed_sio ROOT::Tree) endif() - #--- set some dependencies between the different tests to ensure input generating ones are run first set_property(TEST read PROPERTY DEPENDS write) set_property(TEST read-multiple PROPERTY DEPENDS write) @@ -114,3 +105,42 @@ set_property(TEST pyunittest PYTHONPATH=${CMAKE_SOURCE_DIR}/python:$ENV{PYTHONPATH} ROOT_INCLUDE_PATH=${CMAKE_SOURCE_DIR}/tests/datamodel:${ROOT_INCLUDE_PATH}) set_property(TEST pyunittest PROPERTY DEPENDS write) + +# Customize CTest to potentially disable some of the tests with known problems +configure_file(CTestCustom.cmake ${CMAKE_BINARY_DIR}/CTestCustom.cmake) + + +add_executable(unittest unittest.cpp) +target_link_libraries(unittest PUBLIC TestDataModel PRIVATE Catch2::Catch2WithMain) + +# The unittests are a bit better and they are labelled so we can put together a +# list of labels that we want to ignore +set(filter_tests "") +if (NOT FORCE_RUN_ALL_TESTS) + if(USE_SANITIZER MATCHES "Address") + set(filter_tests "~[LEAK-FAIL]~[ASAN-FAIL]") + elseif(USE_SANITIZER MATCHES "Leak") + set(filter_tests "~[LEAK-FAIL]") + elseif(USE_SANITIZER MATCHES "Thread") + set(filter_tests "~[THREAD-FAIL]") + endif() +endif() + +if (USE_SANITIZER MATCHES "Memory(WithOrigin)?") + # Automatic test discovery fails with Memory sanitizers due to some issues in + # Catch2. So in that case we skip the discovery step and simply run the thing + # directly in the tests. + if (FORCE_RUN_ALL_TESTS) + # Unfortunately Memory sanitizer seems to be really unhappy with Catch2 and + # it fails to succesfully launch the executable and execute any test. Here + # we just include them in order to have them show up as failing + add_test(NAME unittest COMMAND unittest ${filter_tests}) + endif() +else() + include(Catch) + catch_discover_tests(unittest + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + TEST_PREFIX "UT_" # make it possible to filter easily with -R ^UT + TEST_SPEC ${filter_tests} # discover only tests that are known to not fail + ) +endif() diff --git a/tests/CTestCustom.cmake b/tests/CTestCustom.cmake new file mode 100644 index 000000000..00ccb0308 --- /dev/null +++ b/tests/CTestCustom.cmake @@ -0,0 +1,41 @@ +# Some of the tests currently fail when run under some sanitizers, ignore them +# for now (but report that they have been ignored). This allows us to still run +# this in CI + +# See: https://gitlab.kitware.com/cmake/community/-/wikis/doc/ctest/Testing-With-CTest#customizing-ctest + +# "Integration style tests" pretty much all have problems at the moment with any +# sanitizer + +if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQUAL "")) + set(CTEST_CUSTOM_TESTS_IGNORE + ${CTEST_CUSTOM_TESTS_IGNORE} + + write + read + read_and_write + write_timed + read_timed + check_benchmark_outputs + read-multiple + + write_sio + read_sio + read_and_write_sio + write_timed_sio + read_timed_sio + check_benchmark_outputs_sio + + write_ascii + + ostream_operator + relation_range + + pyunittest + ) + + # ostream_operator is working with Memory sanitizer (at least locally) + if("@USE_SANITIZER@" MATCHES "Memory(WithOrigin)?") + list(REMOVE_ITEM CTEST_CUSTOM_TESTS_IGNORE ostream_operator) + endif() +endif() diff --git a/tests/unittest.cpp b/tests/unittest.cpp index e974be951..4518412ce 100644 --- a/tests/unittest.cpp +++ b/tests/unittest.cpp @@ -60,7 +60,7 @@ TEST_CASE("Assignment-operator ref count", "[basics][memory-management]") { } } -TEST_CASE("Clearing", "[ASAN-FAIL][basics][memory-management]"){ +TEST_CASE("Clearing", "[ASAN-FAIL][THREAD-FAIL][basics][memory-management]"){ bool success = true; auto store = podio::EventStore(); auto& hits = store.create("hits");