tests: make tests run in CI on windows

They were failing due to missing dll. This commit - inject the location of Qt dll in the PATH - Qt plugin directory in QT_PLUGIN_PATH - Qml import path in QML_IMPORT_PATH & QML2_IMPORT_PATH They are not necessary to every tests, but at least they should be robust for the future when more tests will be added. This commit use `ENVIRONMENT_MODIFICATION` instead of `ENVIRONMENT`, since adding a value to the PATH in windows is a mess otherwise with CMake, because `;` is used as a list separator. I took the liberty to refactor some boilerplate code, to make it working more easily. More refactoring is needed to remove boilerplate code.
KDAB · Mar 31, 2023 · 25a9c53 · 25a9c53
1 parent c435d65
commit 25a9c53
Show file tree

Hide file tree

Showing 6 changed files with 127 additions and 64 deletions.
diff --git a/.github/workflows/github-cxx-qt-tests.yml b/.github/workflows/github-cxx-qt-tests.yml
@@ -91,7 +91,7 @@ jobs:
             # FIXME: many tests fail to link
             # https://github.com/KDAB/cxx-qt/issues/111
             # cxx_qt_lib_cargo_tests fails to run with a missing DLL error.
-            ctest_args: --exclude-regex '^(demo_threading_cargo_tests|example_qml.*|qml.*tests|test.*|reuse_lint|cpp_clang_format|.*valgrind|cxx_qt_lib_cargo_tests)$'
+            ctest_args: --exclude-regex '^(test_basic_cxx_only_cargo_tests|reuse_lint|cpp_clang_format|.*valgrind)$'
             exe_suffix: .exe
             qt_qpa_platform: windows
             compiler_cache_path: C:\Users\runneradmin\AppData\Local\Mozilla\sccache\cache
@@ -106,7 +106,7 @@ jobs:
             # FIXME: many tests fail to link
             # https://github.com/KDAB/cxx-qt/issues/111
             # cxx_qt_lib_cargo_tests fails to run with a missing DLL error.
-            ctest_args: --exclude-regex '^(demo_threading_cargo_tests|example_qml.*|qml.*tests|test.*|reuse_lint|cpp_clang_format|.*valgrind|cxx_qt_lib_cargo_tests)$'
+            ctest_args: --exclude-regex '^(test_basic_cxx_only_cargo_tests|reuse_lint|cpp_clang_format|.*valgrind)$'
             exe_suffix: .exe
             qt_qpa_platform: windows
             compiler_cache_path: C:\Users\runneradmin\AppData\Local\Mozilla\sccache\cache

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -138,9 +138,6 @@ function(add_valgrind_test NAME_WITH_PREFIX BINARY WORKING_DIRECTORY)
     )
 endfunction()
 
-add_subdirectory(book)
-add_subdirectory(examples)
-
 # QMAKE environment variable is needed by qt-build-utils to ensure that Cargo
 # uses the same installation of Qt as CMake does.
 if(NOT USE_QT5)
@@ -150,25 +147,86 @@ if(NOT Qt6_FOUND)
     find_package(Qt5 5.15 COMPONENTS Core Gui Test REQUIRED)
 endif()
 get_target_property(QMAKE Qt::qmake IMPORTED_LOCATION)
-set(CARGO_ENV "QMAKE=${QMAKE}")
+set(CARGO_ENV "QMAKE=set:${QMAKE}")
+set(RUNTIME_ENV "")
+
+# On windows, Qt dll needs to be in the PATH for the tests to run
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+    execute_process(
+        COMMAND ${QMAKE} -query QT_INSTALL_BINS
+        OUTPUT_VARIABLE QT_INSTALL_BINS
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
+    execute_process(
+        COMMAND ${QMAKE} -query QT_INSTALL_PLUGINS
+        OUTPUT_VARIABLE QT_INSTALL_PLUGINS
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
+    execute_process(
+        COMMAND ${QMAKE} -query QT_INSTALL_QML
+        OUTPUT_VARIABLE QT_INSTALL_QML
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
+    list(
+        APPEND
+        RUNTIME_ENV
+        "PATH=path_list_append:${QT_INSTALL_BINS}"
+        "QT_PLUGIN_PATH=path_list_append:${QT_INSTALL_PLUGINS}"
+        "QML_IMPORT_PATH=path_list_append:${QT_INSTALL_QML}"
+        "QML2_IMPORT_PATH=path_list_append:${QT_INSTALL_QML}"
+    )
+    list(APPEND CARGO_ENV ${RUNTIME_ENV})
+endif()
 
 # Create helper method which adds relevent cargo tests for a given manifest
 function(add_test_cargo TEST_NAME_PREFIX MANIFEST_PATH ADD_DOCTESTS)
     # Add cargo as a test
-    add_test(NAME ${TEST_NAME_PREFIX}_cargo_tests COMMAND cargo test --all-features --all-targets --manifest-path ${MANIFEST_PATH})
-    set_property(TEST ${TEST_NAME_PREFIX}_cargo_tests PROPERTY ENVIRONMENT ${CARGO_ENV})
+    set(CARGO_TEST_NAME ${TEST_NAME_PREFIX}_cargo_tests)
+    add_test(NAME ${CARGO_TEST_NAME}
+             COMMAND cargo test --all-features --all-targets --manifest-path
+                     ${MANIFEST_PATH}
+    )
+    set_tests_properties(
+        ${CARGO_TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION "${CARGO_ENV}"
+    )
+
     # Check if we should enable doc tests
-    if (${ADD_DOCTESTS} STREQUAL "DOCTESTS_ON")
+    if(${ADD_DOCTESTS} STREQUAL "DOCTESTS_ON")
+        set(CARGO_DOC_TEST_NAME ${TEST_NAME_PREFIX}_cargo_doc_tests)
         # Add cargo docs as a test
-        add_test(NAME ${TEST_NAME_PREFIX}_cargo_doc_tests COMMAND cargo test --all-features --doc --manifest-path ${MANIFEST_PATH})
-        set_property(TEST ${TEST_NAME_PREFIX}_cargo_doc_tests PROPERTY ENVIRONMENT ${CARGO_ENV})
+        add_test(NAME ${CARGO_DOC_TEST_NAME}
+                 COMMAND cargo test --all-features --doc --manifest-path
+                         ${MANIFEST_PATH}
+        )
+        set_tests_properties(
+            ${CARGO_DOC_TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION
+                                              "${CARGO_ENV}"
+        )
     endif()
+
     # Add clippy as a test
-    add_test(NAME ${TEST_NAME_PREFIX}_cargo_clippy COMMAND cargo clippy --all-features --all-targets --manifest-path ${MANIFEST_PATH} -- -D warnings)
-    set_property(TEST ${TEST_NAME_PREFIX}_cargo_clippy PROPERTY ENVIRONMENT ${CARGO_ENV})
+    set(CARGO_CLIPPY_TEST_NAME ${TEST_NAME_PREFIX}_cargo_clippy)
+    add_test(NAME ${CARGO_CLIPPY_TEST_NAME}
+             COMMAND cargo clippy --all-features --all-targets --manifest-path
+                     ${MANIFEST_PATH} -- -D warnings
+    )
+    set_tests_properties(
+        ${CARGO_CLIPPY_TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION
+                                             "${CARGO_ENV}"
+    )
+
     # Add rustfmt as a test
-    add_test(NAME ${TEST_NAME_PREFIX}_cargo_fmt COMMAND cargo fmt --manifest-path ${MANIFEST_PATH} -- --check)
-    set_property(TEST ${TEST_NAME_PREFIX}_cargo_fmt PROPERTY ENVIRONMENT ${CARGO_ENV})
+    set(CARGO_FMT_TEST_NAME ${TEST_NAME_PREFIX}_cargo_fmt)
+    add_test(NAME ${CARGO_FMT_TEST_NAME} COMMAND cargo fmt --manifest-path
+                                                 ${MANIFEST_PATH} -- --check
+    )
+    set_tests_properties(
+        ${CARGO_FMT_TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION
+                                          "${CARGO_ENV}"
+    )
 endfunction()
 
 # Add cargo tests for all our manifests
@@ -199,4 +257,6 @@ add_test(NAME cpp_clang_format COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/clang_format
 # Add test which checks all files have a valid license
 add_test(NAME reuse_lint COMMAND reuse lint)
 
+add_subdirectory(book)
+add_subdirectory(examples)
 add_subdirectory(tests)
diff --git a/examples/qml_extension_plugin/tests/CMakeLists.txt b/examples/qml_extension_plugin/tests/CMakeLists.txt
@@ -19,12 +19,26 @@ target_link_libraries(${APP_NAME} Qt::QuickTest)
 
 add_dependencies(${APP_NAME} core_qmlplugin)
 
-set(test_cmd "./${APP_NAME}" -import "${CMAKE_CURRENT_BINARY_DIR}/../qml" -input "${CMAKE_CURRENT_SOURCE_DIR}")
-add_test(NAME ${APP_NAME}
+set(test_cmd
+    "./${APP_NAME}"
+    -import
+    "${CMAKE_CURRENT_BINARY_DIR}/../qml"
+    -input
+    "${CMAKE_CURRENT_SOURCE_DIR}"
+    -plugins
+    "$<TARGET_FILE_DIR:core_qmlplugin>"
+)
+
+add_test(
+    NAME ${APP_NAME}
     COMMAND ${test_cmd}
     WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
 )
 
+set_tests_properties(
+    ${APP_NAME} PROPERTIES ENVIRONMENT_MODIFICATION "${RUNTIME_ENV}"
+)
+
 # TODO: Remove valgrind_additional_suppressions, there is a currently unknown issue with Qt5,
 # which we can't precisely suppress, so we need to suppress all conditional checks in valgrind.
 if(Qt5_FOUND)

diff --git a/examples/qml_features/CMakeLists.txt b/examples/qml_features/CMakeLists.txt
@@ -64,46 +64,25 @@ qt_import_qml_plugins(${APP_NAME})
 # Unit test
 #
 
-# Copy QML unit test to build folder
-# this is so that our test executable can find the QML files in the same folder
-function(copy_qml_test TEST_NAME)
-    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/tst_${TEST_NAME}.qml ${CMAKE_CURRENT_BINARY_DIR}/tests/tst_${TEST_NAME}.qml COPYONLY)
-endfunction()
-
-copy_qml_test(containers)
-copy_qml_test(custom_base_class)
-copy_qml_test(invokables)
-copy_qml_test(multiple_qobjects)
-copy_qml_test(nested_qobjects)
-copy_qml_test(properties)
-copy_qml_test(serialisation)
-copy_qml_test(signals)
-copy_qml_test(singleton)
-copy_qml_test(threading)
-copy_qml_test(types)
-copy_qml_test(uncreatable)
-
-add_executable(${APP_NAME}_test
-    tests/main.cpp
+set(APP_TEST_NAME ${APP_NAME}_test)
+add_executable(${APP_TEST_NAME} tests/main.cpp)
+target_include_directories(${APP_TEST_NAME} PRIVATE cpp)
+target_link_libraries(
+    ${APP_TEST_NAME} PRIVATE ${APP_NAME}_lib Qt::Test Qt::QuickTest
 )
-target_include_directories(${APP_NAME}_test PRIVATE cpp)
-target_link_libraries(${APP_NAME}_test PRIVATE
-    ${APP_NAME}_lib
-    Qt::Test
-    Qt::QuickTest
-)
-qt_import_qml_plugins(${APP_NAME}_test)
+qt_import_qml_plugins(${APP_TEST_NAME})
 
-if(WIN32)
-    set(EXE_SUFFIX ".exe")
-else()
-    set(EXE_SUFFIX "")
-endif()
-
-add_test(NAME ${APP_NAME}_test COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/${APP_NAME}_test${EXE_SUFFIX} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests)
+add_test(
+    NAME ${APP_TEST_NAME}
+    COMMAND $<TARGET_FILE:${APP_TEST_NAME}> -input
+            ${CMAKE_CURRENT_SOURCE_DIR}/tests
+)
 
-add_valgrind_test(${APP_NAME}_test ${CMAKE_CURRENT_BINARY_DIR}/tests/${APP_NAME}_test${EXE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}/tests)
+add_valgrind_test(
+    ${APP_TEST_NAME} "$<TARGET_FILE:${APP_TEST_NAME}> -input
+    ${CMAKE_CURRENT_SOURCE_DIR}/tests" ${CMAKE_CURRENT_BINARY_DIR}
+)
 
-set_target_properties(${APP_NAME}_test PROPERTIES
-    RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
+set_tests_properties(
+    ${APP_TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION "${RUNTIME_ENV}"
 )
diff --git a/examples/qml_minimal/CMakeLists.txt b/examples/qml_minimal/CMakeLists.txt
@@ -122,6 +122,11 @@ function(add_qml_test TEST_NAME)
 
     add_test(NAME ${APP_NAME}_${TEST_NAME}_test COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}/${APP_NAME}_${TEST_NAME}_test${EXE_SUFFIX} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME})
 
+    set_tests_properties(
+        ${APP_NAME}_${TEST_NAME}_test PROPERTIES ENVIRONMENT_MODIFICATION
+                                                 "${RUNTIME_ENV}"
+    )
+
     add_valgrind_test(${APP_NAME}_${TEST_NAME}_test ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}/${APP_NAME}_${TEST_NAME}_test${EXE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME})
 
     set_target_properties(${APP_NAME}_${TEST_NAME}_test PROPERTIES

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -10,21 +10,26 @@ add_subdirectory(qt_types_standalone)
 # Create helper method which adds relevent tests for the given acceptance test
 function(add_acceptance_tests TEST_NAME)
     set(NAME_WITH_PREFIX test_${TEST_NAME})
+    set(TARGET_NAME tests_${TEST_NAME})
 
     # Add all the normal tests used on the other modules
-    add_test_cargo(${NAME_WITH_PREFIX} "${CMAKE_CURRENT_SOURCE_DIR}/${TEST_NAME}/rust/Cargo.toml" DOCTESTS_OFF)
+    add_test_cargo(
+        ${NAME_WITH_PREFIX}
+        "${CMAKE_CURRENT_SOURCE_DIR}/${TEST_NAME}/rust/Cargo.toml" DOCTESTS_OFF
+    )
 
-    # The executable itself is a test that needs to be run
-    if(WIN32)
-        set(EXE_SUFFIX ".exe")
-    else()
-        set(EXE_SUFFIX "")
-    endif()
+    set(CPP_TEST_NAME ${NAME_WITH_PREFIX}_cpp_tests)
+    add_test(NAME ${CPP_TEST_NAME} COMMAND $<TARGET_FILE:${TARGET_NAME}>)
 
-    add_test(NAME ${NAME_WITH_PREFIX}_cpp_tests COMMAND "${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}/tests_${TEST_NAME}${EXE_SUFFIX}")
+    set_tests_properties(
+        ${CPP_TEST_NAME} PROPERTIES ENVIRONMENT_MODIFICATION "${RUNTIME_ENV}"
+    )
 
     # Add valgrind test
-    add_valgrind_test(${NAME_WITH_PREFIX} "${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}/tests_${TEST_NAME}${EXE_SUFFIX}" "${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}")
+    add_valgrind_test(
+        ${NAME_WITH_PREFIX} $<TARGET_FILE:${TARGET_NAME}>
+        ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}
+    )
 endfunction()
 
 # Add tests for all the acceptance tests