Js/device testing (#6964)

* add a combined test executable

* disable a recently added test on mobile

* simplify link dependencies

* add missing namespace

* fix merge

* fix object-store benchmarks on android

CHANGELOG.md

@@ -17,7 +17,7 @@
 -----------
 
 ### Internals
-* None.
+* Added a CombinedTests target that runs tests from object-store, sync, and core together. ([PR #6964](https://github.com/realm/realm-core/pull/6964))
 
 ----------------------------------------------
 

Jenkinsfile

@@ -668,32 +668,19 @@ def doBuildApplePlatform(String platform, String buildType, boolean test = false
                 if (test) {
                     dir('build-xcode-platforms') {
                         if (platform != 'iphonesimulator') error 'Testing is only available for iOS Simulator'
-                        sh "xcodebuild -scheme CoreTests -configuration ${buildType} -sdk iphonesimulator -arch x86_64"
-                        // sh "xcodebuild -scheme SyncTests -configuration ${buildType} -sdk iphonesimulator -arch x86_64 IPHONEOS_DEPLOYMENT_TARGET=13"
-                        sh "xcodebuild -scheme ObjectStoreTests -configuration ${buildType} -sdk iphonesimulator -arch x86_64 IPHONEOS_DEPLOYMENT_TARGET=13"
+                        sh "xcodebuild -scheme CombinedTests -configuration ${buildType} -sdk iphonesimulator -arch x86_64"
 
                         def env = environment().collect { v -> "SIMCTL_CHILD_${v}" }
-                        def resultFile = "${WORKSPACE}/core-test-report.xml"
+                        def resultFile = "${WORKSPACE}/combined-test-report.xml"
                         withEnv(env + ["SIMCTL_CHILD_UNITTEST_XML=${resultFile}", "SIMCTL_CHILD_UNITTEST_SUITE_NAME=iOS-${buildType}-Core"]) {
-                            sh "$WORKSPACE/tools/run-in-simulator.sh 'test/${buildType}-${platform}/realm-tests.app' 'io.realm.CoreTests' '${resultFile}'"
-                        }
-                        // Sync tests currently don't work on iOS because they require an unimplemented server feature
-                        // resultFile = "${WORKSPACE}/sync-test-report.xml"
-                        // withEnv(env + ["SIMCTL_CHILD_UNITTEST_XML=${resultFile}", "SIMCTL_CHILD_UNITTEST_SUITE_NAME=iOS-${buildType}-Sync"]) {
-                        //     sh "$WORKSPACE/tools/run-in-simulator.sh 'test/${buildType}-${platform}/realm-sync-tests.app' 'io.realm.SyncTests' '${resultFile}'"
-                        // }
-                        resultFile = "${WORKSPACE}/object-store-test-report.xml"
-                        withEnv(env + ["SIMCTL_CHILD_UNITTEST_XML=${resultFile}", "SIMCTL_CHILD_UNITTEST_SUITE_NAME=iOS-${buildType}-Object-Store"]) {
-                            sh "$WORKSPACE/tools/run-in-simulator.sh 'test/object-store/${buildType}-${platform}/realm-object-store-tests.app' 'io.realm.ObjectStoreTests' '${resultFile}'"
+                            sh "$WORKSPACE/tools/run-in-simulator.sh 'test/${buildType}-${platform}/realm-combined-tests.app' 'io.realm.CombinedTests' '${resultFile}'"
                         }
                     }
                 }
             }
 
             if (test) {
-                junit testResults: 'core-test-report.xml'
-                // junit testResults: 'sync-test-report.xml'
-                junit testResults: 'object-store-test-report.xml'
+                junit testResults: 'combined-test-report.xml'
             }
 
             String tarball = "realm-${buildType}-${gitDescribeVersion}-${platform}-devel.tar.gz";

test/CMakeLists.txt

@@ -3,7 +3,7 @@ add_custom_target(benchmarks)
 add_subdirectory(object-store)
 
 # AFL and LIBFUZZER not yet supported by Windows
-if(NOT CMAKE_SYSTEM_NAME MATCHES "^Windows" AND NOT EMSCRIPTEN)
+if(NOT CMAKE_SYSTEM_NAME MATCHES "^Windows" AND NOT EMSCRIPTEN AND NOT ANDROID)
     add_subdirectory(fuzzy)
     add_subdirectory(realm-fuzzer)
 endif()
@@ -23,7 +23,6 @@ set(CORE_TEST_SOURCES
     test_lang_bind_helper.cpp
     test_parser.cpp
 
-    test_all.cpp
     test_alloc.cpp
     test_array.cpp
     test_array_blob.cpp
@@ -126,8 +125,6 @@ set_source_files_properties(test_query_geo.cpp PROPERTIES
     COMPILE_FLAGS "$<$<CXX_COMPILER_ID:GNU>: -Wno-unknown-pragmas>"
 )
 
-# FIXME: Benchmarks
-
 if (MSVC)
     append_source_file_compile_options(
         FILES test_parser.cpp test_table.cpp test_query.cpp test_query_big.cpp
@@ -136,12 +133,17 @@ if (MSVC)
 endif()
 
 # Resources required for running the tests
-file(GLOB REQUIRED_TEST_FILES RELATIVE ${CMAKE_CURRENT_BINARY_DIR}
+file(GLOB REQUIRED_TEST_FILES
      "*.json"
      "*.realm"
      "expect_string.txt")
 
-add_executable(CoreTests main.cpp ${CORE_TESTS} ${REQUIRED_TEST_FILES} ${REALM_TEST_HEADERS})
+add_library(CoreTestLib OBJECT ${CORE_TESTS} ${REQUIRED_TEST_FILES} ${REALM_TEST_HEADERS})
+enable_stdfilesystem(CoreTestLib)
+target_link_libraries(CoreTestLib QueryParser)
+
+add_executable(CoreTests main.cpp test_all.cpp ${REQUIRED_TEST_FILES})
+target_link_libraries(CoreTests CoreTestLib TestUtil)
 set_target_resources(CoreTests "${REQUIRED_TEST_FILES}")
 set_target_properties(CoreTests PROPERTIES
     OUTPUT_NAME "realm-tests"
@@ -167,7 +169,6 @@ if(WINDOWS_STORE)
     set_property(SOURCE ${UWP_ASSETS} PROPERTY VS_DEPLOYMENT_LOCATION "Assets")
 endif()
 
-target_link_libraries(CoreTests TestUtil QueryParser)
 enable_stdfilesystem(CoreTests)
 
 if(UNIX AND NOT APPLE)
@@ -179,9 +180,9 @@ target_include_directories(CoreTests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
 
 add_bundled_test(CoreTests)
 
+# SyncTests
 if(REALM_ENABLE_SYNC)
     set(SYNC_TESTS
-        test_all.cpp
         test_array_sync.cpp
         test_changeset_encoding.cpp
         test_client_reset.cpp
@@ -210,27 +211,15 @@ if(REALM_ENABLE_SYNC)
         test_util_websocket.cpp
     )
 
-    set(TEST_HEADERS
+    set(SYNC_TEST_HEADERS
         test.hpp
         test_all.hpp
         fuzz_tester.hpp
         peer.hpp
         sync_fixtures.hpp
     )
 
-    set(TEST_CLIENT_SOURCES
-        client/main.cpp
-        client/peer.cpp
-    )
-
-    set(TEST_CLIENT_HEADERS
-        client/metrics.hpp
-        client/peer.hpp
-        client/object_observer.hpp
-        client/statistics.hpp
-    )
-
-    file(GLOB TEST_RESOURCES RELATIVE ${CMAKE_CURRENT_BINARY_DIR}
+    file(GLOB SYNC_TEST_RESOURCES
         *.json *.pem
         ../certificate-authority/certs/*
         ../certificate-authority/root-ca/*.pem
@@ -243,16 +232,49 @@ if(REALM_ENABLE_SYNC)
         list(APPEND SYNC_TESTS test.manifest)
     endif()
 
-    add_executable(SyncTests main.cpp ${SYNC_TESTS} ${TEST_HEADERS} ${TEST_RESOURCES})
+    add_library(SyncTestLib OBJECT ${SYNC_TESTS} ${SYNC_TEST_HEADERS} ${SYNC_TEST_RESOURCES})
+    enable_stdfilesystem(SyncTestLib)
+    target_link_libraries(SyncTestLib Sync SyncServer Storage)
+
+    add_executable(SyncTests main.cpp test_all.cpp ${SYNC_TEST_RESOURCES})
     set_target_properties(SyncTests PROPERTIES OUTPUT_NAME "realm-sync-tests")
-    set_target_resources(SyncTests "${TEST_RESOURCES}")
+    set_target_resources(SyncTests "${SYNC_TEST_RESOURCES}")
     enable_stdfilesystem(SyncTests)
-    # Sync lib is included with SyncServer
-    target_link_libraries(SyncTests TestUtil SyncServer)
+    target_link_libraries(SyncTests SyncTestLib TestUtil)
     add_bundled_test(SyncTests)
 
     if(UNIX AND NOT APPLE)
         # This enables symbols in backtraces
         target_link_libraries(SyncTests "-rdynamic")
     endif()
 endif()
+
+# CombinedTests
+get_directory_property(OS_TEST_RESOURCES DIRECTORY object-store TEST_RESOURCES)
+set(COMBINED_RESOURCES "${REQUIRED_TEST_FILES};${SYNC_TEST_RESOURCES};${OS_TEST_RESOURCES}")
+
+add_executable(CombinedTests combined_tests.cpp test_all.cpp ${COMBINED_RESOURCES})
+set_target_resources(CombinedTests "${COMBINED_RESOURCES}")
+set_target_properties(CombinedTests PROPERTIES
+    OUTPUT_NAME "realm-combined-tests"
+    VS_WINDOWS_TARGET_PLATFORM_MIN_VERSION 10.0.17134.0
+)
+target_include_directories(CombinedTests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}" "object-store")
+
+if(WINDOWS_STORE)
+    target_sources(CombinedTests PRIVATE ${UWP_SOURCES})
+endif()
+
+if(REALM_ENABLE_SYNC)
+    target_link_libraries(CombinedTests ObjectStoreTestLib CoreTestLib SyncTestLib TestUtil)
+else()
+    target_link_libraries(CombinedTests ObjectStoreTestLib CoreTestLib TestUtil)
+endif()
+
+enable_stdfilesystem(CombinedTests)
+
+if(UNIX AND NOT APPLE)
+    # This enables symbols in backtraces
+    target_link_libraries(CombinedTests "-rdynamic")
+endif()
+

test/combined_tests.cpp

@@ -0,0 +1,38 @@
+/*************************************************************************
+ *
+ * Copyright 2023 Realm Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ **************************************************************************/
+
+#include "test_all.hpp"
+#include "util/test_path.hpp"
+
+#include <iostream>
+
+// see test/object-store/test_runner.cpp
+extern int run_object_store_tests(int, const char**);
+
+int main(int argc, const char* argv[])
+{
+    if (!realm::test_util::initialize_test_path(argc, argv))
+        return 1;
+    int status = test_all();
+    if (status) {
+        std::cerr << "core and sync tests failed: " << status << std::endl;
+        return status;
+    }
+    std::cout << "core and sync tests passed\n";
+    return run_object_store_tests(argc, argv);
+}

test/object-store/CMakeLists.txt

@@ -7,13 +7,15 @@ set(HEADERS
 )
 
 set(SOURCES
+    # slowest to compile first
+    set.cpp
+
     backup.cpp
     collection_change_indices.cpp
     dictionary.cpp
     frozen_objects.cpp
     index_set.cpp
     list.cpp
-    main.cpp
     migrations.cpp
     object.cpp
     object_store.cpp
@@ -22,12 +24,12 @@ set(SOURCES
     results.cpp
     schema.cpp
     sectioned_results.cpp
-    set.cpp
+    test_runner.cpp
     thread_safe_reference.cpp
     transaction_log_parsing.cpp
     uuid.cpp
     c_api/c_api.cpp
-    c_api/c_api.c
+    c_api/c_api_file_tests.c
 
     util/event_loop.cpp
     util/test_file.cpp
@@ -38,9 +40,7 @@ if (REALM_ENABLE_GEOSPATIAL)
     list(APPEND SOURCES geospatial.cpp)
 endif()
 
-file(GLOB RESOURCES RELATIVE ${CMAKE_CURRENT_BINARY_DIR}
-    ../*.pem
-     *.realm)
+file(GLOB RESOURCES "*.realm" "../*.pem")
 
 if(REALM_ENABLE_AUTH_TESTS)
     list(APPEND SOURCES util/sync/baas_admin_api.cpp)
@@ -78,16 +78,22 @@ if(REALM_ENABLE_SYNC)
     endif()
 endif()
 
-add_executable(ObjectStoreTests ${SOURCES} ${HEADERS} ${RESOURCES})
-set_target_properties(ObjectStoreTests PROPERTIES OUTPUT_NAME realm-object-store-tests)
-set_target_resources(ObjectStoreTests "${RESOURCES}")
+set_property(DIRECTORY PROPERTY TEST_RESOURCES "${RESOURCES}")
+
+add_library(ObjectStoreTestLib OBJECT ${SOURCES} ${HEADERS} ${RESOURCES})
 
 if(MSVC)
     # increase the number of sections supported in an obj file for the heavily templated tests
-    target_compile_options(ObjectStoreTests PRIVATE /bigobj)
+    target_compile_options(ObjectStoreTestLib PRIVATE /bigobj)
 endif()
 
-target_link_libraries(ObjectStoreTests Catch2::Catch2 ObjectStore TestUtil RealmFFIStatic)
+target_link_libraries(ObjectStoreTestLib Catch2::Catch2 ObjectStore RealmFFIStatic TestUtil)
+enable_stdfilesystem(ObjectStoreTestLib)
+
+add_executable(ObjectStoreTests main.cpp ${RESOURCES})
+set_target_properties(ObjectStoreTests PROPERTIES OUTPUT_NAME realm-object-store-tests)
+target_link_libraries(ObjectStoreTests ObjectStoreTestLib TestUtil)
+set_target_resources(ObjectStoreTests "${RESOURCES}")
 enable_stdfilesystem(ObjectStoreTests)
 create_coverage_target(generate-coverage ObjectStoreTests)
 
@@ -120,15 +126,15 @@ if(REALM_ENABLE_SYNC)
 endif()
 
 if(REALM_ENABLE_SYNC)
-    target_link_libraries(ObjectStoreTests SyncServer)
+    target_link_libraries(ObjectStoreTestLib SyncServer)
     option(REALM_ENABLE_AUTH_TESTS "" OFF)
     if(REALM_ENABLE_AUTH_TESTS)
         if(NOT REALM_MONGODB_ENDPOINT)
             message(FATAL_ERROR "REALM_MONGODB_ENDPOINT must be set when specifying REALM_ENABLE_AUTH_TESTS.")
         endif()
 
         message(STATUS "Auth tests enabled: ${REALM_MONGODB_ENDPOINT}")
-        target_compile_definitions(ObjectStoreTests PRIVATE
+        target_compile_definitions(ObjectStoreTestLib PRIVATE
             REALM_ENABLE_AUTH_TESTS=1
             REALM_MONGODB_ENDPOINT="${REALM_MONGODB_ENDPOINT}"
         )
@@ -141,18 +147,18 @@ if(REALM_ENABLE_SYNC)
         endif()
 
         find_package(CURL REQUIRED)
-        target_link_libraries(ObjectStoreTests CURL::libcurl)
+        target_link_libraries(ObjectStoreTestLib CURL::libcurl)
     endif()
 endif()
 
 if(REALM_TEST_LOGGING)
-    target_compile_definitions(ObjectStoreTests PRIVATE
+    target_compile_definitions(ObjectStoreTestLib PRIVATE
         TEST_ENABLE_LOGGING=1
     )
 
     if(REALM_TEST_LOGGING_LEVEL)
         message(STATUS "Test logging level: ${REALM_TEST_LOGGING_LEVEL}")
-        target_compile_definitions(ObjectStoreTests PRIVATE
+        target_compile_definitions(ObjectStoreTestLib PRIVATE
             TEST_LOGGING_LEVEL=${REALM_TEST_LOGGING_LEVEL}
         )
     else()
@@ -168,7 +174,7 @@ if(REALM_TEST_TIMEOUT_EXTRA)
     message(STATUS "Test wait timeouts extended by ${REALM_TEST_TIMEOUT_EXTRA} seconds")
 endif()
 
-target_include_directories(ObjectStoreTests PRIVATE
+target_include_directories(ObjectStoreTestLib PRIVATE
     ${CATCH_INCLUDE_DIR}
     ${JSON_INCLUDE_DIR}
     ${CMAKE_CURRENT_SOURCE_DIR}
@@ -177,7 +183,7 @@ target_include_directories(ObjectStoreTests PRIVATE
 # on Apple platforms we use the built-in CFRunLoop
 # on WebAssembly we use an Emscripten-specific Scheduler and runloop
 # everywhere else it's libuv, except UWP where it doesn't build
-if(NOT APPLE AND NOT EMSCRIPTEN AND NOT WINDOWS_STORE)
+if(NOT APPLE AND NOT EMSCRIPTEN AND NOT WINDOWS_STORE AND NOT ANDROID)
     option(REALM_FETCH_MISSING_DEPENDENCIES "Download missing dependencies with CMake's FetchContent where possible" ON)
     if(REALM_FETCH_MISSING_DEPENDENCIES)
         find_package(LibUV)
@@ -204,8 +210,8 @@ if(NOT APPLE AND NOT EMSCRIPTEN AND NOT WINDOWS_STORE)
         set(libuv_target uv_a)
     endif()
 
-    target_link_libraries(ObjectStoreTests ${libuv_target})
-    target_compile_definitions(ObjectStoreTests PRIVATE TEST_SCHEDULER_UV=1)
+    target_link_libraries(ObjectStoreTestLib ${libuv_target})
+    target_compile_definitions(ObjectStoreTestLib PRIVATE TEST_SCHEDULER_UV=1)
 
     if (MSVC)
         get_target_property(comp_opts ${libuv_target} COMPILE_OPTIONS)
@@ -216,6 +222,6 @@ endif()
 
 add_subdirectory(notifications-fuzzer)
 
-if(NOT EMSCRIPTEN AND NOT WINDOWS_STORE)
+if(NOT EMSCRIPTEN AND NOT WINDOWS_STORE AND NOT ANDROID)
     add_subdirectory(benchmarks)
 endif()

test/object-store/benchmarks/CMakeLists.txt

@@ -43,7 +43,7 @@ add_dependencies(benchmarks object-store-benchmarks)
 
 # on Apple platforms we use the built-in CFRunLoop
 # everywhere else it's libuv, except UWP where it doesn't build
-if(NOT APPLE AND NOT WINDOWS_STORE AND NOT EMSCRIPTEN)
+if(NOT APPLE AND NOT WINDOWS_STORE AND NOT EMSCRIPTEN AND NOT ANDROID)
     # libuv_target is defined in the parent CMakeLists.txt file
     target_link_libraries(object-store-benchmarks ${libuv_target})
     target_compile_definitions(object-store-benchmarks PRIVATE TEST_SCHEDULER_UV=1)