diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf2e98e1d2..9eb8ada9b1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ ----------- ### Internals -* None. +* Fix emscripten build and add emscripten debug/release compile tasks to evergreen. ([PR #7916](https://github.com/realm/realm-core/pull/7916)) ---------------------------------------------- diff --git a/evergreen/config.yml b/evergreen/config.yml index abdc37706ff..a1bb6ccc141 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -197,6 +197,48 @@ functions: -j ${max_jobs|$(grep -c proc /proc/cpuinfo)} \ $target + "compile emscripten": + - command: shell.exec + params: + working_dir: realm-core + shell: bash + script: |- + set -o errexit + set -o pipefail + set -o verbose + + if [ -n "${cmake_bindir|}" ]; then + export CMAKE="$(./evergreen/abspath.sh "${cmake_bindir}/cmake")" + else + export CMAKE="cmake" + fi + + # Check out the Emscripten SDK and activate the specified version + echo "Activating emscripten SDK version: ${emsdk_version|latest}" + git clone https://github.com/emscripten-core/emsdk.git + + pushd emsdk/ > /dev/null # realm-core/emsdk + ./emsdk install ${emsdk_version|latest} + ./emsdk activate ${emsdk_version|latest} + source ./emsdk_env.sh + popd > /dev/null # realm-core + + if [[ "$(uname -s)" =~ Darwin* ]]; then + NPROC="$(sysctl -n hw.ncpu)" + else + NPROC="$(nproc)" + fi + + mkdir -p build-emscripten + pushd build-emscripten > /dev/null # realm-core/build-emscripten + + # shellcheck disable=SC2086,SC2090 + emcmake $CMAKE -D CMAKE_BUILD_TYPE="${cmake_build_type|Debug}" \ + .. + + make "-j$NPROC" 2>&1 + popd > /dev/null # realm-core + "run benchmark": - command: shell.exec params: @@ -749,6 +791,12 @@ tasks: commands: - func: "compile" +- name: compile-emscripten + exec_timeout_secs: 1800 + tags: [ "for_pull_requests" ] + commands: + - func: "compile emscripten" + - name: package exec_timeout_secs: 1800 commands: @@ -1297,6 +1345,15 @@ task_groups: - compile - .test_suite +- name: compile_emscripten + max_hosts: 1 + setup_group_can_fail_task: true + setup_group: + - func: "fetch source" + - func: "fetch binaries" + tasks: + - compile-emscripten + # Runs object-store-tests against baas running on remote host and runs # the network simulation tests as a separate task for nightly builds - name: network_tests @@ -1648,6 +1705,27 @@ buildvariants: tasks: - name: fuzzer-tests +- name: ubuntu-emscripten + display_name: "Ubuntu (Emscripten x86_64)" + run_on: ubuntu2204-large + expansions: + cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-linux-x86_64.tar.gz" + cmake_bindir: "./cmake_binaries/bin" + emsdk_version: "3.1.37" + tasks: + - name: compile_emscripten + +- name: ubuntu-emscripten-release + display_name: "Ubuntu (Emscripten x86_64 Release build)" + run_on: ubuntu2204-large + expansions: + cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-linux-x86_64.tar.gz" + cmake_bindir: "./cmake_binaries/bin" + cmake_build_type: RelWithDebInfo + emsdk_version: "3.1.37" + tasks: + - name: compile_emscripten + # disable these builders since there are constantly failing and not yet ready for nightly builds # - name: ubuntu2004-network-nonideal # display_name: "Ubuntu 20.04 x86_64 (Utunbu2004 - nonideal transfer)" diff --git a/src/realm/alloc_slab.hpp b/src/realm/alloc_slab.hpp index df7cf413b9e..d3392cc3d6e 100644 --- a/src/realm/alloc_slab.hpp +++ b/src/realm/alloc_slab.hpp @@ -147,8 +147,8 @@ class SlabAlloc : public Allocator { /// This can happen if the conflicting thread (or process) terminates or /// crashes before the next retry. /// - /// \throw FileAccessError - /// \throw SlabAlloc::Retry + /// \throw FileAccessError if unable to access the file + /// \throw SlabAlloc::Retry if the request cannot be completed right now ref_type attach_file(const std::string& file_path, Config& cfg, util::WriteObserver* write_observer = nullptr); /// @brief Expand or contract file @@ -179,7 +179,7 @@ class SlabAlloc : public Allocator { /// /// \sa own_buffer() /// - /// \throw InvalidDatabase + /// \throw InvalidDatabase if an error occurs while attaching the allocator ref_type attach_buffer(const char* data, size_t size); void init_in_memory_buffer(); diff --git a/src/realm/object-store/sync/app.cpp b/src/realm/object-store/sync/app.cpp index 2b2a8b0fca0..9306807912a 100644 --- a/src/realm/object-store/sync/app.cpp +++ b/src/realm/object-store/sync/app.cpp @@ -210,12 +210,27 @@ SharedApp App::get_app(CacheMode mode, const AppConfig& config) NO_THREAD_SAFETY std::lock_guard lock(s_apps_mutex); auto& app = s_apps_cache[config.app_id][base_url_from_app_config(config)]; if (!app) { - app = std::make_shared(Private(), config); + app = App::make_app(config); } return app; } REALM_ASSERT(mode == CacheMode::Disabled); + return App::make_app(config); +} + +SharedApp App::make_app(const AppConfig& config) +{ +#ifdef __EMSCRIPTEN__ + if (!config.transport) { + // Make a copy and provide a default transport if not provided + AppConfig config_copy = config; + config_copy.transport = std::make_shared<_impl::EmscriptenNetworkTransport>(); + return std::make_shared(Private(), config_copy); + } return std::make_shared(Private(), config); +#else + return std::make_shared(Private(), config); +#endif } SharedApp App::get_cached_app(const std::string& app_id, const std::optional& base_url) @@ -257,11 +272,6 @@ App::App(Private, const AppConfig& config) , m_metadata_store(create_metadata_store(config, *m_file_manager)) , m_sync_manager(SyncManager::create(config.sync_client_config)) { -#ifdef __EMSCRIPTEN__ - if (!m_config.transport) { - m_config.transport = std::make_shared<_impl::EmscriptenNetworkTransport>(); - } -#endif REALM_ASSERT(m_config.transport); // if a base url is provided, then verify the value diff --git a/src/realm/object-store/sync/app.hpp b/src/realm/object-store/sync/app.hpp index 40508984381..3fe735ca648 100644 --- a/src/realm/object-store/sync/app.hpp +++ b/src/realm/object-store/sync/app.hpp @@ -449,6 +449,8 @@ class App : public std::enable_shared_from_this, private: const AppConfig m_config; + static SharedApp make_app(const AppConfig& config); + util::CheckedMutex m_route_mutex; // The following variables hold the different paths to Atlas, depending on the // request being performed diff --git a/src/realm/sync/instruction_applier.cpp b/src/realm/sync/instruction_applier.cpp index 3106858f890..60dfa4ae0c9 100644 --- a/src/realm/sync/instruction_applier.cpp +++ b/src/realm/sync/instruction_applier.cpp @@ -373,12 +373,12 @@ void InstructionApplier::operator()(const Instruction::Update& instr) field_name, table_name, col.get_type(), mixed_ptr->get_type()); } } - else if (const auto obj_val_ptr = mpark::get_if(&arg)) { + else if (mpark::get_if(&arg)) { if (obj.is_null(col)) { obj.create_and_set_linked_object(col); } } - else if (const auto erase_ptr = mpark::get_if(&arg)) { + else if (mpark::get_if(&arg)) { m_applier->bad_transaction_log("Update: Dictionary erase at object field"); } else if (mpark::get_if(&arg)) { diff --git a/src/realm/util/file.cpp b/src/realm/util/file.cpp index 3f059a0a9dd..3e32d530084 100644 --- a/src/realm/util/file.cpp +++ b/src/realm/util/file.cpp @@ -1774,7 +1774,7 @@ bool File::MapBase::try_extend_to(size_t size) noexcept #ifndef _WIN32 char* extension_start_addr = (char*)m_addr + m_size; size_t extension_size = size - m_size; - size_t extension_start_offset = m_offset + m_size; + size_t extension_start_offset = static_cast(m_offset) + m_size; #if REALM_ENABLE_ENCRYPTION if (m_encrypted_mapping) { void* got_addr = ::mmap(extension_start_addr, extension_size, PROT_READ | PROT_WRITE, diff --git a/test/util/CMakeLists.txt b/test/util/CMakeLists.txt index 4f652f1d6a8..8f04c9454be 100644 --- a/test/util/CMakeLists.txt +++ b/test/util/CMakeLists.txt @@ -58,8 +58,8 @@ if(UNIX AND NOT APPLE) find_library(LIBRT rt) if(LIBRT) target_link_libraries(TestUtil ${LIBRT}) - # Android has librt included in libc - elseif(NOT ANDROID) + # Android and Emscripten have librt included in libc + elseif(NOT ANDROID AND NOT EMSCRIPTEN) message(WARNING "librt was not found. This means that the benchmarks will not be able to link properly.") endif() endif() diff --git a/tools/cross_compile.sh b/tools/cross_compile.sh index 2ef59406733..0bdd966e99e 100755 --- a/tools/cross_compile.sh +++ b/tools/cross_compile.sh @@ -6,40 +6,50 @@ set -e SCRIPT=$(basename "${BASH_SOURCE[0]}") function usage { - echo "Usage: ${SCRIPT} -t -o -v [-a ] [-f ]" + echo "Usage: ${SCRIPT} -t -o [-v ] [-a ] [-f ]" echo "" echo "Arguments:" - echo " build_type=" - echo " target_os=" - echo " android_abi=" + echo " build_type Release|Debug|MinSizeDebug|RelWithDebInfo" + echo " target_os android|iphoneos|iphonesimulator|watchos|" + echo " watchsimulator|appletvos|appletvsimulator|" + echo " emscripten" + echo " android_abi armeabi-v7a|x86|x86_64|arm64-v8a" exit 1; } +# Variables +VERSION= +CMAKE_FLAGS= +EMCMAKE=emcmake +CMAKE=${CMAKE:-cmake} + # Parse the options while getopts ":o:a:t:v:f:" opt; do case "${opt}" in o) OS=${OPTARG} - [ "${OS}" == "android" ] || - [ "${OS}" == "iphoneos" ] || - [ "${OS}" == "iphonesimulator" ] || - [ "${OS}" == "watchos" ] || - [ "${OS}" == "watchsimulator" ] || - [ "${OS}" == "appletvos" ] || - [ "${OS}" == "appletvsimulator" ] || usage + [[ "${OS}" == "android" ]] || + [[ "${OS}" == "iphoneos" ]] || + [[ "${OS}" == "iphonesimulator" ]] || + [[ "${OS}" == "watchos" ]] || + [[ "${OS}" == "watchsimulator" ]] || + [[ "${OS}" == "appletvos" ]] || + [[ "${OS}" == "appletvsimulator" ]] || + [[ "${OS}" == "emscripten" ]] || usage ;; a) ARCH=${OPTARG} - [ "${ARCH}" == "armeabi-v7a" ] || - [ "${ARCH}" == "x86" ] || - [ "${ARCH}" == "x86_64" ] || - [ "${ARCH}" == "arm64-v8a" ] || usage + [[ "${ARCH}" == "armeabi-v7a" ]] || + [[ "${ARCH}" == "x86" ]] || + [[ "${ARCH}" == "x86_64" ]] || + [[ "${ARCH}" == "arm64-v8a" ]] || usage ;; t) BUILD_TYPE=${OPTARG} - [ "${BUILD_TYPE}" == "Debug" ] || - [ "${BUILD_TYPE}" == "MinSizeDebug" ] || - [ "${BUILD_TYPE}" == "Release" ] || usage + [[ "${BUILD_TYPE}" == "Debug" ]] || + [[ "${BUILD_TYPE}" == "MinSizeDebug" ]] || + [[ "${BUILD_TYPE}" == "Release" ]] || + [[ "${BUILD_TYPE}" == "RelWithDebInfo" ]] || usage ;; v) VERSION=${OPTARG};; f) CMAKE_FLAGS=${OPTARG};; @@ -50,52 +60,86 @@ done shift $((OPTIND-1)) # Check for obligatory fields -if [ -z "${OS}" ] || [ -z "${BUILD_TYPE}" ]; then - echo "ERROR: options -o, -t and -v are always needed"; +if [[ -z "${OS}" ]] || [[ -z "${BUILD_TYPE}" ]]; then + echo "ERROR: options -o and -t are required"; usage fi -# Check for android-related obligatory fields +if [[ -n "${VERSION}" ]]; then + # shellcheck disable=SC2089 + CMAKE_FLAGS="-D REALM_VERSION='${VERSION}' ${CMAKE_FLAGS}" +fi + if [[ "${OS}" == "android" ]]; then + # Check for android-related obligatory fields if [[ -z "${ARCH}" ]]; then - echo "ERROR: option -a is needed for android builds"; + echo "ERROR: option -a is required for android builds"; usage elif [[ -z "${ANDROID_NDK}" ]]; then echo "ERROR: set ANDROID_NDK to the top level path for the Android NDK"; usage fi -fi -if [ "${OS}" == "android" ]; then mkdir -p "build-android-${ARCH}-${BUILD_TYPE}" cd "build-android-${ARCH}-${BUILD_TYPE}" || exit 1 - cmake -D CMAKE_SYSTEM_NAME=Android \ - -D CMAKE_ANDROID_NDK="${ANDROID_NDK}" \ - -D CMAKE_INSTALL_PREFIX=install \ - -D CMAKE_BUILD_TYPE="${BUILD_TYPE}" \ - -D CMAKE_ANDROID_ARCH_ABI="${ARCH}" \ - -D CMAKE_TOOLCHAIN_FILE="./tools/cmake/android.toolchain.cmake" \ - -D REALM_ENABLE_ENCRYPTION=1 \ - -D REALM_VERSION="${VERSION}" \ - -D CPACK_SYSTEM_NAME="Android-${ARCH}" \ - -D CMAKE_MAKE_PROGRAM=ninja \ - -G Ninja \ - ${CMAKE_FLAGS} \ - .. + + # shellcheck disable=SC2086,SC2090 + ${CMAKE} -D CMAKE_SYSTEM_NAME=Android \ + -D CMAKE_ANDROID_NDK="${ANDROID_NDK}" \ + -D CMAKE_INSTALL_PREFIX=install \ + -D CMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -D CMAKE_ANDROID_ARCH_ABI="${ARCH}" \ + -D CMAKE_TOOLCHAIN_FILE="./tools/cmake/android.toolchain.cmake" \ + -D REALM_ENABLE_ENCRYPTION=On \ + -D CPACK_SYSTEM_NAME="Android-${ARCH}" \ + -D CMAKE_MAKE_PROGRAM=ninja \ + -G Ninja \ + ${CMAKE_FLAGS} \ + .. ninja -v ninja package +elif [[ "${OS}" == "emscripten" ]]; then + if [[ -n "${EMSDK}" ]]; then + EMCMAKE="${EMSDK}/upstream/emscripten/emcmake" + if ! [[ -e "${EMCMAKE}" ]]; then + echo "ERROR: emcmake not found: ${EMCMAKE}" + usage + fi + elif ! which emcmake > /dev/null; then + echo "ERROR: emcmake not found in path or set EMSDK to the" + echo " top level emscripten emsdk directory" + usage + fi + + if [[ "$(uname -s)" =~ Darwin* ]]; then + NPROC="$(sysctl -n hw.ncpu)" + else + NPROC="$(nproc)" + fi + + mkdir -p build-emscripten + cd build-emscripten || exit 1 + + # shellcheck disable=SC2086,SC2090 + ${EMCMAKE} ${CMAKE} -D CMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -D REALM_NO_TESTS=On \ + -D REALM_BUILD_LIB_ONLY=On \ + ${CMAKE_FLAGS} \ + .. + + make "-j${NPROC}" 2>&1 else mkdir -p build-xcode-platforms cd build-xcode-platforms || exit 1 - cmake -D CMAKE_TOOLCHAIN_FILE="../tools/cmake/xcode.toolchain.cmake" \ - -D CMAKE_BUILD_TYPE="${BUILD_TYPE}" \ - -D REALM_NO_TESTS=1 \ - -D REALM_BUILD_LIB_ONLY=1 \ - -D REALM_VERSION="${VERSION}" \ - ${CMAKE_FLAGS} \ - -G Xcode .. + # shellcheck disable=SC2086,SC2090 + ${CMAKE} -D CMAKE_TOOLCHAIN_FILE="../tools/cmake/xcode.toolchain.cmake" \ + -D CMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -D REALM_NO_TESTS=On \ + -D REALM_BUILD_LIB_ONLY=On \ + ${CMAKE_FLAGS} \ + -G Xcode .. xcodebuild -scheme ALL_BUILD -configuration "${BUILD_TYPE}" -destination "generic/platform=${OS}" PLATFORM_NAME="${OS}" EFFECTIVE_PLATFORM_NAME="-${OS}" cpack -C "${BUILD_TYPE}" fi