From 1ca0865ec73238b5247ff557c934c8140f9d45b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Wed, 25 Aug 2021 16:10:54 +0200 Subject: [PATCH 1/5] Workaround issue when initiating sync on a copied realm file The problem is that the server does not know that the client has already integrated some server versions, and therefore it sends down 0 as progress/upload/last_integrated_server_version. On the client this number may be bigger and in this case that number will not be overwritten with 0. There may still be issues on the server, however. --- src/realm/sync/noinst/client_history_impl.cpp | 12 +- test/object-store/sync/app.cpp | 121 ++++++++++++++++++ 2 files changed, 129 insertions(+), 4 deletions(-) diff --git a/src/realm/sync/noinst/client_history_impl.cpp b/src/realm/sync/noinst/client_history_impl.cpp index 29466f26624..f4f5ac14168 100644 --- a/src/realm/sync/noinst/client_history_impl.cpp +++ b/src/realm/sync/noinst/client_history_impl.cpp @@ -953,8 +953,10 @@ void ClientHistoryImpl::update_sync_progress(const SyncProgress& progress, version_type(root.get_as_ref_or_tagged(s_progress_download_client_version_iip).get_as_int())); REALM_ASSERT(progress.upload.client_version >= version_type(root.get_as_ref_or_tagged(s_progress_upload_client_version_iip).get_as_int())); - REALM_ASSERT(progress.upload.last_integrated_server_version >= - version_type(root.get_as_ref_or_tagged(s_progress_upload_server_version_iip).get_as_int())); + if (progress.upload.last_integrated_server_version > 0) { + REALM_ASSERT(progress.upload.last_integrated_server_version >= + version_type(root.get_as_ref_or_tagged(s_progress_upload_server_version_iip).get_as_int())); + } auto uploaded_bytes = std::uint_fast64_t(root.get_as_ref_or_tagged(s_progress_uploaded_bytes_iip).get_as_int()); auto previous_upload_client_version = @@ -971,8 +973,10 @@ void ClientHistoryImpl::update_sync_progress(const SyncProgress& progress, RefOrTagged::make_tagged(progress.latest_server_version.salt)); // Throws root.set(s_progress_upload_client_version_iip, RefOrTagged::make_tagged(progress.upload.client_version)); // Throws - root.set(s_progress_upload_server_version_iip, - RefOrTagged::make_tagged(progress.upload.last_integrated_server_version)); // Throws + if (progress.upload.last_integrated_server_version > 0) { + root.set(s_progress_upload_server_version_iip, + RefOrTagged::make_tagged(progress.upload.last_integrated_server_version)); // Throws + } if (downloadable_bytes) { root.set(s_progress_downloadable_bytes_iip, RefOrTagged::make_tagged(*downloadable_bytes)); // Throws diff --git a/test/object-store/sync/app.cpp b/test/object-store/sync/app.cpp index 4cf35814cfa..04333b47c51 100644 --- a/test/object-store/sync/app.cpp +++ b/test/object-store/sync/app.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include "util/baas_admin_api.hpp" #include "util/event_loop.hpp" @@ -1988,6 +1989,126 @@ TEST_CASE("app: set new embedded object", "[sync][app]") { } } +TEST_CASE("app: make distributable client file", "[sync][app]") { + std::unique_ptr (*factory)() = [] { + return std::unique_ptr(new IntTestTransport); + }; + std::string base_url = get_base_url(); + REQUIRE(!base_url.empty()); + auto app_session = get_runtime_app_session(base_url); + + auto app_id = app_session.client_app_id; + auto config = App::Config{app_id, + factory, + base_url, + util::none, + Optional("A Local App Version"), + util::none, + "Object Store Platform Tests", + "Object Store Platform Version Blah", + "An sdk version"}; + + auto base_path = util::make_temp_dir(); + util::try_remove_dir_recursive(base_path); + util::try_make_dir(base_path); + util::try_make_dir(base_path + "/orig"); + util::try_make_dir(base_path + "/copy"); + + // Create realm file without client file id + { + TestSyncManager sync_manager(TestSyncManager::Config(config), {}); + auto app = sync_manager.app(); + app->log_in_with_credentials(AppCredentials::anonymous(), + [&](std::shared_ptr user, Optional error) { + REQUIRE(!error); + REQUIRE(user); + }); + + ThreadSafeReference realm_ref; + + realm::Realm::Config realm_config; + realm_config.sync_config = std::make_shared(app->current_user(), bson::Bson("foo")); + realm_config.sync_config->client_resync_mode = ClientResyncMode::Manual; + realm_config.sync_config->error_handler = [](std::shared_ptr, SyncError error) { + std::cerr << error.message << std::endl; + }; + realm_config.schema_version = 1; + realm_config.path = base_path + "/orig/default.realm"; + + std::mutex mutex; + auto task = realm::Realm::get_synchronized_realm(realm_config); + task->start([&](ThreadSafeReference ref, std::exception_ptr error) { + std::lock_guard lock(mutex); + REQUIRE(!error); + realm_ref = std::move(ref); + }); + util::EventLoop::main().run_until([&] { + std::lock_guard lock(mutex); + return bool(realm_ref); + }); + SharedRealm realm = Realm::get_shared_realm(std::move(realm_ref)); + + // Write some data + realm->begin_transaction(); + CppContext c; + Object::create(c, realm, "Person", + util::Any(realm::AnyDict{{"_id", util::Any(ObjectId::gen())}, + {"age", INT64_C(64)}, + {"firstName", std::string("Paul")}, + {"lastName", std::string("McCartney")}})); + realm->commit_transaction(); + wait_for_upload(*realm); + wait_for_download(*realm); + + realm->write_copy(base_path + "/copy/default.realm", BinaryData()); + + // Write some additional data + realm->begin_transaction(); + Object::create(c, realm, "Dog", + util::Any(realm::AnyDict{{"_id", util::Any(ObjectId::gen())}, + {"breed", std::string("stabyhoun")}, + {"name", std::string("albert")}, + {"realm_id", std::string("foo")}})); + realm->commit_transaction(); + wait_for_upload(*realm); + } + // Starting a new session based on the copy + { + TestSyncManager sync_manager(TestSyncManager::Config(config), {}); + auto app = sync_manager.app(); + app->log_in_with_credentials(AppCredentials::anonymous(), + [&](std::shared_ptr user, Optional error) { + REQUIRE(!error); + REQUIRE(user); + }); + + ThreadSafeReference realm_ref; + + realm::Realm::Config realm_config; + realm_config.sync_config = std::make_shared(app->current_user(), bson::Bson("foo")); + realm_config.sync_config->client_resync_mode = ClientResyncMode::Manual; + realm_config.sync_config->error_handler = [](std::shared_ptr, SyncError error) { + std::cerr << error.message << std::endl; + }; + realm_config.schema_version = 1; + realm_config.path = base_path + "/copy/default.realm"; + + SharedRealm realm = realm::Realm::get_shared_realm(realm_config); + wait_for_download(*realm); + + // Check that we can continue committing to this realm + realm->begin_transaction(); + CppContext c; + Object::create(c, realm, "Dog", + util::Any(realm::AnyDict{{"_id", util::Any(ObjectId::gen())}, + {"breed", std::string("bulldog")}, + {"name", std::string("fido")}, + {"realm_id", std::string("foo")}})); + realm->commit_transaction(); + wait_for_upload(*realm); + } +} + TEST_CASE("app: sync integration", "[sync][app]") { std::unique_ptr (*factory)() = [] { return std::unique_ptr(new IntTestTransport); From a00e55aa6b18153bd3fb66a8cddf2eb24a217d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 30 Aug 2021 09:10:33 +0200 Subject: [PATCH 2/5] Misc changes --- src/realm/exec/CMakeLists.txt | 4 ++-- tools/run_baas.sh | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100755 tools/run_baas.sh diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index 53b4407fc9f..9f1ed148081 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -24,14 +24,14 @@ target_link_libraries(RealmTrawler Storage) add_executable(RealmBrowser EXCLUDE_FROM_ALL realm_browser.cpp ) set_target_properties(RealmBrowser PROPERTIES - OUTPUT_NAME "realm-browser-6" + OUTPUT_NAME "realm-browser-10" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(RealmBrowser Storage) add_executable(Realm2JSON realm2json.cpp ) set_target_properties(Realm2JSON PROPERTIES - OUTPUT_NAME "realm2json-6" + OUTPUT_NAME "realm2json-10" DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} ) target_link_libraries(Realm2JSON Storage) diff --git a/tools/run_baas.sh b/tools/run_baas.sh new file mode 100755 index 00000000000..3290e9d04b8 --- /dev/null +++ b/tools/run_baas.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +PROJECT_DIR=$(git rev-parse --show-toplevel) +MDBREALM_TEST_SERVER_TAG=$(grep MDBREALM_TEST_SERVER_TAG ${PROJECT_DIR}/dependencies.list |cut -f 2 -d=) +docker run --rm -p 9090:9090 -v ~/.aws/credentials:/root/.aws/credentials -it docker.pkg.github.com/realm/ci/mongodb-realm-test-server:${MDBREALM_TEST_SERVER_TAG} From 0e9713a9ffbf45d776496b2211fa749eae927a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 30 Aug 2021 11:49:46 +0200 Subject: [PATCH 3/5] Make internal sync server accept pre-downloaded file If the IDENT message indicates that the newly created client has already integraded some server changesets, adjust the upload threshold accordingly- If the threshold is ahead of the upload cursor stored in the history, use the threshold values. --- src/realm/sync/noinst/server_history.cpp | 16 +++++++++++++--- src/realm/sync/server.cpp | 2 +- test/object-store/realm.cpp | 10 ++++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/realm/sync/noinst/server_history.cpp b/src/realm/sync/noinst/server_history.cpp index 49718546b58..a97c720f9ea 100644 --- a/src/realm/sync/noinst/server_history.cpp +++ b/src/realm/sync/noinst/server_history.cpp @@ -850,8 +850,16 @@ auto ServerHistory::do_bootstrap_client_session(SaltedFileIdent client_file_iden if (download_progress.server_version > current_server_version) return BootstrapError::bad_download_server_version; auto last_integrated_client_version = version_type(m_acc->cf_client_versions.get(client_file_index)); - if (download_progress.last_integrated_client_version > last_integrated_client_version) - return BootstrapError::bad_download_client_version; + if (download_progress.last_integrated_client_version > last_integrated_client_version) { + if (last_integrated_client_version == 0) { + // We assume we are booting on a pre-downloaded client file + last_integrated_client_version = download_progress.last_integrated_client_version; + recip_hist_base_version = download_progress.server_version; + } + else { + return BootstrapError::bad_download_client_version; + } + } // Validate `server_version` { @@ -1014,7 +1022,9 @@ bool ServerHistory::fetch_download_info(file_ident_type client_file_ident, Downl download_progress = download_progress_2; cumulative_byte_size_current = std::uint_fast64_t(cumulative_byte_size_current_2); cumulative_byte_size_total = std::uint_fast64_t(cumulative_byte_size_total_2); - upload_progress = UploadCursor{upload_client_version, upload_server_version}; + if (upload_client_version > upload_progress.client_version) { + upload_progress = UploadCursor{upload_client_version, upload_server_version}; + } return true; } diff --git a/src/realm/sync/server.cpp b/src/realm/sync/server.cpp index b726f82fc9a..1dc1b10fa0e 100644 --- a/src/realm/sync/server.cpp +++ b/src/realm/sync/server.cpp @@ -3639,7 +3639,7 @@ class Session final : private FileIdentReceiver { bool body_is_compressed = false; version_type end_version = last_server_version.version; DownloadCursor download_progress; - UploadCursor upload_progress = {0, 0}; + UploadCursor upload_progress = m_upload_threshold; std::uint_fast64_t downloadable_bytes = 0; std::size_t num_changesets; std::size_t accum_original_size; diff --git a/test/object-store/realm.cpp b/test/object-store/realm.cpp index d6923afb172..5d4c6d9c0f7 100644 --- a/test/object-store/realm.cpp +++ b/test/object-store/realm.cpp @@ -566,7 +566,13 @@ TEST_CASE("Get Realm using Async Open", "[asyncOpen]") { std::lock_guard lock(mutex); return bool(realm_ref); }); + // Write some data SharedRealm realm = Realm::get_shared_realm(std::move(realm_ref)); + realm->begin_transaction(); + realm->read_group().get_table("class_object")->create_object_with_primary_key(2); + realm->commit_transaction(); + wait_for_upload(*realm); + wait_for_download(*realm); client_file_id = realm->read_group().get_sync_file_id(); realm->write_copy(config3.path, BinaryData()); } @@ -582,7 +588,7 @@ TEST_CASE("Get Realm using Async Open", "[asyncOpen]") { wait_for_download(*realm); // Make sure we have got a new client file id REQUIRE(realm->read_group().get_sync_file_id() != client_file_id); - REQUIRE(realm->read_group().get_table("class_object")->size() == 2); + REQUIRE(realm->read_group().get_table("class_object")->size() == 3); // Check that we can continue committing to this realm realm->begin_transaction(); @@ -593,7 +599,7 @@ TEST_CASE("Get Realm using Async Open", "[asyncOpen]") { // Check that this change is now in the original realm wait_for_download(*origin); origin->refresh(); - REQUIRE(origin->read_group().get_table("class_object")->size() == 3); + REQUIRE(origin->read_group().get_table("class_object")->size() == 4); } SECTION("downloads Realms which exist on the server") { From adb915589a60e0c16618e2742810acd1f5d319c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Fri, 10 Sep 2021 14:22:48 +0200 Subject: [PATCH 4/5] Update after review --- test/object-store/sync/app.cpp | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/test/object-store/sync/app.cpp b/test/object-store/sync/app.cpp index 602e39e51e2..af87fce60c2 100644 --- a/test/object-store/sync/app.cpp +++ b/test/object-store/sync/app.cpp @@ -1778,24 +1778,7 @@ TEST_CASE("app: set new embedded object", "[sync][app]") { } TEST_CASE("app: make distributable client file", "[sync][app]") { - std::unique_ptr (*factory)() = [] { - return std::unique_ptr(new IntTestTransport); - }; - std::string base_url = get_base_url(); - REQUIRE(!base_url.empty()); - auto app_session = get_runtime_app_session(base_url); - - auto app_id = app_session.client_app_id; - auto config = App::Config{app_id, - factory, - base_url, - util::none, - Optional("A Local App Version"), - util::none, - "Object Store Platform Tests", - "Object Store Platform Version Blah", - "An sdk version"}; - + auto config = get_integration_config(); auto base_path = util::make_temp_dir(); util::try_remove_dir_recursive(base_path); util::try_make_dir(base_path); @@ -1816,10 +1799,6 @@ TEST_CASE("app: make distributable client file", "[sync][app]") { realm::Realm::Config realm_config; realm_config.sync_config = std::make_shared(app->current_user(), bson::Bson("foo")); - realm_config.sync_config->client_resync_mode = ClientResyncMode::Manual; - realm_config.sync_config->error_handler = [](std::shared_ptr, SyncError error) { - std::cerr << error.message << std::endl; - }; realm_config.schema_version = 1; realm_config.path = base_path + "/orig/default.realm"; @@ -1874,10 +1853,6 @@ TEST_CASE("app: make distributable client file", "[sync][app]") { realm::Realm::Config realm_config; realm_config.sync_config = std::make_shared(app->current_user(), bson::Bson("foo")); - realm_config.sync_config->client_resync_mode = ClientResyncMode::Manual; - realm_config.sync_config->error_handler = [](std::shared_ptr, SyncError error) { - std::cerr << error.message << std::endl; - }; realm_config.schema_version = 1; realm_config.path = base_path + "/copy/default.realm"; From 2c942d15a3a3a08cb5854a44233546dac11a83d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 13 Sep 2021 16:31:40 +0200 Subject: [PATCH 5/5] Another update after review --- tools/{run_baas.sh => run_baas_docker_image.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/{run_baas.sh => run_baas_docker_image.sh} (100%) diff --git a/tools/run_baas.sh b/tools/run_baas_docker_image.sh similarity index 100% rename from tools/run_baas.sh rename to tools/run_baas_docker_image.sh