Automatic handling backlinks migration

CHANGELOG.md

@@ -5,6 +5,7 @@
 * Notify when read transaction version is advanced. ([PR #5704](https://github.com/realm/realm-core/pull/5704)).
 * Action returned from the server in the json error messages is surfaced through the SyncError. ([PR #5690](https://github.com/realm/realm-core/pull/5690)).
 * `NotificationToken` grew an `unregister()` method as an alternative to destroying it or doing `token = {};` ([PR #5776](https://github.com/realm/realm-core/pull/5776)).
+* Automatic handling backlinks for schema migrations where a class changes to being embedded. ([PR #5737](https://github.com/realm/realm-core/pull/5737)).
 * Expose `Obj::add_int()` in the CAPI. ([PR #5770](https://github.com/realm/realm-core/pull/5770)).
 
 ### Fixed

Package.swift

@@ -80,6 +80,7 @@ let notSyncServerSources: [String] = [
     "realm/node.cpp",
     "realm/obj.cpp",
     "realm/obj_list.cpp",
+    "realm/object_converter.cpp",
     "realm/object_id.cpp",
     "realm/query.cpp",
     "realm/query_engine.cpp",

src/realm.h

@@ -822,6 +822,11 @@ RLM_API void realm_config_set_cached(realm_config_t*, bool cached) RLM_API_NOEXC
  */
 RLM_API bool realm_config_get_cached(realm_config_t*) RLM_API_NOEXCEPT;
 
+/**
+ * Allow realm to manage automatically embedded objects when a migration from TopLevel to Embedded takes place.
+ */
+RLM_API void realm_config_set_automatic_backlink_handling(realm_config_t*, bool) RLM_API_NOEXCEPT;
+
 /**
  * Create a custom scheduler object from callback functions.
  *

src/realm/obj.cpp

@@ -17,7 +17,6 @@
  **************************************************************************/
 
 #include "realm/obj.hpp"
-#include "realm/cluster_tree.hpp"
 #include "realm/array_basic.hpp"
 #include "realm/array_integer.hpp"
 #include "realm/array_bool.hpp"
@@ -30,14 +29,15 @@
 #include "realm/array_fixed_bytes.hpp"
 #include "realm/array_backlink.hpp"
 #include "realm/array_typed_link.hpp"
+#include "realm/cluster_tree.hpp"
 #include "realm/column_type_traits.hpp"
+#include "realm/dictionary.hpp"
 #include "realm/index_string.hpp"
-#include "realm/cluster_tree.hpp"
-#include "realm/spec.hpp"
+#include "realm/object_converter.hpp"
+#include "realm/replication.hpp"
 #include "realm/set.hpp"
-#include "realm/dictionary.hpp"
+#include "realm/spec.hpp"
 #include "realm/table_view.hpp"
-#include "realm/replication.hpp"
 #include "realm/util/base64.hpp"
 
 namespace realm {
@@ -1934,70 +1934,68 @@ bool Obj::remove_backlink(ColKey col_key, ObjLink old_link, CascadeState& state)
     return false;
 }
 
-void Obj::assign(const Obj& other)
+void Obj::handle_multiple_backlinks_during_schema_migration()
 {
-    REALM_ASSERT(get_table() == other.get_table());
-    auto cols = m_table->get_column_keys();
-    for (auto col : cols) {
-        if (col.get_attrs().test(col_attr_List)) {
-            auto src_list = other.get_listbase_ptr(col);
-            auto dst_list = get_listbase_ptr(col);
-            auto sz = src_list->size();
-            dst_list->clear();
-            for (size_t i = 0; i < sz; i++) {
-                Mixed val = src_list->get_any(i);
-                dst_list->insert_any(i, val);
-            }
-        }
-        else {
-            Mixed val = other.get_any(col);
-            if (val.is_null()) {
-                this->set_null(col);
-                continue;
-            }
-            switch (val.get_type()) {
-                case type_String: {
-                    // Need to take copy. Values might be in same cluster
-                    std::string str{val.get_string()};
-                    this->set(col, str);
-                    break;
-                }
-                case type_Binary: {
-                    // Need to take copy. Values might be in same cluster
-                    std::string str{val.get_binary()};
-                    this->set(col, BinaryData(str));
-                    break;
-                }
-                default:
-                    this->set_any(col, val);
-                    break;
-            }
-        }
-    }
-
-    auto copy_links = [this, &other](ColKey col) {
-        auto t = m_table->get_opposite_table(col);
-        auto c = m_table->get_opposite_column(col);
-        auto backlinks = other.get_all_backlinks(col);
-        for (auto bl : backlinks) {
-            auto linking_obj = t->get_object(bl);
-            if (c.get_type() == col_type_Link) {
-                // Single link
-                REALM_ASSERT(!linking_obj.get<ObjKey>(c) || linking_obj.get<ObjKey>(c) == other.get_key());
-                linking_obj.set(c, get_key());
-            }
-            else {
-                auto l = linking_obj.get_linklist(c);
-                auto n = l.find_first(other.get_key());
-                REALM_ASSERT(n != realm::npos);
-                l.set(n, get_key());
-            }
+    REALM_ASSERT(!m_table->get_primary_key_column());
+    auto copy_links = [this](ColKey col) {
+        auto embedded_obj_tracker = std::make_shared<converters::EmbeddedObjectConverter>();
+        auto opposite_table = m_table->get_opposite_table(col);
+        auto opposite_column = m_table->get_opposite_column(col);
+        auto backlinks = get_all_backlinks(col);
+        for (auto backlink : backlinks) {
+            // create a new obj
+            auto obj = m_table->create_object();
+            embedded_obj_tracker->track(*this, obj);
+            auto linking_obj = opposite_table->get_object(backlink);
+            fix_linking_object_during_schema_migration(linking_obj, obj, opposite_column);
         }
+        embedded_obj_tracker->process_pending();
         return false;
     };
     m_table->for_each_backlink_column(copy_links);
 }
 
+void Obj::fix_linking_object_during_schema_migration(Obj linking_obj, Obj obj, ColKey opposite_col_key) const
+{
+    // fix linking obj links in order to point to the new object just created
+    if (opposite_col_key.get_type() == col_type_LinkList) {
+        auto linking_obj_list = linking_obj.get_linklist(opposite_col_key);
+        auto n = linking_obj_list.find_first(get_key());
+        REALM_ASSERT(n != realm::npos);
+        linking_obj_list.set(n, obj.get_key());
+    }
+    else if (opposite_col_key.get_attrs().test(col_attr_List)) {
+        auto linking_obj_list = linking_obj.get_listbase_ptr(opposite_col_key);
+        auto n = linking_obj_list->find_any(ObjLink{m_table->get_key(), get_key()});
+        REALM_ASSERT(n != realm::npos);
+        linking_obj_list->insert_any(n, ObjLink{m_table->get_key(), obj.get_key()});
+    }
+    else if (opposite_col_key.get_attrs().test(col_attr_Set)) {
+        // this SHOULD NEVER HAPPEN, since SET sematincs forbids to have a backlink to the same object store
+        // multiple times. update_schema would have thrown way before to reach this point.
+        REALM_UNREACHABLE();
+    }
+    else if (opposite_col_key.get_attrs().test(col_attr_Dictionary)) {
+        auto linking_obj_dictionary = linking_obj.get_dictionary_ptr(opposite_col_key);
+        auto pos = linking_obj_dictionary->find_any(ObjLink{m_table->get_key(), get_key()});
+        REALM_ASSERT(pos != realm::npos);
+        Mixed key = linking_obj_dictionary->get_key(pos);
+        linking_obj_dictionary->insert(key, ObjLink{m_table->get_key(), obj.get_key()});
+    }
+    else if (opposite_col_key.get_type() == col_type_Mixed &&
+             linking_obj.get_any(opposite_col_key).get_type() == type_TypedLink) {
+        REALM_ASSERT(!linking_obj.get<ObjKey>(opposite_col_key) ||
+                     linking_obj.get<ObjKey>(opposite_col_key) == get_key());
+        linking_obj.set_any(opposite_col_key, ObjLink{m_table->get_key(), obj.get_key()});
+    }
+    else if (opposite_col_key.get_type() == col_type_Link) {
+        // Single link
+        REALM_ASSERT(!linking_obj.get<ObjKey>(opposite_col_key) ||
+                     linking_obj.get<ObjKey>(opposite_col_key) == get_key());
+        linking_obj.set(opposite_col_key, obj.get_key());
+    }
+}
+
 Dictionary Obj::get_dictionary(ColKey col_key) const
 {
     REALM_ASSERT(col_key.is_dictionary());

src/realm/obj.hpp

@@ -260,7 +260,13 @@ class Obj {
     template <class Head, class... Tail>
     Obj& set_all(Head v, Tail... tail);
 
-    void assign(const Obj& other);
+    // The main algorithm for handling schema migrations if we try to convert
+    // from TopLevel* to Embedded, in this case all the orphan objects are deleted
+    // and all the objects with multiple backlinks are cloned in order to avoid to
+    // get schema violations during the migration.
+    // By default this alogirithm is disabled. RealmConfig contains a boolean flag
+    // to enable it.
+    void handle_multiple_backlinks_during_schema_migration();
 
     Obj get_linked_object(ColKey link_col_key) const
     {
@@ -420,6 +426,8 @@ class Obj {
     bool remove_backlink(ColKey col_key, ObjLink old_link, CascadeState& state) const;
     template <class T>
     inline void set_spec(T&, ColKey);
+
+    void fix_linking_object_during_schema_migration(Obj linking_obj, Obj obj, ColKey opposite_col_key) const;
 };
 
 std::ostream& operator<<(std::ostream&, const Obj& obj);

src/realm/object-store/c_api/config.cpp

@@ -222,3 +222,9 @@ RLM_API bool realm_config_get_cached(realm_config_t* realm_config) noexcept
 {
     return realm_config->cache;
 }
+
+RLM_API void realm_config_set_automatic_backlink_handling(realm_config_t* realm_config,
+                                                          bool enable_automatic_handling) noexcept
+{
+    realm_config->automatic_handle_backlicks_in_migrations = enable_automatic_handling;
+}

src/realm/object-store/object_store.cpp

@@ -762,23 +762,28 @@ static void apply_pre_migration_changes(Group& group, std::vector<SchemaChange>
 }
 
 enum class DidRereadSchema { Yes, No };
+enum class HandleBackLinksAutomatically { Yes, No };
 
 static void apply_post_migration_changes(Group& group, std::vector<SchemaChange> const& changes,
-                                         Schema const& initial_schema, DidRereadSchema did_reread_schema)
+                                         Schema const& initial_schema, DidRereadSchema did_reread_schema,
+                                         HandleBackLinksAutomatically handle_backlinks_automatically)
 {
     using namespace schema_change;
     struct Applier {
-        Applier(Group& group, Schema const& initial_schema, DidRereadSchema did_reread_schema)
+        Applier(Group& group, Schema const& initial_schema, DidRereadSchema did_reread_schema,
+                HandleBackLinksAutomatically handle_backlinks_automatically)
             : group{group}
             , initial_schema(initial_schema)
             , table(group)
             , did_reread_schema(did_reread_schema == DidRereadSchema::Yes)
+            , handle_backlinks_automatically(handle_backlinks_automatically == HandleBackLinksAutomatically::Yes)
         {
         }
         Group& group;
         Schema const& initial_schema;
         TableHelper table;
         bool did_reread_schema;
+        bool handle_backlinks_automatically;
 
         void operator()(RemoveProperty op)
         {
@@ -821,14 +826,52 @@ static void apply_post_migration_changes(Group& group, std::vector<SchemaChange>
 
         void operator()(ChangeTableType op)
         {
+            if (handle_backlinks_automatically)
+                post_migration_embedded_objects_backlinks_handling(op.object);
             table(op.object).set_table_type(static_cast<Table::Type>(*op.new_table_type));
         }
         void operator()(RemoveTable) {}
         void operator()(ChangePropertyType) {}
         void operator()(MakePropertyNullable) {}
         void operator()(MakePropertyRequired) {}
         void operator()(AddProperty) {}
-    } applier{group, initial_schema, did_reread_schema};
+
+        void post_migration_embedded_objects_backlinks_handling(const ObjectSchema* object_schema)
+        {
+            // check if we are doing a migration from TopLevel Object to Embedded.
+            // check back link count.
+            //  1. if object is an orphan (no backlicks, then delete it if instructed to do so)
+            //  2. if object has multiple backlicks, then just clone N times the object (for each backlick) and assign
+            //  to each a different parent
+            // object_schema->handle_automatically_backlinks_for_embedded_object = true;
+
+            if (object_schema->table_type == ObjectSchema::ObjectType::Embedded) {
+                auto original_object_schema = initial_schema.find(object_schema->name);
+                if (original_object_schema != initial_schema.end() &&
+                    (original_object_schema->table_type == ObjectSchema::ObjectType::TopLevel ||
+                     original_object_schema->table_type == ObjectSchema::ObjectType::TopLevelAsymmetric)) {
+                    auto table = table_for_object_schema(group, *original_object_schema);
+                    std::vector<Obj> objects_to_erase;
+                    std::vector<Obj> objects_to_fix_backlinks;
+                    for (auto& object : *table) {
+                        size_t backlink_count = object.get_backlink_count();
+                        if (backlink_count == 0)
+                            objects_to_erase.push_back(object);
+                        else if (backlink_count > 1)
+                            objects_to_fix_backlinks.push_back(object);
+                    }
+
+                    for (auto& object : objects_to_erase)
+                        object.remove();
+
+                    for (auto& object : objects_to_fix_backlinks) {
+                        object.handle_multiple_backlinks_during_schema_migration();
+                        object.remove();
+                    }
+                }
+            }
+        }
+    } applier{group, initial_schema, did_reread_schema, handle_backlinks_automatically};
 
     for (auto& change : changes) {
         change.visit(applier);
@@ -838,7 +881,7 @@ static void apply_post_migration_changes(Group& group, std::vector<SchemaChange>
 
 void ObjectStore::apply_schema_changes(Transaction& group, uint64_t schema_version, Schema& target_schema,
                                        uint64_t target_schema_version, SchemaMode mode,
-                                       std::vector<SchemaChange> const& changes,
+                                       std::vector<SchemaChange> const& changes, bool handle_automatically_backlinks,
                                        std::function<void()> migration_function)
 {
     create_metadata_tables(group);
@@ -888,17 +931,20 @@ void ObjectStore::apply_schema_changes(Transaction& group, uint64_t schema_versi
 
     auto old_schema = schema_from_group(group);
     apply_pre_migration_changes(group, changes);
+    HandleBackLinksAutomatically handle_backlinks =
+        handle_automatically_backlinks ? HandleBackLinksAutomatically::Yes : HandleBackLinksAutomatically::No;
     if (migration_function) {
         set_schema_keys(group, target_schema);
         migration_function();
 
         // Migration function may have changed the schema, so we need to re-read it
         auto schema = schema_from_group(group);
-        apply_post_migration_changes(group, schema.compare(target_schema, mode), old_schema, DidRereadSchema::Yes);
+        apply_post_migration_changes(group, schema.compare(target_schema, mode), old_schema, DidRereadSchema::Yes,
+                                     handle_backlinks);
         group.validate_primary_columns();
     }
     else {
-        apply_post_migration_changes(group, changes, {}, DidRereadSchema::No);
+        apply_post_migration_changes(group, changes, {}, DidRereadSchema::No, handle_backlinks);
     }
 
     set_schema_version(group, target_schema_version);

src/realm/object-store/object_store.hpp

@@ -84,7 +84,7 @@ class ObjectStore {
     // NOTE: must be performed within a write transaction
     static void apply_schema_changes(Transaction& group, uint64_t schema_version, Schema& target_schema,
                                      uint64_t target_schema_version, SchemaMode mode,
-                                     std::vector<SchemaChange> const& changes,
+                                     std::vector<SchemaChange> const& changes, bool handle_automatically_backlinks,
                                      std::function<void()> migration_function = {});
 
     static void apply_additive_changes(Group&, std::vector<SchemaChange> const&, bool update_indexes);

src/realm/object-store/shared_realm.cpp

@@ -460,11 +460,12 @@ void Realm::update_schema(Schema schema, uint64_t version, MigrationFunction mig
         });
 
         ObjectStore::apply_schema_changes(transaction(), version, m_schema, m_schema_version, m_config.schema_mode,
-                                          required_changes, wrapper);
+                                          required_changes, m_config.automatic_handle_backlicks_in_migrations,
+                                          wrapper);
     }
     else {
         ObjectStore::apply_schema_changes(transaction(), m_schema_version, schema, version, m_config.schema_mode,
-                                          required_changes);
+                                          required_changes, m_config.automatic_handle_backlicks_in_migrations);
         REALM_ASSERT_DEBUG(additive ||
                            (required_changes = ObjectStore::schema_from_group(read_group()).compare(schema)).empty());
     }

src/realm/object-store/shared_realm.hpp

@@ -175,6 +175,9 @@ struct RealmConfig {
 
     // Disable automatic backup at file format upgrade by setting to false
     bool backup_at_file_format_change = true;
+
+    // delete embedded orphan objects
+    bool automatic_handle_backlicks_in_migrations = false;
 };
 
 class Realm : public std::enable_shared_from_this<Realm> {