Support query on nested collections

This includes supporting using index in query on list of primitives

CHANGELOG.md

@@ -3,6 +3,8 @@
 ### Enhancements
 * <New feature description> (PR [#????](https://github.com/realm/realm-core/pull/????))
 * Storage of Decimal128 properties has been optimised so that the individual values will take up 0 bits (if all nulls), 32 bits, 64 bits or 128 bits depending on what is needed. (PR [#6111]https://github.com/realm/realm-core/pull/6111))
+* You can have a collection embedded in any Mixed property (except Set<Mixed>).
+* Querying a specific entry in a list-of-primitives (in particular 'first and 'last') is supported. (PR [#4269](https://github.com/realm/realm-core/issues/4269))
 
 ### Fixed
 * <How do the end-user experience this issue? what was the impact?> ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?)

src/realm/collection.cpp

@@ -1,6 +1,8 @@
 #include <realm/collection.hpp>
 #include <realm/bplustree.hpp>
 #include <realm/array_key.hpp>
+#include <realm/array_string.hpp>
+#include <realm/array_mixed.hpp>
 
 namespace realm {
 
@@ -91,6 +93,48 @@ void check_for_last_unresolved(BPlusTree<ObjKey>* tree)
 
 Collection::~Collection() {}
 
+Mixed Collection::get_any(Mixed val, Path::const_iterator begin, Path::const_iterator end, Allocator& alloc)
+{
+    auto path_size = end - begin;
+    if (val.is_type(type_Dictionary) && begin->is_key()) {
+        Array top(alloc);
+        top.init_from_ref(val.get_ref());
+
+        BPlusTree<StringData> keys(alloc);
+        keys.set_parent(&top, 0);
+        keys.init_from_parent();
+        size_t ndx = keys.find_first(StringData(begin->get_key()));
+        if (ndx != realm::not_found) {
+            BPlusTree<Mixed> values(alloc);
+            values.set_parent(&top, 1);
+            values.init_from_parent();
+            val = values.get(ndx);
+            if (path_size > 1) {
+                val = Collection::get_any(val, begin + 1, end, alloc);
+            }
+            return val;
+        }
+    }
+    if (val.is_type(type_List) && begin->is_ndx()) {
+        ArrayMixed list(alloc);
+        list.init_from_ref(val.get_ref());
+        if (size_t sz = list.size()) {
+            auto idx = begin->get_ndx();
+            if (idx == size_t(-1)) {
+                idx = sz - 1;
+            }
+            if (idx < sz) {
+                val = list.get(idx);
+            }
+            if (path_size > 1) {
+                val = Collection::get_any(val, begin + 1, end, alloc);
+            }
+            return val;
+        }
+    }
+    return {};
+}
+
 std::pair<std::string, std::string> CollectionBase::get_open_close_strings(size_t link_depth,
                                                                            JSONOutputMode output_mode) const
 {

src/realm/collection.hpp

@@ -94,6 +94,8 @@ class Collection {
     virtual Path get_short_path() const = 0;
     // Return a path based on keys instead of indices
     virtual StablePath get_stable_path() const = 0;
+
+    static Mixed get_any(Mixed, Path::const_iterator, Path::const_iterator, Allocator&);
 };
 
 using CollectionPtr = std::shared_ptr<Collection>;
@@ -585,14 +587,9 @@ class CollectionBaseImpl : public Interface, protected ArrayParent {
         return *this;
     }
 
-    ref_type get_collection_ref() const noexcept
+    ref_type get_collection_ref() const
     {
-        try {
-            return m_parent->get_collection_ref(m_index, Interface::s_collection_type);
-        }
-        catch (...) {
-            return ref_type(0);
-        }
+        return m_parent->get_collection_ref(m_index, Interface::s_collection_type);
     }
 
     void set_collection_ref(ref_type ref)

src/realm/collection_parent.cpp

@@ -33,7 +33,16 @@ namespace realm {
 std::ostream& operator<<(std::ostream& ostr, const PathElement& elem)
 {
     if (elem.is_ndx()) {
-        ostr << elem.get_ndx();
+        size_t ndx = elem.get_ndx();
+        if (ndx == 0) {
+            ostr << "FIRST";
+        }
+        else if (ndx == size_t(-1)) {
+            ostr << "LAST";
+        }
+        else {
+            ostr << elem.get_ndx();
+        }
     }
     else if (elem.is_col_key()) {
         ostr << elem.get_col_key();
@@ -45,10 +54,24 @@ std::ostream& operator<<(std::ostream& ostr, const PathElement& elem)
     return ostr;
 }
 
+std::ostream& operator<<(std::ostream& ostr, const Path& path)
+{
+    for (auto& elem : path) {
+        ostr << '[' << elem << ']';
+    }
+    return ostr;
+}
+
 /***************************** CollectionParent ******************************/
 
 CollectionParent::~CollectionParent() {}
 
+void CollectionParent::check_level() const
+{
+    if (m_level + 1 > s_max_level) {
+        throw LogicError(ErrorCodes::LimitExceeded, "Max nesting level reached");
+    }
+}
 void CollectionParent::set_backlink(ColKey col_key, ObjLink new_link) const
 {
     if (new_link && new_link.get_obj_key()) {

src/realm/collection_parent.hpp

@@ -199,10 +199,11 @@ struct PathElement {
     }
 };
 
-std::ostream& operator<<(std::ostream& ostr, const PathElement& elem);
-
 using Path = std::vector<PathElement>;
 
+std::ostream& operator<<(std::ostream& ostr, const PathElement& elem);
+std::ostream& operator<<(std::ostream& ostr, const Path& path);
+
 // Path from the group level.
 struct FullPath {
     TableKey top_table;
@@ -222,6 +223,7 @@ class CollectionParent : public std::enable_shared_from_this<CollectionParent> {
     {
         return m_level;
     }
+    void check_level() const;
     // Return the path to this object. The path is calculated from
     // the topmost Obj - which must be an Obj with a primary key.
     virtual FullPath get_path() const = 0;
@@ -240,6 +242,11 @@ class CollectionParent : public std::enable_shared_from_this<CollectionParent> {
     friend class CollectionBaseImpl;
     friend class CollectionList;
 
+#ifdef REALM_DEBUG
+    static constexpr size_t s_max_level = 4;
+#else
+    static constexpr size_t s_max_level = 100;
+#endif
     size_t m_level = 0;
 
     constexpr CollectionParent(size_t level = 0)

src/realm/dictionary.cpp

@@ -428,6 +428,7 @@ Obj Dictionary::create_and_insert_linked_object(Mixed key)
 
 void Dictionary::insert_collection(const PathElement& path_elem, CollectionType dict_or_list)
 {
+    check_level();
     insert(path_elem.get_key(), Mixed(0, dict_or_list));
 }
 

src/realm/list.cpp

@@ -418,6 +418,7 @@ void Lst<Mixed>::swap(size_t ndx1, size_t ndx2)
 void Lst<Mixed>::insert_collection(const PathElement& path_elem, CollectionType dict_or_list)
 {
     ensure_created();
+    check_level();
     m_tree->ensure_keys();
     insert(path_elem.get_ndx(), Mixed(0, dict_or_list));
     int64_t key = generate_key(size());
@@ -435,6 +436,8 @@ void Lst<Mixed>::set_collection(const PathElement& path_elem, CollectionType typ
     Mixed old_val = do_get(ndx, "set()");
     Mixed new_val(0, type);
 
+    check_level();
+
     if (old_val != new_val) {
         m_tree->ensure_keys();
         set(ndx, Mixed(0, type));

src/realm/parser/driver.cpp

@@ -537,7 +537,7 @@ Query EqualityNode::visit(ParserDriver* drv)
     else if (right->has_single_value() && (left_type == right_type || left_type == type_Mixed)) {
         Mixed val = right->get_mixed();
         const ObjPropertyBase* prop = dynamic_cast<const ObjPropertyBase*>(left.get());
-        if (prop && !prop->links_exist()) {
+        if (prop && !prop->links_exist() && !prop->has_path()) {
             auto col_key = prop->column_key();
             if (val.is_null()) {
                 switch (op) {
@@ -848,17 +848,29 @@ std::unique_ptr<Subexpr> PropertyNode::visit(ParserDriver* drv, DataType)
     }
     std::unique_ptr<Subexpr> subexpr{drv->column(m_link_chain, path)};
 
-    if (!path->at_end()) {
-        if (path->is_identifier()) {
-            auto trailing = path->next_identifier();
+    Path indexes;
+    while (!path->at_end()) {
+        indexes.emplace_back(std::move(*(path->current_path_elem++)));
+    }
+
+    if (!indexes.empty()) {
+        const PathElement& first_index = indexes.front();
+        if (indexes.size() > 1 && subexpr->get_type() != type_Mixed) {
+            throw InvalidQueryError("Only Property of type 'any' can have nested collections");
+        }
+        if (auto mixed = dynamic_cast<Columns<Mixed>*>(subexpr.get())) {
+            mixed->path(indexes);
+        }
+        else if (first_index.is_key()) {
+            auto trailing = first_index.get_key();
             if (auto dict = dynamic_cast<Columns<Dictionary>*>(subexpr.get())) {
                 if (trailing == "@values") {
                 }
                 else if (trailing == "@keys") {
                     subexpr = std::make_unique<ColumnDictionaryKeys>(*dict);
                 }
                 else {
-                    dict->key(trailing);
+                    dict->key(indexes);
                 }
             }
             else {
@@ -879,7 +891,24 @@ std::unique_ptr<Subexpr> PropertyNode::visit(ParserDriver* drv, DataType)
             }
         }
         else {
-            throw InvalidQueryError("Index not supported");
+            // This must be an integer index
+            REALM_ASSERT(first_index.is_ndx());
+            auto ok = false;
+            if (indexes.size() == 1) {
+                if (auto coll = dynamic_cast<ColumnListBase*>(subexpr.get())) {
+                    ok = coll->index(first_index);
+                }
+            }
+            else {
+                if (auto coll = dynamic_cast<Columns<Lst<Mixed>>*>(subexpr.get())) {
+                    ok = coll->indexes(indexes);
+                }
+            }
+            if (!ok) {
+                throw InvalidQueryError(util::format("Property '%1.%2' does not support index '%3'",
+                                                     m_link_chain.get_current_table()->get_class_name(), identifier,
+                                                     first_index));
+            }
         }
     }
     if (post_op) {

src/realm/parser/driver.hpp

@@ -284,18 +284,10 @@ class PathNode : public ParserNode {
     {
         return current_path_elem == path_elems.end();
     }
-    bool is_identifier() const
-    {
-        return current_path_elem->is_key();
-    }
     const std::string& next_identifier()
     {
         return (current_path_elem++)->get_key();
     }
-    size_t next_index()
-    {
-        return (current_path_elem++)->get_ndx();
-    }
     const std::string& last_identifier()
     {
         return path_elems.back().get_key();