CBL-6651: createIndex doesn't support creating partial index with N1Q… (

#2214)

* CBL-6651: createIndex doesn't support creating partial index with N1QL language

Extended IndexSpec to allow the where clause when the query langugage is N1QL. We put it in IndexSpec::options. In this commit, we only do it for ValueIndex and FTSIndex.

With JSON query language, the where clause can be either in the new option or in IndexSpec::expression together with the what clause. With N1QL query language, the where clause has to be in the field of IndexSpec, and IndexSpec::expression contains only the what clause.

This commit is to get ready for C4IndexOptions

LiteCore/Query/IndexSpec.cc

@@ -53,7 +53,14 @@ namespace litecore {
             switch ( queryLanguage ) {
                 case QueryLanguage::kJSON:
                     try {
-                        _doc = Doc::fromJSON(expression);
+                        if ( canPartialIndex() && !whereClause.empty() ) {
+                            std::stringstream ss;
+                            ss << R"({"WHAT": )" << expression.asString() << R"(, "WHERE": )" << whereClause.asString()
+                               << "}";
+                            _doc = Doc::fromJSON(ss.str());
+                        } else {
+                            _doc = Doc::fromJSON(expression);
+                        }
                     } catch ( const FleeceException& ) {
                         error::_throw(error::InvalidQuery, "Invalid JSON in index expression");
                     }
@@ -63,17 +70,37 @@ namespace litecore {
                         int         errPos;
                         alloc_slice json;
                         if ( !expression.empty() ) {
-                            FLMutableDict result = n1ql::parse(string(expression), &errPos);
-                            if ( !result ) { throw Query::parseError("N1QL syntax error in index expression", errPos); }
-                            json = ((MutableDict*)result)->toJSON(true);
-                            FLMutableDict_Release(result);
+                            MutableDict*      result   = nullptr;
+                            bool              hasWhere = false;
+                            std::stringstream ss;
+                            if ( canPartialIndex() && !whereClause.empty() ) {
+                                hasWhere = true;
+                                ss << "SELECT ( " << expression.asString() << " ) FROM _ WHERE ( "
+                                   << whereClause.asString() << " )";
+                                result = (MutableDict*)n1ql::parse(ss.str(), &errPos);
+                            } else {
+                                result = (MutableDict*)n1ql::parse(expression.asString(), &errPos);
+                            }
+                            if ( !result ) {
+                                string errExpr = "Invalid N1QL in index expression \"";
+                                if ( ss.peek() != EOF ) errExpr += ss.str();
+                                else
+                                    errExpr += expression.asString();
+                                errExpr += "\"";
+                                throw Query::parseError(errExpr.c_str(), errPos);
+                            }
+                            if ( hasWhere ) result->remove("FROM"_sl);
+                            json = result->toJSON(true);
+                            FLMutableDict_Release((FLMutableDict)result);
                         } else {
                             // n1ql parser won't compile empty string to empty array. Do it manually.
-                            json = "[]";
+                            json = "[]";  // empty WHAT cannot be followed by WHERE clause.
                         }
                         _doc = Doc::fromJSON(json);
-                    } catch ( const std::runtime_error& ) {
-                        error::_throw(error::InvalidQuery, "Invalid N1QL in index expression");
+                    } catch ( const std::runtime_error& exc ) {
+                        if ( dynamic_cast<const Query::parseError*>(&exc) ) throw;
+                        else
+                            error::_throw(error::InvalidQuery, "Invalid N1QL in index expression");
                     }
                     break;
             }

LiteCore/Query/IndexSpec.hh

@@ -39,6 +39,10 @@ namespace litecore {
             kVector,      ///< Index of ML vector similarity. Uses IndexSpec::VectorOptions.
         };
 
+        static bool canPartialIndex(Type type_) { return type_ == kValue || type_ == kFullText; }
+
+        bool canPartialIndex() const { return canPartialIndex(type); }
+
         /// Options for a full-text index.
         struct FTSOptions {
             const char* language{};          ///< NULL or an ISO language code ("en", etc)
@@ -71,6 +75,19 @@ namespace litecore {
         IndexSpec(std::string name_, Type type_, alloc_slice expression_,
                   QueryLanguage queryLanguage = QueryLanguage::kJSON, Options options_ = {});
 
+        /// Constructs an index spec.
+        /// @param name_  Name of the index (must be unique in its collection.)
+        /// @param type_  Type of the index.
+        /// @param expression_  The value(s) to be indexed.
+        /// @param whereClause_ The where clause for the partial index
+        /// @param queryLanguage  Language used for `expression_`; either JSON or N1QL.
+        /// @param options_  Options; if given, its type must match the index type.
+        IndexSpec(std::string name_, Type type_, string_view expression_, string_view whereClause_ = {},
+                  QueryLanguage queryLanguage = QueryLanguage::kJSON, Options options_ = {})
+            : IndexSpec(name_, type_, alloc_slice(expression_), queryLanguage, options_) {
+            const_cast<alloc_slice&>(this->whereClause) = whereClause_;
+        }
+
         IndexSpec(const IndexSpec&) = delete;
         IndexSpec(IndexSpec&&);
 
@@ -101,6 +118,7 @@ namespace litecore {
         std::string const name;           ///< Name of index
         Type const        type;           ///< Type of index
         alloc_slice const expression;     ///< The query expression
+        alloc_slice const whereClause;    ///< The where clause. If given, expression should be the what clause
         QueryLanguage     queryLanguage;  ///< Is expression JSON or N1QL?
         Options const     options;        ///< Options for FTS and vector indexes
 

LiteCore/tests/FTSTest.cc

@@ -170,8 +170,26 @@ TEST_CASE_METHOD(FTSTest, "Query Full-Text Stop-words In Target", "[Query][FTS]"
 TEST_CASE_METHOD(FTSTest, "Query Full-Text Partial Index", "[Query][FTS]") {
     // the WHERE clause prevents row 4 from being indexed/searched.
     IndexSpec::FTSOptions options{"english", true};
-    store->createIndex("sentence", R"-({"WHAT": [[".sentence"]], "WHERE": [">", ["length()", [".sentence"]], 70]})-",
-                       IndexSpec::kFullText, options);
+
+    SECTION("JSON Index Spec with combinged \"what\" and \"where\"") {
+        REQUIRE(store->createIndex({"sentence",
+                                    IndexSpec::kFullText,
+                                    R"-({"WHAT": [[".sentence"]], "WHERE": [">", ["length()", [".sentence"]], 70]})-",
+                                    {},
+                                    QueryLanguage::kJSON,
+                                    options}));
+    }
+
+    SECTION("JSON Index Spec with separate \"what\" and \"where\"") {
+        REQUIRE(store->createIndex({"sentence", IndexSpec::kFullText, R"-([[".sentence"]])-",
+                                    R"-([">", ["length()", [".sentence"]], 70])-", QueryLanguage::kJSON, options}));
+    }
+
+    SECTION("N1QL Index Spec") {
+        REQUIRE(store->createIndex({"sentence", IndexSpec::kFullText, "sentence", "length(sentence) > 70",
+                                    QueryLanguage::kN1QL, options}));
+    }
+
     testQuery("['SELECT', {'WHERE': ['MATCH()', 'sentence', 'search'],\
                     ORDER_BY: [['DESC', ['rank()', 'sentence']]],\
                         WHAT: [['.sentence']]}]",

LiteCore/tests/LiteCoreTest.hh

@@ -146,4 +146,11 @@ class DataFileTestFixture
     alloc_slice blobAccessor(const fleece::impl::Dict*) const override;
 
     string _databaseName{"db"};
+
+  protected:
+    static void logSection(const string& name, int level = 0) {
+        size_t      numSpaces = 8 + level * 4;
+        std::string spaces(numSpaces, ' ');
+        fprintf(stderr, "%s--- %s\n", spaces.c_str(), name.c_str());
+    }
 };

LiteCore/tests/QueryTest.cc

@@ -562,14 +562,26 @@ TEST_CASE_METHOD(QueryTest, "Create Partial Index", "[Query]") {
     addNumberedDocs(1, 100);
     addArrayDocs(101, 100);
 
-    store->createIndex("nums"_sl, R"({"WHAT":[[".num"]], "WHERE":["=",[".type"],"number"]})"_sl);
-
     auto [queryJson, expectOptimized] =
             GENERATE(pair<const char*, bool>{"['AND', ['=', ['.type'], 'number'], "
                                              "['>=', ['.', 'num'], 30], ['<=', ['.', 'num'], 40]]",
                                              true},
                      pair<const char*, bool>{"['AND', ['>=', ['.', 'num'], 30], ['<=', ['.', 'num'], 40]]", false});
-    logSection(string("Query: ") + queryJson);
+
+    SECTION("JSON Index Spec with combined \"what\" and \"where\"") {
+        REQUIRE(store->createIndex("nums"_sl, R"({"WHAT":[[".num"]], "WHERE":["=",[".type"],"number"]})"_sl));
+    }
+
+    SECTION("JSON Index Spec with separate \"what\" and \"where\"") {
+        REQUIRE(store->createIndex(
+                {"nums", IndexSpec::kValue, R"([[".num"]])", R"(["=",[".type"],"number"])", QueryLanguage::kJSON}));
+    }
+
+    SECTION("N1QL Index Spec") {
+        REQUIRE(store->createIndex({"nums", IndexSpec::kValue, "num", "type = 'number'", QueryLanguage::kN1QL}));
+    }
+
+    logSection(string("Query: ") + queryJson + (expectOptimized ? ", " : ", not ") + "optimized");
     Retained<Query> query = store->compileQuery(json5(queryJson));
     checkOptimized(query, expectOptimized);
 

LiteCore/tests/QueryTest.hh

@@ -63,12 +63,6 @@ class QueryTest : public DataFileTestFixture {
         }
     }
 
-    static void logSection(const string& name, int level = 0) {
-        size_t      numSpaces = 8 + level * 4;
-        std::string spaces(numSpaces, ' ');
-        fprintf(stderr, "%s--- %s\n", spaces.c_str(), name.c_str());
-    }
-
     static string numberString(int n) {
         static const char* kDigit[10] = {"zero", "one", "two",   "three", "four",
                                          "five", "six", "seven", "eight", "nine"};