Parse complete SPARQL queries using ANTLR

e2e/scientists_queries.yaml

@@ -1120,7 +1120,7 @@ queries:
     checks:
       - num_cols: 2
       - num_rows: 11
-      - selected: ["?x", "?t"]
+      - selected: [ "?x", "?t", "?ql_textscore_t" ]
       - contains_row: ["<Grete_Hermann>","Hermann's algorithm for primary decomposition is still in use now."]
 
 

src/parser/ConstructClause.h

@@ -15,16 +15,16 @@ struct ConstructClause : ClauseBase {
   explicit ConstructClause(ad_utility::sparql_types::Triples triples)
       : triples_(std::move(triples)) {}
 
-  vector<const Variable*> containedVariables() {
-    vector<const Variable*> out;
+  // Yields all variables that appear in this `ConstructClause`. Variables that
+  // appear multiple times are also yielded multiple times.
+  cppcoro::generator<const Variable> containedVariables() const {
     for (const auto& triple : triples_) {
       for (const auto& varOrTerm : triple) {
         if (auto variable = std::get_if<Variable>(&varOrTerm)) {
-          out.emplace_back(variable);
+          co_yield *variable;
         }
       }
     }
-    return out;
   }
 };
 }  // namespace parsedQuery

src/parser/ParsedQuery.cpp

@@ -120,26 +120,27 @@ Variable ParsedQuery::addInternalBind(
 
 // ________________________________________________________________________
 void ParsedQuery::addSolutionModifiers(SolutionModifiers modifiers) {
+  auto checkVariableIsVisible = [this](const Variable& var,
+                                       const std::string& locationDescription) {
+    if (!ad_utility::contains(getVisibleVariables(), var)) {
+      throw ParseException("Variable " + var.name() + " was used in " +
+                           locationDescription +
+                           " is not visible in the Query Body.");
+    }
+  };
   auto checkUsedVariablesAreVisible =
-      [this](const sparqlExpression::SparqlExpressionPimpl& expression,
-             const std::string& locationDescription) {
+      [&checkVariableIsVisible](
+          const sparqlExpression::SparqlExpressionPimpl& expression,
+          const std::string& locationDescription) {
         for (const auto* var : expression.containedVariables()) {
-          if (!ad_utility::contains(getVisibleVariables(), *var)) {
-            throw ParseException("Variable " + var->name() + " used in " +
-                                 locationDescription +
-                                 expression.getDescriptor() +
-                                 " is not visible in the Query Body.");
-          }
+          checkVariableIsVisible(*var, locationDescription);
         }
       };
 
   // Process groupClause
-  auto processVariable = [this](const Variable& groupKey) {
-    if (!ad_utility::contains(getVisibleVariables(), groupKey)) {
-      throw ParseException(
-          "Variable " + groupKey.name() +
-          " was used in an GROUP BY but is not visible in the query body.");
-    }
+  auto processVariable = [this,
+                          &checkVariableIsVisible](const Variable& groupKey) {
+    checkVariableIsVisible(groupKey, "GROUP BY");
 
     _groupByVariables.emplace_back(groupKey.name());
   };
@@ -169,19 +170,22 @@ void ParsedQuery::addSolutionModifiers(SolutionModifiers modifiers) {
   _havingClauses = std::move(modifiers.havingClauses_);
 
   // Process orderClause
-  // TODO<qup42, joka921> Check that all variables that are part of an
-  //  expression that is ordered on are visible in the Query Body.
-  auto processVariableOrderKey = [this](VariableOrderKey orderKey) {
+  auto processVariableOrderKey = [this, &checkVariableIsVisible](
+                                     VariableOrderKey orderKey) {
     // Check whether grouping is done. The variable being ordered by
     // must then be either grouped or the result of an alias in the select.
     const vector<Variable>& groupByVariables = _groupByVariables;
-    if (!groupByVariables.empty() &&
-        !ad_utility::contains(groupByVariables, orderKey.variable_) &&
-        (hasConstructClause() ||
-         !ad_utility::contains_if(selectClause().getAliases(),
-                                  [&orderKey](const Alias& alias) {
-                                    return alias._target == orderKey.variable_;
-                                  }))) {
+    if (groupByVariables.empty()) {
+      checkVariableIsVisible(orderKey.variable_, "ORDERY BY");
+    } else if (!ad_utility::contains(groupByVariables, orderKey.variable_) &&
+               // `ConstructClause` has no Aliases. So the variable can never be
+               // the result of an Alias.
+               (hasConstructClause() ||
+                !ad_utility::contains_if(selectClause().getAliases(),
+                                         [&orderKey](const Alias& alias) {
+                                           return alias._target ==
+                                                  orderKey.variable_;
+                                         }))) {
       throw ParseException(
           "Variable " + orderKey.variable_.name() +
           " was used in an ORDER BY "
@@ -298,13 +302,13 @@ void ParsedQuery::addSolutionModifiers(SolutionModifiers modifiers) {
       return;
     }
 
-    for (const auto* variable : constructClause().containedVariables()) {
-      if (!ad_utility::contains(_groupByVariables, *variable)) {
-        throw ParseException("Variable " + variable->name() +
+    for (const auto& variable : constructClause().containedVariables()) {
+      if (!ad_utility::contains(_groupByVariables, variable)) {
+        throw ParseException("Variable " + variable.name() +
                              " is used but not "
                              "aggregated despite the query not being "
                              "grouped by " +
-                             variable->name() + ".");
+                             variable.name() + ".");
       }
     }
   }

src/parser/ParsedQuery.h

@@ -137,7 +137,7 @@ class ParsedQuery {
     auto addVariable = [&variable](auto& clause) {
       clause.addVisibleVariable(variable);
     };
-    std::visit(ad_utility::OverloadCallOperator{addVariable}, _clause);
+    std::visit(addVariable, _clause);
   }
 
   // Add variables, that were found in the SubQuery body.
@@ -149,14 +149,7 @@ class ParsedQuery {
 
   // Returns all variables that are visible in the Query Body.
   const std::vector<Variable>& getVisibleVariables() {
-    // TODO: clang-tidy was complaining with visit.
-    if (hasSelectClause()) {
-      return selectClause().getVisibleVariables();
-    } else if (hasConstructClause()) {
-      return constructClause().getVisibleVariables();
-    } else {
-      AD_FAIL();
-    }
+    return std::visit(&parsedQuery::ClauseBase::getVisibleVariables, _clause);
   }
 
   auto& children() { return _rootGraphPattern._graphPatterns; }

src/parser/SparqlParser.cpp

@@ -25,13 +25,10 @@ ParsedQuery SparqlParser::parseQuery(std::string query) {
       std::move(query),
       {{INTERNAL_PREDICATE_PREFIX_NAME, INTERNAL_PREDICATE_PREFIX_IRI}}};
   auto resultOfParseAndRemainingText = p.parseTypesafe(&AntlrParser::query);
-  if (!resultOfParseAndRemainingText.remainingText_.empty()) {
-    // TODO: add Exception Metadata
-    throw ParseException(
-        "A query was successfully parsed, but the following remainder of the "
-        "input was ignored by the parsed: " +
-        resultOfParseAndRemainingText.remainingText_);
-  }
+  // The query rule ends with <EOF> so the parse always has to consume the whole
+  // input. If this is not the case a ParseException should have been thrown at
+  // an earlier point.
+  AD_CHECK(resultOfParseAndRemainingText.remainingText_.empty());
   return std::move(resultOfParseAndRemainingText.resultOfParse_);
 }
 

src/parser/SparqlParser.h

@@ -15,8 +15,12 @@
 
 using std::string;
 
-// A simple parser of SPARQL.
-// No supposed to feature the complete query language.
+// The SPARQL parser used by QLever. The actual parsing is delegated to a parser
+// that is based on ANTLR4, which recognises the complete SPARQL 1.1 QL grammar.
+// For valid SPARQL queries that are not supported by QLever a reasonable error
+// message is given. The only thing that is still parsed manually by this class
+// are FILTER clauses, because QLever doesn't support arbitrary expressions in
+// filters yet.
 class SparqlParser {
  public:
   explicit SparqlParser(const string& query);

src/parser/sparqlParser/SparqlQleverVisitor.cpp

@@ -113,9 +113,7 @@ PathTuples joinPredicateAndObject(VarOrPath predicate, ObjectList objectList) {
     // TODO The fulltext index should perform the splitting of its keywords,
     //  and not the SparqlParser.
     if (PropertyPath* path = std::get_if<PropertyPath>(&predicate)) {
-      if (path->asString() == CONTAINS_WORD_PREDICATE ||
-          // TODO _NS no longer needed?
-          path->asString() == CONTAINS_WORD_PREDICATE_NS) {
+      if (path->asString() == CONTAINS_WORD_PREDICATE) {
         if (const Literal* literal =
                 unwrapVariant<VarOrTerm, GraphTerm, Literal>(object)) {
           object = Literal{stripAndLowercaseKeywordLiteral(literal->literal())};
@@ -947,13 +945,27 @@ Node Visitor::visitTypesafe(Parser::ObjectRContext* ctx) {
 // ___________________________________________________________________________
 vector<TripleWithPropertyPath> SparqlQleverVisitor::visitTypesafe(
     SparqlAutomaticParser::TriplesSameSubjectPathContext* ctx) {
+  auto setTextscoreVisibleIfPresent = [this](VarOrTerm& subject,
+                                             VarOrPath& predicate) {
+    if (auto* var = std::get_if<Variable>(&subject)) {
+      if (auto* propertyPath = std::get_if<PropertyPath>(&predicate)) {
+        if (propertyPath->asString() == CONTAINS_ENTITY_PREDICATE ||
+            propertyPath->asString() == CONTAINS_WORD_PREDICATE) {
+          addVisibleVariable(
+              absl::StrCat(TEXTSCORE_VARIABLE_PREFIX, var->name().substr(1)));
+        }
+      }
+    }
+  };
+
   if (ctx->varOrTerm()) {
     vector<TripleWithPropertyPath> triples;
     auto subject = visitTypesafe(ctx->varOrTerm());
     auto tuples = visitTypesafe(ctx->propertyListPathNotEmpty());
     for (auto& [predicate, object] : tuples) {
       // TODO<clang,c++20> clang does not yet support emplace_back for
       // aggregates.
+      setTextscoreVisibleIfPresent(subject, predicate);
       triples.push_back(TripleWithPropertyPath{subject, std::move(predicate),
                                                std::move(object)});
     }

test/SparqlAntlrParserTest.cpp

@@ -905,8 +905,6 @@ TEST(SparqlParser, SelectQuery) {
           m::SelectQuery(m::Select({Variable{"?x"}}), DummyGraphPatternMatcher),
           m::pq::Having({{SparqlFilter::FilterType::GT, "?x", "5"}}),
           m::pq::OrderKeys({{Variable{"?y"}, false}})));
-  // TODO<qup42, joka921> enable Tests once the corresponding checks are
-  //  implemented.
 
   // Grouping by a variable or expression which contains a variable
   // that is not visible in the query body is not allowed.
@@ -915,7 +913,7 @@ TEST(SparqlParser, SelectQuery) {
       "SELECT (COUNT(?a) as ?d) WHERE { ?a ?b ?c } GROUP BY (?x - 10)");
   // Ordering by a variable or expression which contains a variable that is not
   // visible in the query body is not allowed.
-  // expectSelectQueryFails("SELECT ?a WHERE { ?a ?b ?c } ORDER BY ?x");
+  expectSelectQueryFails("SELECT ?a WHERE { ?a ?b ?c } ORDER BY ?x");
   expectSelectQueryFails("SELECT ?a WHERE { ?a ?b ?c } ORDER BY (?x - 10)");
   // All variables used in an aggregate must be visible in the query body.
   expectSelectQueryFails(
@@ -936,9 +934,14 @@ TEST(SparqlParser, Query) {
   auto expectQuery = ExpectCompleteParse<&Parser::query>{
       {{INTERNAL_PREDICATE_PREFIX_NAME, INTERNAL_PREDICATE_PREFIX_IRI}}};
   auto expectQueryFails = ExpectParseFails<&Parser::query>{};
+  // TODO: re-add simple tests that also assert the ParsedQuery.
   // Test that `_originalString` is correctly set.
   expectQuery("SELECT * WHERE { ?a <bar> ?foo }",
-              m::pq::OriginalString("SELECT * WHERE { ?a <bar> ?foo }"));
+              testing::AllOf(
+                  m::SelectQuery(
+                      m::AsteriskSelect(),
+                      m::GraphPattern(m::Triples({{"?a", "<bar>", "?foo"}}))),
+                  m::pq::OriginalString("SELECT * WHERE { ?a <bar> ?foo }")));
   expectQuery("SELECT * WHERE { ?x ?y ?z }",
               m::pq::OriginalString("SELECT * WHERE { ?x ?y ?z }"));
   expectQuery(

test/SparqlParserTest.cpp

@@ -856,7 +856,7 @@ TEST(ParserTest, testSolutionModifiers) {
   {
     auto pq = SparqlParser::parseQuery(
         "SELECT DISTINCT ?x ?ql_textscore_x ?y WHERE \t {?x "
-        "<test:myrel> ?y}\n"
+        "ql:contains-entity ?y}\n"
         "ORDER BY ASC(?y) DESC(?ql_textscore_x) LIMIT 10 OFFSET 15");
     ASSERT_TRUE(pq.hasSelectClause());
     const auto& selectClause = pq.selectClause();