[Optimizer]Embed edge all predicate into Traverse

[Optimizer]Embed edge all predicate into Traverse

Move rules

fix expr util

small change

fix compile

rename rule

Split opt rule

Push edge all filter

fmt

smll fix

Add tck

fix tck

src/common/meta/SchemaManager.h

@@ -40,13 +40,33 @@ class SchemaManager {
 
   virtual StatusOr<int32_t> getPartsNum(GraphSpaceID space) = 0;
 
+  std::shared_ptr<const NebulaSchemaProvider> getTagSchema(GraphSpaceID space,
+                                                           const std::string& tag,
+                                                           SchemaVer ver = -1) {
+    auto tagId = toTagID(space, tag);
+    if (!tagId.ok()) {
+      return nullptr;
+    }
+    return getTagSchema(space, tagId.value(), ver);
+  }
+
   virtual std::shared_ptr<const NebulaSchemaProvider> getTagSchema(GraphSpaceID space,
                                                                    TagID tag,
                                                                    SchemaVer ver = -1) = 0;
 
   // Returns a negative number when the schema does not exist
   virtual StatusOr<SchemaVer> getLatestTagSchemaVersion(GraphSpaceID space, TagID tag) = 0;
 
+  std::shared_ptr<const NebulaSchemaProvider> getEdgeSchema(GraphSpaceID space,
+                                                            const std::string& edge,
+                                                            SchemaVer ver = -1) {
+    auto edgeType = toEdgeType(space, edge);
+    if (!edgeType.ok()) {
+      return nullptr;
+    }
+    return getEdgeSchema(space, edgeType.value(), ver);
+  }
+
   virtual std::shared_ptr<const NebulaSchemaProvider> getEdgeSchema(GraphSpaceID space,
                                                                     EdgeType edge,
                                                                     SchemaVer ver = -1) = 0;

src/graph/executor/query/TraverseExecutor.cpp

@@ -62,8 +62,12 @@ Status TraverseExecutor::buildRequestVids() {
     auto vidType = SchemaUtil::propTypeToValueType(metaVidType.get_type());
     for (; iter->valid(); iter->next()) {
       const auto& vid = src->eval(ctx(iter));
-      DCHECK_EQ(vid.type(), vidType)
-          << "Mismatched vid type: " << vid.type() << ", space vid type: " << vidType;
+      // FIXME(czp): Remove this DCHECK for now, we should check vid type at compile-time
+      if (vid.type() != vidType) {
+        return Status::Error("Vid type mismatched.");
+      }
+      // DCHECK_EQ(vid.type(), vidType)
+      //     << "Mismatched vid type: " << vid.type() << ", space vid type: " << vidType;
       if (vid.type() == vidType) {
         vids_.emplace(vid);
       }

src/graph/optimizer/CMakeLists.txt

@@ -35,6 +35,7 @@ nebula_add_library(
     rule/PushFilterDownInnerJoinRule.cpp
     rule/PushFilterDownNodeRule.cpp
     rule/PushFilterDownScanVerticesRule.cpp
+    rule/PushFilterDownTraverseRule.cpp
     rule/PushVFilterDownScanVerticesRule.cpp
     rule/OptimizeEdgeIndexScanByFilterRule.cpp
     rule/OptimizeTagIndexScanByFilterRule.cpp
@@ -57,8 +58,9 @@ nebula_add_library(
     rule/PushLimitDownScanEdgesAppendVerticesRule.cpp
     rule/PushTopNDownIndexScanRule.cpp
     rule/PushLimitDownScanEdgesRule.cpp
-    rule/PushFilterDownTraverseRule.cpp
+    rule/PushFilterThroughAppendVerticesRule.cpp
     rule/RemoveAppendVerticesBelowJoinRule.cpp
+    rule/EmbedEdgeAllPredIntoTraverseRule.cpp
     rule/PushFilterThroughAppendVerticesRule.cpp
 )
 

src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.cpp

@@ -0,0 +1,220 @@
+/* Copyright (c) 2023 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+
+#include "graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.h"
+
+#include "common/expression/AttributeExpression.h"
+#include "common/expression/ConstantExpression.h"
+#include "common/expression/Expression.h"
+#include "common/expression/PredicateExpression.h"
+#include "common/expression/PropertyExpression.h"
+#include "common/expression/VariableExpression.h"
+#include "graph/optimizer/OptContext.h"
+#include "graph/optimizer/OptGroup.h"
+#include "graph/planner/plan/PlanNode.h"
+#include "graph/planner/plan/Query.h"
+#include "graph/util/ExpressionUtils.h"
+#include "graph/visitor/RewriteVisitor.h"
+
+using nebula::Expression;
+using nebula::graph::Filter;
+using nebula::graph::PlanNode;
+using nebula::graph::QueryContext;
+using nebula::graph::Traverse;
+
+namespace nebula {
+namespace opt {
+
+std::unique_ptr<OptRule> EmbedEdgeAllPredIntoTraverseRule::kInstance =
+    std::unique_ptr<EmbedEdgeAllPredIntoTraverseRule>(new EmbedEdgeAllPredIntoTraverseRule());
+
+EmbedEdgeAllPredIntoTraverseRule::EmbedEdgeAllPredIntoTraverseRule() {
+  RuleSet::QueryRules().addRule(this);
+}
+
+const Pattern& EmbedEdgeAllPredIntoTraverseRule::pattern() const {
+  static Pattern pattern =
+      Pattern::create(PlanNode::Kind::kFilter, {Pattern::create(PlanNode::Kind::kTraverse)});
+  return pattern;
+}
+
+bool EmbedEdgeAllPredIntoTraverseRule::match(OptContext* ctx, const MatchedResult& matched) const {
+  return OptRule::match(ctx, matched);
+}
+
+bool isEdgeAllPredicate(const Expression* e,
+                        const std::string& edgeAlias,
+                        std::string& innerEdgeVar) {
+  // reset the inner edge var name
+  innerEdgeVar = "";
+  if (e->kind() != Expression::Kind::kPredicate) {
+    return false;
+  }
+  auto* pe = static_cast<const PredicateExpression*>(e);
+  if (pe->name() != "all" || !pe->hasInnerVar()) {
+    return false;
+  }
+  auto var = pe->innerVar();
+  if (!pe->collection()->isPropertyExpr()) {
+    return false;
+  }
+  // check edge collection expression
+  if (static_cast<const PropertyExpression*>(pe->collection())->prop() != edgeAlias) {
+    return false;
+  }
+  auto ves = graph::ExpressionUtils::collectAll(pe->filter(), {Expression::Kind::kAttribute});
+  for (const auto& ve : ves) {
+    auto iv = static_cast<const AttributeExpression*>(ve)->left();
+
+    // check inner vars
+    if (iv->kind() != Expression::Kind::kVar) {
+      return false;
+    }
+    // only care inner edge vars
+    if (!static_cast<const VariableExpression*>(iv)->isInner()) {
+      // FIXME(czp): support parameter/variables in edge `all` predicate
+      return false;
+    }
+
+    // edge property in AttributeExpression must be Constant string
+    auto ep = static_cast<const AttributeExpression*>(ve)->right();
+    if (ep->kind() != Expression::Kind::kConstant) {
+      return false;
+    }
+    if (!static_cast<const ConstantExpression*>(ep)->value().isStr()) {
+      return false;
+    }
+  }
+
+  innerEdgeVar = var;
+  return true;
+}
+
+// Pick sub-predicate
+// rewrite edge `all` predicates to single-hop edge predicate
+Expression* rewriteEdgeAllPredicate(const Expression* expr, const std::string& edgeAlias) {
+  std::string innerEdgeVar;
+  auto matcher = [&edgeAlias, &innerEdgeVar](const Expression* e) -> bool {
+    return isEdgeAllPredicate(e, edgeAlias, innerEdgeVar);
+  };
+  auto rewriter = [&innerEdgeVar](const Expression* e) -> Expression* {
+    DCHECK_EQ(e->kind(), Expression::Kind::kPredicate);
+    auto fe = static_cast<const PredicateExpression*>(e)->filter();
+
+    auto innerMatcher = [&innerEdgeVar](const Expression* ae) {
+      if (ae->kind() != Expression::Kind::kAttribute) {
+        return false;
+      }
+      auto innerEdgeVarExpr = static_cast<const AttributeExpression*>(ae)->left();
+      if (innerEdgeVarExpr->kind() != Expression::Kind::kVar) {
+        return false;
+      }
+      return static_cast<const VariableExpression*>(innerEdgeVarExpr)->var() == innerEdgeVar;
+    };
+
+    auto innerRewriter = [](const Expression* ae) {
+      DCHECK_EQ(ae->kind(), Expression::Kind::kAttribute);
+      auto attributeExpr = static_cast<const AttributeExpression*>(ae);
+      auto* right = attributeExpr->right();
+      // edge property name expressions have been checked in the external matcher
+      DCHECK_EQ(right->kind(), Expression::Kind::kConstant);
+      auto& prop = static_cast<const ConstantExpression*>(right)->value().getStr();
+      return EdgePropertyExpression::make(ae->getObjPool(), "*", prop);
+    };
+    // Rewrite all the inner var edge attribute expressions of `all` predicate's oldFilterNode to
+    // EdgePropertyExpression
+    return graph::RewriteVisitor::transform(fe, std::move(innerMatcher), std::move(innerRewriter));
+  };
+  return graph::RewriteVisitor::transform(expr, std::move(matcher), std::move(rewriter));
+}
+
+StatusOr<OptRule::TransformResult> EmbedEdgeAllPredIntoTraverseRule::transform(
+    OptContext* octx, const MatchedResult& matched) const {
+  auto* oldFilterGroupNode = matched.node;
+  auto* oldFilterGroup = oldFilterGroupNode->group();
+  auto* oldFilterNode = static_cast<graph::Filter*>(oldFilterGroupNode->node());
+  auto* condition = oldFilterNode->condition();
+  auto* oldTvGroupNode = matched.dependencies[0].node;
+  auto* oldTvNode = static_cast<graph::Traverse*>(oldTvGroupNode->node());
+  auto& edgeAlias = oldTvNode->edgeAlias();
+  auto qctx = octx->qctx();
+
+  // Pick all predicates containing edge `all` predicates under the AND semantics
+  auto picker = [&edgeAlias](const Expression* expr) -> bool {
+    bool neverPicked = false;
+    auto finder = [&neverPicked, &edgeAlias](const Expression* e) -> bool {
+      if (neverPicked) {
+        return false;
+      }
+      // UnaryNot change the semantics of `all` predicate to `any`, resulting in the inability to
+      // scatter the edge `all` predicate into a single-hop edge predicate(not cover double-not
+      // cases)
+      if (e->kind() == Expression::Kind::kUnaryNot) {
+        neverPicked = true;
+        return false;
+      }
+      // Not used, the picker only cares if there is an edge `all` predicate in the current operand
+      std::string innerVar;
+      return isEdgeAllPredicate(e, edgeAlias, innerVar);
+    };
+    graph::FindVisitor visitor(finder);
+    const_cast<Expression*>(expr)->accept(&visitor);
+    return !visitor.results().empty();
+  };
+  Expression* filterPicked = nullptr;
+  Expression* filterUnpicked = nullptr;
+  graph::ExpressionUtils::splitFilter(condition, picker, &filterPicked, &filterUnpicked);
+
+  if (!filterPicked) {
+    return TransformResult::noTransform();
+  }
+
+  // reconnect the existing edge filters
+  auto* edgeFilter = rewriteEdgeAllPredicate(filterPicked, edgeAlias);
+  auto* oldEdgeFilter = oldTvNode->eFilter();
+  Expression* newEdgeFilter =
+      oldEdgeFilter ? LogicalExpression::makeAnd(
+                          oldEdgeFilter->getObjPool(), edgeFilter, oldEdgeFilter->clone())
+                    : edgeFilter;
+
+  // produce new Traverse node
+  auto* newTvNode = static_cast<graph::Traverse*>(oldTvNode->clone());
+  newTvNode->setEdgeFilter(newEdgeFilter);
+  newTvNode->setInputVar(oldTvNode->inputVar());
+  newTvNode->setColNames(oldTvNode->outputVarPtr()->colNames);
+
+  // connect the optimized plan
+  TransformResult result;
+  result.eraseAll = true;
+  if (filterUnpicked) {
+    // assemble the new Filter node with the old Filter group
+    auto* newAboveFilterNode = graph::Filter::make(qctx, newTvNode, filterUnpicked);
+    newAboveFilterNode->setOutputVar(oldFilterNode->outputVar());
+    newAboveFilterNode->setColNames(oldFilterNode->colNames());
+    auto newAboveFilterGroupNode = OptGroupNode::create(octx, newAboveFilterNode, oldFilterGroup);
+    // assemble the new Traverse group below Filter
+    auto newTvGroup = OptGroup::create(octx);
+    auto newTvGroupNode = newTvGroup->makeGroupNode(newTvNode);
+    newTvGroupNode->setDeps(oldTvGroupNode->dependencies());
+    newAboveFilterGroupNode->setDeps({newTvGroup});
+    newAboveFilterNode->setInputVar(newTvNode->outputVar());
+    result.newGroupNodes.emplace_back(newAboveFilterGroupNode);
+  } else {
+    // replace the new Traverse node with the old Filter group
+    auto newTvGroupNode = OptGroupNode::create(octx, newTvNode, oldFilterGroup);
+    newTvNode->setOutputVar(oldFilterNode->outputVar());
+    newTvGroupNode->setDeps(oldTvGroupNode->dependencies());
+    result.newGroupNodes.emplace_back(newTvGroupNode);
+  }
+
+  return result;
+}
+
+std::string EmbedEdgeAllPredIntoTraverseRule::toString() const {
+  return "EmbedEdgeAllPredIntoTraverseRule";
+}
+
+}  // namespace opt
+}  // namespace nebula

src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.h

@@ -0,0 +1,39 @@
+/* Copyright (c) 2023 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+
+#pragma once
+
+#include "graph/optimizer/OptRule.h"
+
+namespace nebula {
+namespace opt {
+
+/*
+ * Before:
+ * Filter(all(i in e where i.likeness > 78))
+ *        |
+ *     Traverse
+ *
+ * After :
+ *     Traverse(eFilter_: *.likeness > 78)
+ */
+class EmbedEdgeAllPredIntoTraverseRule final : public OptRule {
+ public:
+  const Pattern &pattern() const override;
+
+  bool match(OptContext *ctx, const MatchedResult &matched) const override;
+
+  StatusOr<TransformResult> transform(OptContext *ctx, const MatchedResult &matched) const override;
+
+  std::string toString() const override;
+
+ private:
+  EmbedEdgeAllPredIntoTraverseRule();
+
+  static std::unique_ptr<OptRule> kInstance;
+};
+
+}  // namespace opt
+}  // namespace nebula

src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp

@@ -164,7 +164,7 @@ StatusOr<TransformResult> GeoPredicateIndexScanBaseRule::transform(
   }
   TransformResult result;
   result.newGroupNodes.emplace_back(optScanNode);
-  result.eraseCurr = true;
+  result.eraseAll = true;
   return result;
 }
 

src/graph/util/ExpressionUtils.cpp

@@ -1038,6 +1038,11 @@ void ExpressionUtils::splitFilter(const Expression *expr,
 
   std::vector<Expression *> &operands = logicExpr->operands();
   for (auto &operand : operands) {
+    // TODO(czp): Sink all NOTs to second layer [[Refactor]]
+    // TODO(czp): If find any not, dont pick this operand for now
+    if (ExpressionUtils::findAny(operand, {Expression::Kind::kUnaryNot})) {
+      filterUnpickedPtr->addOperand(operand->clone());
+    }
     if (picker(operand)) {
       filterPickedPtr->addOperand(operand->clone());
     } else {
@@ -1668,5 +1673,40 @@ bool ExpressionUtils::isOneStepEdgeProp(const std::string &edgeAlias, const Expr
   return graph::RewriteVisitor::transform(expr, matcher, rewriter);
 }
 
+// Transform Label Tag property expression like $-.v.player.name to Tag property like player.name
+// for more friendly to push down
+// \param pool object pool to hold ownership of objects alloacted
+// \param node the name of node, i.e. v in pattern (v)
+// \param expr the filter expression
+/*static*/ Expression *ExpressionUtils::rewriteVertexPropertyFilter(ObjectPool *pool,
+                                                                    const std::string &node,
+                                                                    Expression *expr) {
+  graph::RewriteVisitor::Matcher matcher = [&node](const Expression *e) -> bool {
+    if (e->kind() != Expression::Kind::kLabelTagProperty) {
+      return false;
+    }
+    auto *ltpExpr = static_cast<const LabelTagPropertyExpression *>(e);
+    auto *labelExpr = ltpExpr->label();
+    DCHECK(labelExpr->kind() == Expression::Kind::kInputProperty ||
+           labelExpr->kind() == Expression::Kind::kVarProperty);
+    if (labelExpr->kind() != Expression::Kind::kInputProperty &&
+        labelExpr->kind() != Expression::Kind::kVarProperty) {
+      return false;
+    }
+    auto *inputExpr = static_cast<const PropertyExpression *>(labelExpr);
+    if (inputExpr->prop() != node) {
+      return false;
+    }
+    return true;
+  };
+  graph::RewriteVisitor::Rewriter rewriter = [pool](const Expression *e) -> Expression * {
+    DCHECK_EQ(e->kind(), Expression::Kind::kLabelTagProperty);
+    auto *ltpExpr = static_cast<const LabelTagPropertyExpression *>(e);
+    auto *tagPropExpr = TagPropertyExpression::make(pool, ltpExpr->sym(), ltpExpr->prop());
+    return tagPropExpr;
+  };
+  return graph::RewriteVisitor::transform(expr, matcher, rewriter);
+}
+
 }  // namespace graph
 }  // namespace nebula