From 08002ff46aabcb058ad0a219f3a7b5a6d17f2662 Mon Sep 17 00:00:00 2001 From: "kyle.cao" Date: Thu, 6 Apr 2023 15:50:40 +0800 Subject: [PATCH 1/2] [Optimizer]Embed edge all predicate into Traverse [Optimizer]Embed edge all predicate into Traverse Move rules fix expr util small change fix compile rename rule Split opt rule Push edge all filter fmt smll fix Add tck fix tck fmt Add tck Add tck Add tck tmp fix fix Fix tck --- src/common/meta/SchemaManager.h | 20 ++ src/graph/executor/query/TraverseExecutor.cpp | 8 +- src/graph/optimizer/CMakeLists.txt | 4 +- .../rule/EmbedEdgeAllPredIntoTraverseRule.cpp | 220 ++++++++++++++ .../rule/EmbedEdgeAllPredIntoTraverseRule.h | 39 +++ .../rule/GeoPredicateIndexScanBaseRule.cpp | 2 +- src/graph/util/ExpressionUtils.cpp | 40 +++ src/graph/util/ExpressionUtils.h | 4 + .../visitor/ExtractFilterExprVisitor.cpp | 83 ++++- src/graph/visitor/ExtractFilterExprVisitor.h | 26 +- .../optimizer/CollapseProjectRule.feature | 1 - .../EmbedEdgeAllPredIntoTraverseRule.feature | 284 ++++++++++++++++++ 12 files changed, 714 insertions(+), 17 deletions(-) create mode 100644 src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.cpp create mode 100644 src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.h create mode 100644 tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature diff --git a/src/common/meta/SchemaManager.h b/src/common/meta/SchemaManager.h index 5e742ea0e20..78b0114b634 100644 --- a/src/common/meta/SchemaManager.h +++ b/src/common/meta/SchemaManager.h @@ -40,6 +40,16 @@ class SchemaManager { virtual StatusOr getPartsNum(GraphSpaceID space) = 0; + std::shared_ptr getTagSchema(GraphSpaceID space, + const std::string& tag, + SchemaVer ver = -1) { + auto tagId = toTagID(space, tag); + if (!tagId.ok()) { + return nullptr; + } + return getTagSchema(space, tagId.value(), ver); + } + virtual std::shared_ptr getTagSchema(GraphSpaceID space, TagID tag, SchemaVer ver = -1) = 0; @@ -47,6 +57,16 @@ class SchemaManager { // Returns a negative number when the schema does not exist virtual StatusOr getLatestTagSchemaVersion(GraphSpaceID space, TagID tag) = 0; + std::shared_ptr getEdgeSchema(GraphSpaceID space, + const std::string& edge, + SchemaVer ver = -1) { + auto edgeType = toEdgeType(space, edge); + if (!edgeType.ok()) { + return nullptr; + } + return getEdgeSchema(space, edgeType.value(), ver); + } + virtual std::shared_ptr getEdgeSchema(GraphSpaceID space, EdgeType edge, SchemaVer ver = -1) = 0; diff --git a/src/graph/executor/query/TraverseExecutor.cpp b/src/graph/executor/query/TraverseExecutor.cpp index 9425917b66d..899f851a92f 100644 --- a/src/graph/executor/query/TraverseExecutor.cpp +++ b/src/graph/executor/query/TraverseExecutor.cpp @@ -62,8 +62,12 @@ Status TraverseExecutor::buildRequestVids() { auto vidType = SchemaUtil::propTypeToValueType(metaVidType.get_type()); for (; iter->valid(); iter->next()) { const auto& vid = src->eval(ctx(iter)); - DCHECK_EQ(vid.type(), vidType) - << "Mismatched vid type: " << vid.type() << ", space vid type: " << vidType; + // FIXME(czp): Remove this DCHECK for now, we should check vid type at compile-time + if (vid.type() != vidType) { + return Status::Error("Vid type mismatched."); + } + // DCHECK_EQ(vid.type(), vidType) + // << "Mismatched vid type: " << vid.type() << ", space vid type: " << vidType; if (vid.type() == vidType) { vids_.emplace(vid); } diff --git a/src/graph/optimizer/CMakeLists.txt b/src/graph/optimizer/CMakeLists.txt index bd22e5204ed..aeb1b0f41da 100644 --- a/src/graph/optimizer/CMakeLists.txt +++ b/src/graph/optimizer/CMakeLists.txt @@ -35,6 +35,7 @@ nebula_add_library( rule/PushFilterDownInnerJoinRule.cpp rule/PushFilterDownNodeRule.cpp rule/PushFilterDownScanVerticesRule.cpp + rule/PushFilterDownTraverseRule.cpp rule/PushVFilterDownScanVerticesRule.cpp rule/OptimizeEdgeIndexScanByFilterRule.cpp rule/OptimizeTagIndexScanByFilterRule.cpp @@ -57,8 +58,9 @@ nebula_add_library( rule/PushLimitDownScanEdgesAppendVerticesRule.cpp rule/PushTopNDownIndexScanRule.cpp rule/PushLimitDownScanEdgesRule.cpp - rule/PushFilterDownTraverseRule.cpp + rule/PushFilterThroughAppendVerticesRule.cpp rule/RemoveAppendVerticesBelowJoinRule.cpp + rule/EmbedEdgeAllPredIntoTraverseRule.cpp rule/PushFilterThroughAppendVerticesRule.cpp ) diff --git a/src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.cpp b/src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.cpp new file mode 100644 index 00000000000..1a58da9c0b9 --- /dev/null +++ b/src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.cpp @@ -0,0 +1,220 @@ +/* Copyright (c) 2023 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.h" + +#include "common/expression/AttributeExpression.h" +#include "common/expression/ConstantExpression.h" +#include "common/expression/Expression.h" +#include "common/expression/PredicateExpression.h" +#include "common/expression/PropertyExpression.h" +#include "common/expression/VariableExpression.h" +#include "graph/optimizer/OptContext.h" +#include "graph/optimizer/OptGroup.h" +#include "graph/planner/plan/PlanNode.h" +#include "graph/planner/plan/Query.h" +#include "graph/util/ExpressionUtils.h" +#include "graph/visitor/RewriteVisitor.h" + +using nebula::Expression; +using nebula::graph::Filter; +using nebula::graph::PlanNode; +using nebula::graph::QueryContext; +using nebula::graph::Traverse; + +namespace nebula { +namespace opt { + +std::unique_ptr EmbedEdgeAllPredIntoTraverseRule::kInstance = + std::unique_ptr(new EmbedEdgeAllPredIntoTraverseRule()); + +EmbedEdgeAllPredIntoTraverseRule::EmbedEdgeAllPredIntoTraverseRule() { + RuleSet::QueryRules().addRule(this); +} + +const Pattern& EmbedEdgeAllPredIntoTraverseRule::pattern() const { + static Pattern pattern = + Pattern::create(PlanNode::Kind::kFilter, {Pattern::create(PlanNode::Kind::kTraverse)}); + return pattern; +} + +bool EmbedEdgeAllPredIntoTraverseRule::match(OptContext* ctx, const MatchedResult& matched) const { + return OptRule::match(ctx, matched); +} + +bool isEdgeAllPredicate(const Expression* e, + const std::string& edgeAlias, + std::string& innerEdgeVar) { + // reset the inner edge var name + innerEdgeVar = ""; + if (e->kind() != Expression::Kind::kPredicate) { + return false; + } + auto* pe = static_cast(e); + if (pe->name() != "all" || !pe->hasInnerVar()) { + return false; + } + auto var = pe->innerVar(); + if (!pe->collection()->isPropertyExpr()) { + return false; + } + // check edge collection expression + if (static_cast(pe->collection())->prop() != edgeAlias) { + return false; + } + auto ves = graph::ExpressionUtils::collectAll(pe->filter(), {Expression::Kind::kAttribute}); + for (const auto& ve : ves) { + auto iv = static_cast(ve)->left(); + + // check inner vars + if (iv->kind() != Expression::Kind::kVar) { + return false; + } + // only care inner edge vars + if (!static_cast(iv)->isInner()) { + // FIXME(czp): support parameter/variables in edge `all` predicate + return false; + } + + // edge property in AttributeExpression must be Constant string + auto ep = static_cast(ve)->right(); + if (ep->kind() != Expression::Kind::kConstant) { + return false; + } + if (!static_cast(ep)->value().isStr()) { + return false; + } + } + + innerEdgeVar = var; + return true; +} + +// Pick sub-predicate +// rewrite edge `all` predicates to single-hop edge predicate +Expression* rewriteEdgeAllPredicate(const Expression* expr, const std::string& edgeAlias) { + std::string innerEdgeVar; + auto matcher = [&edgeAlias, &innerEdgeVar](const Expression* e) -> bool { + return isEdgeAllPredicate(e, edgeAlias, innerEdgeVar); + }; + auto rewriter = [&innerEdgeVar](const Expression* e) -> Expression* { + DCHECK_EQ(e->kind(), Expression::Kind::kPredicate); + auto fe = static_cast(e)->filter(); + + auto innerMatcher = [&innerEdgeVar](const Expression* ae) { + if (ae->kind() != Expression::Kind::kAttribute) { + return false; + } + auto innerEdgeVarExpr = static_cast(ae)->left(); + if (innerEdgeVarExpr->kind() != Expression::Kind::kVar) { + return false; + } + return static_cast(innerEdgeVarExpr)->var() == innerEdgeVar; + }; + + auto innerRewriter = [](const Expression* ae) { + DCHECK_EQ(ae->kind(), Expression::Kind::kAttribute); + auto attributeExpr = static_cast(ae); + auto* right = attributeExpr->right(); + // edge property name expressions have been checked in the external matcher + DCHECK_EQ(right->kind(), Expression::Kind::kConstant); + auto& prop = static_cast(right)->value().getStr(); + return EdgePropertyExpression::make(ae->getObjPool(), "*", prop); + }; + // Rewrite all the inner var edge attribute expressions of `all` predicate's oldFilterNode to + // EdgePropertyExpression + return graph::RewriteVisitor::transform(fe, std::move(innerMatcher), std::move(innerRewriter)); + }; + return graph::RewriteVisitor::transform(expr, std::move(matcher), std::move(rewriter)); +} + +StatusOr EmbedEdgeAllPredIntoTraverseRule::transform( + OptContext* octx, const MatchedResult& matched) const { + auto* oldFilterGroupNode = matched.node; + auto* oldFilterGroup = oldFilterGroupNode->group(); + auto* oldFilterNode = static_cast(oldFilterGroupNode->node()); + auto* condition = oldFilterNode->condition(); + auto* oldTvGroupNode = matched.dependencies[0].node; + auto* oldTvNode = static_cast(oldTvGroupNode->node()); + auto& edgeAlias = oldTvNode->edgeAlias(); + auto qctx = octx->qctx(); + + // Pick all predicates containing edge `all` predicates under the AND semantics + auto picker = [&edgeAlias](const Expression* expr) -> bool { + bool neverPicked = false; + auto finder = [&neverPicked, &edgeAlias](const Expression* e) -> bool { + if (neverPicked) { + return false; + } + // UnaryNot change the semantics of `all` predicate to `any`, resulting in the inability to + // scatter the edge `all` predicate into a single-hop edge predicate(not cover double-not + // cases) + if (e->kind() == Expression::Kind::kUnaryNot) { + neverPicked = true; + return false; + } + // Not used, the picker only cares if there is an edge `all` predicate in the current operand + std::string innerVar; + return isEdgeAllPredicate(e, edgeAlias, innerVar); + }; + graph::FindVisitor visitor(finder); + const_cast(expr)->accept(&visitor); + return !visitor.results().empty(); + }; + Expression* filterPicked = nullptr; + Expression* filterUnpicked = nullptr; + graph::ExpressionUtils::splitFilter(condition, picker, &filterPicked, &filterUnpicked); + + if (!filterPicked) { + return TransformResult::noTransform(); + } + + // reconnect the existing edge filters + auto* edgeFilter = rewriteEdgeAllPredicate(filterPicked, edgeAlias); + auto* oldEdgeFilter = oldTvNode->eFilter(); + Expression* newEdgeFilter = + oldEdgeFilter ? LogicalExpression::makeAnd( + oldEdgeFilter->getObjPool(), edgeFilter, oldEdgeFilter->clone()) + : edgeFilter; + + // produce new Traverse node + auto* newTvNode = static_cast(oldTvNode->clone()); + newTvNode->setEdgeFilter(newEdgeFilter); + newTvNode->setInputVar(oldTvNode->inputVar()); + newTvNode->setColNames(oldTvNode->outputVarPtr()->colNames); + + // connect the optimized plan + TransformResult result; + result.eraseAll = true; + if (filterUnpicked) { + // assemble the new Filter node with the old Filter group + auto* newAboveFilterNode = graph::Filter::make(qctx, newTvNode, filterUnpicked); + newAboveFilterNode->setOutputVar(oldFilterNode->outputVar()); + newAboveFilterNode->setColNames(oldFilterNode->colNames()); + auto newAboveFilterGroupNode = OptGroupNode::create(octx, newAboveFilterNode, oldFilterGroup); + // assemble the new Traverse group below Filter + auto newTvGroup = OptGroup::create(octx); + auto newTvGroupNode = newTvGroup->makeGroupNode(newTvNode); + newTvGroupNode->setDeps(oldTvGroupNode->dependencies()); + newAboveFilterGroupNode->setDeps({newTvGroup}); + newAboveFilterNode->setInputVar(newTvNode->outputVar()); + result.newGroupNodes.emplace_back(newAboveFilterGroupNode); + } else { + // replace the new Traverse node with the old Filter group + auto newTvGroupNode = OptGroupNode::create(octx, newTvNode, oldFilterGroup); + newTvNode->setOutputVar(oldFilterNode->outputVar()); + newTvGroupNode->setDeps(oldTvGroupNode->dependencies()); + result.newGroupNodes.emplace_back(newTvGroupNode); + } + + return result; +} + +std::string EmbedEdgeAllPredIntoTraverseRule::toString() const { + return "EmbedEdgeAllPredIntoTraverseRule"; +} + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.h b/src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.h new file mode 100644 index 00000000000..3e917d51aea --- /dev/null +++ b/src/graph/optimizer/rule/EmbedEdgeAllPredIntoTraverseRule.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2023 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#pragma once + +#include "graph/optimizer/OptRule.h" + +namespace nebula { +namespace opt { + +/* + * Before: + * Filter(all(i in e where i.likeness > 78)) + * | + * Traverse + * + * After : + * Traverse(eFilter_: *.likeness > 78) + */ +class EmbedEdgeAllPredIntoTraverseRule final : public OptRule { + public: + const Pattern &pattern() const override; + + bool match(OptContext *ctx, const MatchedResult &matched) const override; + + StatusOr transform(OptContext *ctx, const MatchedResult &matched) const override; + + std::string toString() const override; + + private: + EmbedEdgeAllPredIntoTraverseRule(); + + static std::unique_ptr kInstance; +}; + +} // namespace opt +} // namespace nebula diff --git a/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp index 8e285de2349..cc0c5a443de 100644 --- a/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp +++ b/src/graph/optimizer/rule/GeoPredicateIndexScanBaseRule.cpp @@ -164,7 +164,7 @@ StatusOr GeoPredicateIndexScanBaseRule::transform( } TransformResult result; result.newGroupNodes.emplace_back(optScanNode); - result.eraseCurr = true; + result.eraseAll = true; return result; } diff --git a/src/graph/util/ExpressionUtils.cpp b/src/graph/util/ExpressionUtils.cpp index 4fa1bad8439..3217e6beee0 100644 --- a/src/graph/util/ExpressionUtils.cpp +++ b/src/graph/util/ExpressionUtils.cpp @@ -1038,6 +1038,11 @@ void ExpressionUtils::splitFilter(const Expression *expr, std::vector &operands = logicExpr->operands(); for (auto &operand : operands) { + // TODO(czp): Sink all NOTs to second layer [[Refactor]] + // TODO(czp): If find any not, dont pick this operand for now + if (ExpressionUtils::findAny(operand, {Expression::Kind::kUnaryNot})) { + filterUnpickedPtr->addOperand(operand->clone()); + } if (picker(operand)) { filterPickedPtr->addOperand(operand->clone()); } else { @@ -1668,5 +1673,40 @@ bool ExpressionUtils::isOneStepEdgeProp(const std::string &edgeAlias, const Expr return graph::RewriteVisitor::transform(expr, matcher, rewriter); } +// Transform Label Tag property expression like $-.v.player.name to Tag property like player.name +// for more friendly to push down +// \param pool object pool to hold ownership of objects alloacted +// \param node the name of node, i.e. v in pattern (v) +// \param expr the filter expression +/*static*/ Expression *ExpressionUtils::rewriteVertexPropertyFilter(ObjectPool *pool, + const std::string &node, + Expression *expr) { + graph::RewriteVisitor::Matcher matcher = [&node](const Expression *e) -> bool { + if (e->kind() != Expression::Kind::kLabelTagProperty) { + return false; + } + auto *ltpExpr = static_cast(e); + auto *labelExpr = ltpExpr->label(); + DCHECK(labelExpr->kind() == Expression::Kind::kInputProperty || + labelExpr->kind() == Expression::Kind::kVarProperty); + if (labelExpr->kind() != Expression::Kind::kInputProperty && + labelExpr->kind() != Expression::Kind::kVarProperty) { + return false; + } + auto *inputExpr = static_cast(labelExpr); + if (inputExpr->prop() != node) { + return false; + } + return true; + }; + graph::RewriteVisitor::Rewriter rewriter = [pool](const Expression *e) -> Expression * { + DCHECK_EQ(e->kind(), Expression::Kind::kLabelTagProperty); + auto *ltpExpr = static_cast(e); + auto *tagPropExpr = TagPropertyExpression::make(pool, ltpExpr->sym(), ltpExpr->prop()); + return tagPropExpr; + }; + return graph::RewriteVisitor::transform(expr, matcher, rewriter); +} + } // namespace graph } // namespace nebula diff --git a/src/graph/util/ExpressionUtils.h b/src/graph/util/ExpressionUtils.h index b1ff2bbc312..eba95c53e42 100644 --- a/src/graph/util/ExpressionUtils.h +++ b/src/graph/util/ExpressionUtils.h @@ -255,6 +255,10 @@ class ExpressionUtils { static Expression* rewriteEdgePropertyFilter(ObjectPool* pool, const std::string& edgeAlias, Expression* expr); + + static Expression* rewriteVertexPropertyFilter(ObjectPool* pool, + const std::string& node, + Expression* expr); }; } // namespace graph diff --git a/src/graph/visitor/ExtractFilterExprVisitor.cpp b/src/graph/visitor/ExtractFilterExprVisitor.cpp index 77692f4d526..f4a97e1ecc7 100644 --- a/src/graph/visitor/ExtractFilterExprVisitor.cpp +++ b/src/graph/visitor/ExtractFilterExprVisitor.cpp @@ -35,6 +35,19 @@ void ExtractFilterExprVisitor::visit(TagPropertyExpression *expr) { canBePushed_ = false; return; } + if (spaceId_ > 0) { + // Storage don't support nonexists tag or property + auto schema = schemaMng_->getTagSchema(spaceId_, expr->sym()); + if (schema == nullptr) { + canBePushed_ = false; + return; + } + auto field = schema->field(expr->prop()); + if (field == nullptr && (expr->prop() != kTag && expr->prop() != kVid)) { + canBePushed_ = false; + return; + } + } switch (pushType_) { case PushType::kGetNeighbors: canBePushed_ = false; @@ -48,7 +61,20 @@ void ExtractFilterExprVisitor::visit(TagPropertyExpression *expr) { } } -void ExtractFilterExprVisitor::visit(EdgePropertyExpression *) { +void ExtractFilterExprVisitor::visit(EdgePropertyExpression *expr) { + if (spaceId_ > 0) { + // Storage don't support nonexists edge or property + auto schema = schemaMng_->getEdgeSchema(spaceId_, expr->sym()); + if (schema == nullptr) { + canBePushed_ = false; + return; + } + auto field = schema->field(expr->prop()); + if (field == nullptr) { + canBePushed_ = false; + return; + } + } switch (pushType_) { case PushType::kGetNeighbors: case PushType::kGetEdges: @@ -88,7 +114,20 @@ void ExtractFilterExprVisitor::visit(DestPropertyExpression *) { } } -void ExtractFilterExprVisitor::visit(SourcePropertyExpression *) { +void ExtractFilterExprVisitor::visit(SourcePropertyExpression *expr) { + if (spaceId_ > 0) { + // Storage don't support nonexists tag or property + auto schema = schemaMng_->getTagSchema(spaceId_, expr->sym()); + if (schema == nullptr) { + canBePushed_ = false; + return; + } + auto field = schema->field(expr->prop()); + if (field == nullptr) { + canBePushed_ = false; + return; + } + } switch (pushType_) { case PushType::kGetNeighbors: canBePushed_ = true; @@ -100,7 +139,15 @@ void ExtractFilterExprVisitor::visit(SourcePropertyExpression *) { } } -void ExtractFilterExprVisitor::visit(EdgeSrcIdExpression *) { +void ExtractFilterExprVisitor::visit(EdgeSrcIdExpression *expr) { + if (spaceId_ > 0) { + // Storage don't support nonexists edge + auto schema = schemaMng_->getEdgeSchema(spaceId_, expr->sym()); + if (schema == nullptr) { + canBePushed_ = false; + return; + } + } switch (pushType_) { case PushType::kGetNeighbors: case PushType::kGetEdges: @@ -112,7 +159,15 @@ void ExtractFilterExprVisitor::visit(EdgeSrcIdExpression *) { } } -void ExtractFilterExprVisitor::visit(EdgeTypeExpression *) { +void ExtractFilterExprVisitor::visit(EdgeTypeExpression *expr) { + if (spaceId_ > 0) { + // Storage don't support nonexists edge + auto schema = schemaMng_->getEdgeSchema(spaceId_, expr->sym()); + if (schema == nullptr) { + canBePushed_ = false; + return; + } + } switch (pushType_) { case PushType::kGetNeighbors: case PushType::kGetEdges: @@ -124,7 +179,15 @@ void ExtractFilterExprVisitor::visit(EdgeTypeExpression *) { } } -void ExtractFilterExprVisitor::visit(EdgeRankExpression *) { +void ExtractFilterExprVisitor::visit(EdgeRankExpression *expr) { + if (spaceId_ > 0) { + // Storage don't support nonexists edge + auto schema = schemaMng_->getEdgeSchema(spaceId_, expr->sym()); + if (schema == nullptr) { + canBePushed_ = false; + return; + } + } switch (pushType_) { case PushType::kGetNeighbors: case PushType::kGetEdges: @@ -136,7 +199,15 @@ void ExtractFilterExprVisitor::visit(EdgeRankExpression *) { } } -void ExtractFilterExprVisitor::visit(EdgeDstIdExpression *) { +void ExtractFilterExprVisitor::visit(EdgeDstIdExpression *expr) { + if (spaceId_ > 0) { + // Storage don't support nonexists edge + auto schema = schemaMng_->getEdgeSchema(spaceId_, expr->sym()); + if (schema == nullptr) { + canBePushed_ = false; + return; + } + } switch (pushType_) { case PushType::kGetNeighbors: case PushType::kGetEdges: diff --git a/src/graph/visitor/ExtractFilterExprVisitor.h b/src/graph/visitor/ExtractFilterExprVisitor.h index ed917092997..67deec512ad 100644 --- a/src/graph/visitor/ExtractFilterExprVisitor.h +++ b/src/graph/visitor/ExtractFilterExprVisitor.h @@ -8,6 +8,7 @@ #include #include "common/expression/ExprVisitorImpl.h" +#include "common/meta/SchemaManager.h" namespace nebula { namespace graph { @@ -15,6 +16,11 @@ namespace graph { class ExtractFilterExprVisitor final : public ExprVisitorImpl { public: explicit ExtractFilterExprVisitor(ObjectPool *ObjPool) : pool_(ObjPool) {} + ExtractFilterExprVisitor(ObjectPool *ObjPool, + GraphSpaceID spaceId, + meta::SchemaManager *schemaMng) + : pool_(ObjPool), spaceId_(spaceId), schemaMng_(schemaMng) {} + explicit ExtractFilterExprVisitor(ObjectPool *ObjPool, std::vector colNames) : pool_(ObjPool), colNames_(std::move(colNames)) {} @@ -30,20 +36,26 @@ class ExtractFilterExprVisitor final : public ExprVisitorImpl { return remainedExpr_; } - static ExtractFilterExprVisitor makePushGetNeighbors(ObjectPool *pool) { - ExtractFilterExprVisitor visitor(pool); + static ExtractFilterExprVisitor makePushGetNeighbors(ObjectPool *pool, + GraphSpaceID spaceId = -1, + meta::SchemaManager *schemaMng = nullptr) { + ExtractFilterExprVisitor visitor(pool, spaceId, schemaMng); visitor.pushType_ = PushType::kGetNeighbors; return visitor; } - static ExtractFilterExprVisitor makePushGetVertices(ObjectPool *pool) { - ExtractFilterExprVisitor visitor(pool); + static ExtractFilterExprVisitor makePushGetVertices(ObjectPool *pool, + GraphSpaceID spaceId = -1, + meta::SchemaManager *schemaMng = nullptr) { + ExtractFilterExprVisitor visitor(pool, spaceId, schemaMng); visitor.pushType_ = PushType::kGetVertices; return visitor; } - static ExtractFilterExprVisitor makePushGetEdges(ObjectPool *pool) { - ExtractFilterExprVisitor visitor(pool); + static ExtractFilterExprVisitor makePushGetEdges(ObjectPool *pool, + GraphSpaceID spaceId = -1, + meta::SchemaManager *schemaMng = nullptr) { + ExtractFilterExprVisitor visitor(pool, spaceId, schemaMng); visitor.pushType_ = PushType::kGetEdges; return visitor; } @@ -97,6 +109,8 @@ class ExtractFilterExprVisitor final : public ExprVisitorImpl { bool remainedExprFromAnd_{false}; Expression *extractedExpr_{nullptr}; PushType pushType_{PushType::kGetNeighbors}; + GraphSpaceID spaceId_{-1}; + meta::SchemaManager *schemaMng_{nullptr}; std::vector colNames_; }; diff --git a/tests/tck/features/optimizer/CollapseProjectRule.feature b/tests/tck/features/optimizer/CollapseProjectRule.feature index a7b7afaf3ed..5e083827567 100644 --- a/tests/tck/features/optimizer/CollapseProjectRule.feature +++ b/tests/tck/features/optimizer/CollapseProjectRule.feature @@ -6,7 +6,6 @@ Feature: Collapse Project Rule Background: Given a graph with space named "nba" - @czp Scenario: Collapse Project When profiling query: """ diff --git a/tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature b/tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature new file mode 100644 index 00000000000..46a14778a5b --- /dev/null +++ b/tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature @@ -0,0 +1,284 @@ +# Copyright (c) 2023 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License. +Feature: Embed edge all predicate into Traverse + + Background: + Given a graph with space named "nba" + + Scenario: Embed edge all predicate into Traverse + When profiling query: + """ + MATCH (v:player)-[e:like*1]->(n) + WHERE all(i in e where i.likeness>90) + RETURN [i in e | i.likeness] AS likeness, n.player.age AS nage + """ + Then the result should be, in any order: + | likeness | nage | + | [99] | 33 | + | [99] | 31 | + | [99] | 29 | + | [99] | 30 | + | [99] | 25 | + | [99] | 34 | + | [99] | 41 | + | [99] | 32 | + | [99] | 30 | + | [99] | 42 | + | [99] | 36 | + | [95] | 41 | + | [95] | 42 | + | [100] | 31 | + | [95] | 30 | + | [95] | 41 | + | [95] | 36 | + | [100] | 43 | + | [99] | 34 | + | [99] | 38 | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 7 | Project | 11 | | | + | 11 | AppendVertices | 13 | | | + | 13 | Traverse | 1 | | {"filter": "(like.likeness>90)"} | + | 1 | IndexScan | 2 | | | + | 2 | Start | | | | + When profiling query: + """ + MATCH (v:player)-[e:like*2]->(n) + WHERE all(i in e where i.likeness>90) + RETURN [i in e | i.likeness] AS likeness, n.player.age AS nage + """ + Then the result should be, in any order: + | likeness | nage | + | [99, 100] | 43 | + | [99, 95] | 41 | + | [99, 95] | 36 | + | [99, 95] | 41 | + | [99, 95] | 42 | + | [95, 95] | 41 | + | [95, 95] | 36 | + | [95, 95] | 41 | + | [95, 95] | 42 | + | [99, 99] | 38 | + | [99, 99] | 34 | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 7 | Project | 11 | | | + | 11 | AppendVertices | 13 | | | + | 13 | Traverse | 1 | | {"filter": "(like.likeness>90)"} | + | 1 | IndexScan | 2 | | | + | 2 | Start | | | | + When profiling query: + """ + MATCH (v:player)-[e:like*2..4]->(n) + WHERE all(i in e where i.likeness>90) + RETURN [i in e | i.likeness] AS likeness, n.player.age AS nage + """ + Then the result should be, in any order: + | likeness | nage | + | [99, 100] | 43 | + | [99, 95] | 41 | + | [99, 95] | 36 | + | [99, 95] | 41 | + | [99, 95] | 42 | + | [99, 95, 95] | 41 | + | [99, 95, 95] | 42 | + | [99, 95, 95] | 41 | + | [99, 95, 95] | 36 | + | [99, 95, 95, 95] | 41 | + | [99, 95, 95, 95] | 41 | + | [95, 95] | 41 | + | [95, 95] | 36 | + | [95, 95, 95] | 41 | + | [95, 95] | 41 | + | [95, 95] | 42 | + | [95, 95, 95] | 41 | + | [99, 99] | 38 | + | [99, 99] | 34 | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 7 | Project | 11 | | | + | 11 | AppendVertices | 13 | | | + | 13 | Traverse | 1 | | {"filter": "(like.likeness>90)"} | + | 1 | IndexScan | 2 | | | + | 2 | Start | | | | + When profiling query: + """ + MATCH (v:player)-[e:like*0..5]->(n) + WHERE all(i in e where i.likeness>90) AND size(e)>0 AND (n.player.age>0) == true + RETURN [i in e | i.likeness] AS likeness, n.player.age AS nage + """ + Then the result should be, in any order: + | likeness | nage | + | [99, 99] | 34 | + | [99] | 38 | + | [99, 99] | 38 | + | [99] | 34 | + | [100] | 43 | + | [95, 95, 95] | 41 | + | [95, 95] | 42 | + | [95, 95] | 41 | + | [95] | 36 | + | [95] | 41 | + | [95] | 30 | + | [100] | 31 | + | [95, 95, 95] | 41 | + | [95, 95] | 36 | + | [95, 95] | 41 | + | [95] | 42 | + | [95] | 41 | + | [99, 95, 95, 95] | 41 | + | [99, 95, 95, 95] | 41 | + | [99, 95, 95] | 36 | + | [99, 95, 95] | 41 | + | [99, 95, 95] | 42 | + | [99, 95, 95] | 41 | + | [99, 95] | 42 | + | [99, 95] | 41 | + | [99, 95] | 36 | + | [99, 95] | 41 | + | [99, 100] | 43 | + | [99] | 36 | + | [99] | 42 | + | [99] | 30 | + | [99] | 32 | + | [99] | 41 | + | [99] | 34 | + | [99] | 25 | + | [99] | 30 | + | [99] | 29 | + | [99] | 31 | + | [99] | 33 | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 7 | Project | 15 | | | + | 15 | Filter | 11 | | {"condition": "((n.player.age>0)==true)"} | + | 11 | AppendVertices | 14 | | | + | 14 | Filter | 13 | | {"condition": "(size($e)>0)"} | + | 13 | Traverse | 1 | | {"edge filter": "(*.likeness>90)"} | + | 1 | IndexScan | 2 | | | + | 2 | Start | | | | + When profiling query: + """ + MATCH (v:player)-[e:like*1..5]->(n) + WHERE all(i in e where i.likeness>90) AND size(e)>0 AND (n.player.age>0) == true + RETURN [i in e | i.likeness] AS likeness, n.player.age AS nage + """ + Then the result should be, in any order: + | likeness | nage | + | [99, 99] | 34 | + | [99] | 38 | + | [99, 99] | 38 | + | [99] | 34 | + | [100] | 43 | + | [95, 95, 95] | 41 | + | [95, 95] | 42 | + | [95, 95] | 41 | + | [95] | 36 | + | [95] | 41 | + | [95] | 30 | + | [100] | 31 | + | [95, 95, 95] | 41 | + | [95, 95] | 36 | + | [95, 95] | 41 | + | [95] | 42 | + | [95] | 41 | + | [99, 95, 95, 95] | 41 | + | [99, 95, 95, 95] | 41 | + | [99, 95, 95] | 36 | + | [99, 95, 95] | 41 | + | [99, 95, 95] | 42 | + | [99, 95, 95] | 41 | + | [99, 95] | 42 | + | [99, 95] | 41 | + | [99, 95] | 36 | + | [99, 95] | 41 | + | [99, 100] | 43 | + | [99] | 36 | + | [99] | 42 | + | [99] | 30 | + | [99] | 32 | + | [99] | 41 | + | [99] | 34 | + | [99] | 25 | + | [99] | 30 | + | [99] | 29 | + | [99] | 31 | + | [99] | 33 | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 7 | Project | 15 | | | + | 15 | Filter | 11 | | {"condition": "((n.player.age>0)==true)"} | + | 11 | AppendVertices | 14 | | | + | 14 | Filter | 13 | | {"condition": "(size($e)>0)"} | + | 13 | Traverse | 1 | | {"filter": "(like.likeness>90)"} | + | 1 | IndexScan | 2 | | | + | 2 | Start | | | | + When profiling query: + """ + MATCH (v:player)-[e:like*0..5]->(n) + WHERE all(i in e where i.likeness>90) AND not all(i in e where i.likeness > 89) + RETURN [i in e | i.likeness] AS likeness, n.player.age AS nage + """ + Then the result should be, in any order: + | likeness | nage | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 7 | Project | 11 | | | + | 11 | AppendVertices | 14 | | | + | 14 | Filter | 13 | | {"condition": "(!(all(__VAR_1 IN $e WHERE ($__VAR_1.likeness>89))) AND !(all(__VAR_1 IN $e WHERE ($__VAR_1.likeness>89))) AND !(all(__VAR_1 IN $e WHERE ($__VAR_1.likeness>89))))"} | + | 13 | Traverse | 1 | | {"edge filter": "(*.likeness>90)"} | + | 1 | IndexScan | 2 | | | + | 2 | Start | | | | + When profiling query: + """ + MATCH (person:player)-[e1:like*1..2]-(friend:player) + WHERE id(person) == "Tony Parker" AND id(friend) != "Tony Parker" AND all(i in e1 where i.likeness > 0) + MATCH (friend)-[served:serve]->(friendTeam:team) + WHERE served.start_year > 2010 AND all(i in e1 where i.likeness > 1) + WITH DISTINCT friend, friendTeam + OPTIONAL MATCH (friend)<-[e2:like*2..4]-(friend2:player)<-[:like]-(friendTeam) + WITH friendTeam, count(friend2) AS numFriends, e2 + WHERE e2 IS NULL OR all(i in e2 where i IS NULL) + RETURN + friendTeam.team.name AS teamName, + numFriends, + [i in e2 | i.likeness] AS likeness + ORDER BY teamName DESC + LIMIT 8 + """ + Then the result should be, in any order, with relax comparison: + | teamName | numFriends | likeness | + | "Warriors" | 0 | NULL | + | "Trail Blazers" | 0 | NULL | + | "Spurs" | 0 | NULL | + | "Rockets" | 0 | NULL | + | "Raptors" | 0 | NULL | + | "Pistons" | 0 | NULL | + | "Lakers" | 0 | NULL | + | "Kings" | 0 | NULL | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 28 | TopN | 24 | | | + | 24 | Project | 30 | | | + | 30 | Aggregate | 29 | | | + | 29 | Filter | 44 | | {"condition": "($e2 IS NULL OR all(__VAR_2 IN $e2 WHERE $__VAR_2 IS NULL))"} | + | 44 | HashLeftJoin | 15,43 | | | + | 15 | Dedup | 14 | | | + | 14 | Project | 35 | | | + | 35 | HashInnerJoin | 53,50 | | | + | 53 | Filter | 52 | | {"condition": "(id($friend)!=\"Tony Parker\")"} | + | 52 | AppendVertices | 59 | | | + | 59 | Filter | 60 | | {"condition": "(id($person)==\"Tony Parker\")"} | + | 60 | Traverse | 2 | | {"filter": "((like.likeness>0) AND (like.likeness>1))"} | + | 2 | Dedup | 1 | | | + | 1 | PassThrough | 3 | | | + | 3 | Start | | | | + | 50 | Project | 55 | | | + | 55 | AppendVertices | 61 | | | + | 61 | Traverse | 8 | | | + | 8 | Argument | | | | + | 43 | Project | 39 | | | + | 39 | Traverse | 17 | | | + | 17 | Traverse | 16 | | | + | 16 | Argument | | | | From 348be3b9b18d2634d58a6b446a831e8c6cb2cd66 Mon Sep 17 00:00:00 2001 From: "kyle.cao" Date: Fri, 7 Apr 2023 16:41:27 +0800 Subject: [PATCH 2/2] Add some test cases --- .../EmbedEdgeAllPredIntoTraverseRule.feature | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature b/tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature index 46a14778a5b..80398fca974 100644 --- a/tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature +++ b/tests/tck/features/optimizer/EmbedEdgeAllPredIntoTraverseRule.feature @@ -1,6 +1,7 @@ # Copyright (c) 2023 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License. +@czp Feature: Embed edge all predicate into Traverse Background: @@ -42,6 +43,79 @@ Feature: Embed edge all predicate into Traverse | 13 | Traverse | 1 | | {"filter": "(like.likeness>90)"} | | 1 | IndexScan | 2 | | | | 2 | Start | | | | + When profiling query: + """ + MATCH (v:player)-[e:like*1]->(n) + WHERE all(i in e where i.likeness>90 or i.likeness<0) + RETURN [i in e | i.likeness] AS likeness, n.player.age AS nage + """ + Then the result should be, in any order: + | likeness | nage | + | [99] | 33 | + | [99] | 31 | + | [99] | 29 | + | [99] | 30 | + | [99] | 25 | + | [99] | 34 | + | [99] | 41 | + | [99] | 32 | + | [99] | 30 | + | [99] | 42 | + | [99] | 36 | + | [95] | 41 | + | [95] | 42 | + | [100] | 31 | + | [95] | 30 | + | [95] | 41 | + | [95] | 36 | + | [100] | 43 | + | [99] | 34 | + | [99] | 38 | + | [-1] | 43 | + | [-1] | 33 | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 7 | Project | 11 | | | + | 11 | AppendVertices | 13 | | | + | 13 | Traverse | 1 | | {"filter": "((like.likeness>90) OR (like.likeness<0))"} | + | 1 | IndexScan | 2 | | | + | 2 | Start | | | | + When profiling query: + """ + MATCH (v:player)-[e:like*1]->(n) + WHERE all(i in e where i.likeness>90 and v.player.name <> "x") + RETURN [i in e | i.likeness] AS likeness, n.player.age AS nage + """ + Then the result should be, in any order: + | likeness | nage | + | [99] | 33 | + | [99] | 31 | + | [99] | 29 | + | [99] | 30 | + | [99] | 25 | + | [99] | 34 | + | [99] | 41 | + | [99] | 32 | + | [99] | 30 | + | [99] | 42 | + | [99] | 36 | + | [95] | 41 | + | [95] | 42 | + | [100] | 31 | + | [95] | 30 | + | [95] | 41 | + | [95] | 36 | + | [100] | 43 | + | [99] | 34 | + | [99] | 38 | + And the execution plan should be: + | id | name | dependencies | profiling data | operator info | + | 7 | Project | 11 | | | + | 11 | Filter | 11 | | {"condition": "all(__VAR_0 IN $e WHERE (($__VAR_0.likeness>90) AND (v.player.name!=\"x\")))"} | + | 11 | AppendVertices | 13 | | | + | 13 | Traverse | 1 | | | + | 1 | IndexScan | 2 | | | + | 2 | Start | | | | When profiling query: """ MATCH (v:player)-[e:like*2]->(n)