Remove useless AppendVertices. (#4277)

* Remove useless AppendVertices.

* Add function to check anonymous variable.

src/graph/optimizer/CMakeLists.txt

@@ -46,6 +46,8 @@ nebula_add_library(
     rule/GetEdgesTransformRule.cpp
     rule/PushLimitDownScanEdgesAppendVerticesRule.cpp
     rule/PushTopNDownIndexScanRule.cpp
+    rule/EliminateAppendVerticesRule.cpp
+    rule/PushLimitDownScanEdgesRule.cpp
 )
 
 nebula_add_subdirectory(test)

src/graph/optimizer/rule/EliminateAppendVerticesRule.cpp

@@ -0,0 +1,84 @@
+// Copyright (c) 2022 vesoft inc. All rights reserved.
+//
+// This source code is licensed under Apache 2.0 License.
+
+#include "graph/optimizer/rule/EliminateAppendVerticesRule.h"
+
+#include "graph/optimizer/OptContext.h"
+#include "graph/optimizer/OptGroup.h"
+#include "graph/planner/plan/PlanNode.h"
+#include "graph/planner/plan/Query.h"
+#include "graph/util/ExpressionUtils.h"
+
+using nebula::graph::AppendVertices;
+using nebula::graph::PlanNode;
+using nebula::graph::Project;
+using nebula::graph::QueryContext;
+
+namespace nebula {
+namespace opt {
+
+// TODO If we prune properties in Optimizer Rules, we could check properties of AppendVertices,
+// and drop AppendVertices if it has no properties directly.
+std::unique_ptr<OptRule> EliminateAppendVerticesRule::kInstance =
+    std::unique_ptr<EliminateAppendVerticesRule>(new EliminateAppendVerticesRule());
+
+EliminateAppendVerticesRule::EliminateAppendVerticesRule() {
+  RuleSet::QueryRules().addRule(this);
+}
+
+const Pattern& EliminateAppendVerticesRule::pattern() const {
+  static Pattern pattern = Pattern::create(
+      graph::PlanNode::Kind::kProject, {Pattern::create(graph::PlanNode::Kind::kAppendVertices)});
+  return pattern;
+}
+
+bool EliminateAppendVerticesRule::match(OptContext* octx, const MatchedResult& matched) const {
+  if (!OptRule::match(octx, matched)) {
+    return false;
+  }
+  const auto* project = static_cast<const Project*>(matched.planNode({0}));
+  DCHECK_EQ(project->kind(), graph::PlanNode::Kind::kProject);
+  for (const auto& col : project->columns()->columns()) {
+    const auto* expr = graph::ExpressionUtils::findAny(col->expr(), {Expression::Kind::kPathBuild});
+    if (expr != nullptr) {
+      return false;
+    }
+  }
+  const auto* appendVertices = static_cast<const AppendVertices*>(matched.planNode({0, 0}));
+  DCHECK_EQ(appendVertices->kind(), graph::PlanNode::Kind::kAppendVertices);
+  if (appendVertices->vFilter() != nullptr || appendVertices->filter() != nullptr) {
+    return false;
+  }
+  if (!graph::AnonVarGenerator::isAnnoVar(appendVertices->colNames().back())) {  // Anonymous node
+    return false;
+  }
+  return true;
+}
+
+StatusOr<OptRule::TransformResult> EliminateAppendVerticesRule::transform(
+    OptContext* octx, const MatchedResult& matched) const {
+  auto projectGroupNode = matched.node;
+  auto appendVerticesGroupNode = matched.dependencies.front().node;
+
+  const auto project = static_cast<const Project*>(projectGroupNode->node());
+  const auto appendVertices = static_cast<const AppendVertices*>(appendVerticesGroupNode->node());
+
+  auto newProj = static_cast<Project*>(project->clone());
+  newProj->setOutputVar(project->outputVar());
+  newProj->setInputVar(appendVertices->inputVar());
+  auto newProjGroupNode = OptGroupNode::create(octx, newProj, projectGroupNode->group());
+  newProjGroupNode->setDeps(appendVerticesGroupNode->dependencies());
+
+  TransformResult result;
+  result.eraseAll = true;
+  result.newGroupNodes.emplace_back(newProjGroupNode);
+  return result;
+}
+
+std::string EliminateAppendVerticesRule::toString() const {
+  return "EliminateAppendVerticesRule";
+}
+
+}  // namespace opt
+}  // namespace nebula

src/graph/optimizer/rule/EliminateAppendVerticesRule.h

@@ -0,0 +1,30 @@
+// Copyright (c) 2022 vesoft inc. All rights reserved.
+//
+// This source code is licensed under Apache 2.0 License.
+
+#pragma once
+
+#include "graph/optimizer/OptRule.h"
+
+namespace nebula {
+namespace opt {
+
+// Eliminate useless AppendVertices node
+class EliminateAppendVerticesRule final : public OptRule {
+ public:
+  const Pattern &pattern() const override;
+
+  bool match(OptContext *ctx, const MatchedResult &matched) const override;
+
+  StatusOr<TransformResult> transform(OptContext *ctx, const MatchedResult &matched) const override;
+
+  std::string toString() const override;
+
+ private:
+  EliminateAppendVerticesRule();
+
+  static std::unique_ptr<OptRule> kInstance;
+};
+
+}  // namespace opt
+}  // namespace nebula

src/graph/optimizer/rule/GetEdgesTransformRule.cpp

@@ -33,7 +33,7 @@ GetEdgesTransformRule::GetEdgesTransformRule() {
 
 const Pattern &GetEdgesTransformRule::pattern() const {
   static Pattern pattern =
-      Pattern::create(PlanNode::Kind::kAppendVertices,
+      Pattern::create({PlanNode::Kind::kAppendVertices, PlanNode::Kind::kLimit},
                       {Pattern::create(PlanNode::Kind::kTraverse,
                                        {Pattern::create(PlanNode::Kind::kScanVertices)})});
   return pattern;
@@ -59,20 +59,22 @@ bool GetEdgesTransformRule::match(OptContext *ctx, const MatchedResult &matched)
 
 StatusOr<OptRule::TransformResult> GetEdgesTransformRule::transform(
     OptContext *ctx, const MatchedResult &matched) const {
-  auto appendVerticesGroupNode = matched.node;
-  auto appendVertices = static_cast<const AppendVertices *>(appendVerticesGroupNode->node());
+  auto appendVerticesOrLimitGroupNode = matched.node;
+  auto appendVerticesOrLimit = appendVerticesOrLimitGroupNode->node();
   auto traverseGroupNode = matched.dependencies.front().node;
   auto traverse = static_cast<const Traverse *>(traverseGroupNode->node());
   auto scanVerticesGroupNode = matched.dependencies.front().dependencies.front().node;
   auto qctx = ctx->qctx();
 
-  auto newAppendVertices = appendVertices->clone();
-  auto colSize = appendVertices->colNames().size();
-  newAppendVertices->setOutputVar(appendVertices->outputVar());
-  newAppendVertices->setColNames(
-      {appendVertices->colNames()[colSize - 2], appendVertices->colNames()[colSize - 1]});
-  auto newAppendVerticesGroupNode =
-      OptGroupNode::create(ctx, newAppendVertices, appendVerticesGroupNode->group());
+  auto newAppendVerticesOrLimit = appendVerticesOrLimit->clone();
+  newAppendVerticesOrLimit->setOutputVar(appendVerticesOrLimit->outputVar());
+  if (newAppendVerticesOrLimit->kind() == PlanNode::Kind::kAppendVertices) {
+    auto colSize = appendVerticesOrLimit->colNames().size();
+    newAppendVerticesOrLimit->setColNames({appendVerticesOrLimit->colNames()[colSize - 2],
+                                           appendVerticesOrLimit->colNames()[colSize - 1]});
+  }
+  auto newAppendVerticesOrLimitGroupNode =
+      OptGroupNode::create(ctx, newAppendVerticesOrLimit, appendVerticesOrLimitGroupNode->group());
 
   auto *newScanEdges = traverseToScanEdges(traverse);
   if (newScanEdges == nullptr) {
@@ -88,15 +90,13 @@ StatusOr<OptRule::TransformResult> GetEdgesTransformRule::transform(
   auto newProjGroup = OptGroup::create(ctx);
   auto newProjGroupNode = newProjGroup->makeGroupNode(newProj);
 
-  newAppendVerticesGroupNode->dependsOn(newProjGroup);
+  newAppendVerticesOrLimitGroupNode->dependsOn(newProjGroup);
   newProjGroupNode->dependsOn(newScanEdgesGroup);
-  for (auto dep : scanVerticesGroupNode->dependencies()) {
-    newScanEdgesGroupNode->dependsOn(dep);
-  }
+  newScanEdgesGroupNode->setDeps(scanVerticesGroupNode->dependencies());
 
   TransformResult result;
-  result.eraseCurr = true;
-  result.newGroupNodes.emplace_back(newAppendVerticesGroupNode);
+  result.eraseAll = true;
+  result.newGroupNodes.emplace_back(newAppendVerticesOrLimitGroupNode);
   return result;
 }
 

src/graph/optimizer/rule/PushLimitDownScanEdgesRule.cpp

@@ -0,0 +1,77 @@
+// Copyright (c) 2022 vesoft inc. All rights reserved.
+//
+// This source code is licensed under Apache 2.0 License.
+
+#include "graph/optimizer/rule/PushLimitDownScanEdgesRule.h"
+
+#include "graph/optimizer/OptContext.h"
+#include "graph/optimizer/OptGroup.h"
+#include "graph/planner/plan/PlanNode.h"
+#include "graph/planner/plan/Query.h"
+#include "graph/util/ExpressionUtils.h"
+
+using nebula::graph::Limit;
+using nebula::graph::PlanNode;
+using nebula::graph::QueryContext;
+using nebula::graph::ScanEdges;
+
+namespace nebula {
+namespace opt {
+
+std::unique_ptr<OptRule> PushLimitDownScanEdgesRule::kInstance =
+    std::unique_ptr<PushLimitDownScanEdgesRule>(new PushLimitDownScanEdgesRule());
+
+PushLimitDownScanEdgesRule::PushLimitDownScanEdgesRule() {
+  RuleSet::QueryRules().addRule(this);
+}
+
+const Pattern &PushLimitDownScanEdgesRule::pattern() const {
+  static Pattern pattern = Pattern::create(graph::PlanNode::Kind::kLimit,
+                                           {Pattern::create(graph::PlanNode::Kind::kScanEdges)});
+  return pattern;
+}
+
+StatusOr<OptRule::TransformResult> PushLimitDownScanEdgesRule::transform(
+    OptContext *octx, const MatchedResult &matched) const {
+  auto *qctx = octx->qctx();
+  auto limitGroupNode = matched.node;
+  auto seGroupNode = matched.dependencies.front().node;
+
+  const auto limit = static_cast<const Limit *>(limitGroupNode->node());
+  const auto se = static_cast<const ScanEdges *>(seGroupNode->node());
+
+  if (!graph::ExpressionUtils::isEvaluableExpr(limit->countExpr())) {
+    return TransformResult::noTransform();
+  }
+  int64_t limitRows = limit->offset() + limit->count(qctx);
+  if (se->limit(qctx) >= 0 && limitRows >= se->limit(qctx)) {
+    return TransformResult::noTransform();
+  }
+
+  auto newLimit = static_cast<Limit *>(limit->clone());
+  newLimit->setOutputVar(limit->outputVar());
+  auto newLimitGroupNode = OptGroupNode::create(octx, newLimit, limitGroupNode->group());
+
+  auto newSe = static_cast<ScanEdges *>(se->clone());
+  newSe->setLimit(limitRows);
+  auto newSeGroup = OptGroup::create(octx);
+  auto newSeGroupNode = newSeGroup->makeGroupNode(newSe);
+
+  newLimitGroupNode->dependsOn(newSeGroup);
+  newLimit->setInputVar(newSe->outputVar());
+  for (auto dep : seGroupNode->dependencies()) {
+    newSeGroupNode->dependsOn(dep);
+  }
+
+  TransformResult result;
+  result.eraseAll = true;
+  result.newGroupNodes.emplace_back(newLimitGroupNode);
+  return result;
+}
+
+std::string PushLimitDownScanEdgesRule::toString() const {
+  return "PushLimitDownScanEdgesRule";
+}
+
+}  // namespace opt
+}  // namespace nebula

src/graph/optimizer/rule/PushLimitDownScanEdgesRule.h

@@ -0,0 +1,58 @@
+// Copyright (c) 2022 vesoft inc. All rights reserved.
+//
+// This source code is licensed under Apache 2.0 License.
+
+#pragma once
+
+#include "graph/optimizer/OptRule.h"
+
+namespace nebula {
+namespace opt {
+
+//  Embedding limit to [[ScanEdges]]
+//  Required conditions:
+//   1. Match the pattern
+//  Benefits:
+//   1. Limit data early to optimize performance
+//
+//  Transformation:
+//  Before:
+//
+//  +--------+--------+
+//  |      Limit      |
+//  |    (limit=3)    |
+//  +--------+--------+
+//           |
+// +---------+---------+
+// |    ScanEdges      |
+// +---------+---------+
+//
+//  After:
+//
+//  +--------+--------+
+//  |      Limit      |
+//  |    (limit=3)    |
+//  +--------+--------+
+//           |
+// +---------+---------+
+// |    ScanEdges      |
+// |     (limit=3)     |
+// +---------+---------+
+
+class PushLimitDownScanEdgesRule final : public OptRule {
+ public:
+  const Pattern &pattern() const override;
+
+  StatusOr<OptRule::TransformResult> transform(OptContext *ctx,
+                                               const MatchedResult &matched) const override;
+
+  std::string toString() const override;
+
+ private:
+  PushLimitDownScanEdgesRule();
+
+  static std::unique_ptr<OptRule> kInstance;
+};
+
+}  // namespace opt
+}  // namespace nebula

src/graph/util/AnonVarGenerator.h

@@ -29,6 +29,14 @@ class AnonVarGenerator final {
     return var;
   }
 
+  // Check is variable anonymous
+  // The parser don't allow user name variable started with `_`,
+  // `_` started variable is generated by nebula only.
+  static bool isAnnoVar(const std::string& var) {
+    DCHECK(!var.empty());
+    return var.front() == '_';
+  }
+
  private:
   SymbolTable* symTable_{nullptr};
   std::unique_ptr<IdGenerator> idGen_;

tests/tck/features/optimizer/EliminateAppendVerticesRule.feature

@@ -0,0 +1,40 @@
+# Copyright (c) 2022 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+Feature: Eliminate AppendVertices rule
+
+  Background: Prepare space
+    Given a graph with space named "nba"
+
+  Scenario: eliminate AppendVertices
+    When profiling query:
+      """
+      MATCH (v:player{name: 'Tim Duncan'})-[e:like{likeness: 95}]->() return v.player.name AS name
+      """
+    Then the result should be, in any order:
+      | name         |
+      | "Tim Duncan" |
+      | "Tim Duncan" |
+    And the execution plan should be:
+      | id | name      | dependencies | operator info |
+      | 5  | Project   | 8            |               |
+      | 8  | Traverse  | 7            |               |
+      | 7  | IndexScan | 0            |               |
+      | 0  | Start     |              |               |
+
+  Scenario: eliminate AppendVertices failed with returned path
+    When profiling query:
+      """
+      MATCH p = (v:player{name: 'Tim Duncan'})-[e:like{likeness: 95}]->() return p
+      """
+    Then the result should be, in any order, with relax comparison:
+      | p                                             |
+      | <("Tim Duncan")-[:like@0]->("Tony Parker")>   |
+      | <("Tim Duncan")-[:like@0]->("Manu Ginobili")> |
+    And the execution plan should be:
+      | id | name           | dependencies | operator info |
+      | 5  | Project        | 9            |               |
+      | 9  | AppendVertices | 8            |               |
+      | 8  | Traverse       | 7            |               |
+      | 7  | IndexScan      | 0            |               |
+      | 0  | Start          |              |               |

tests/tck/features/optimizer/PrunePropertiesRule.feature

@@ -383,8 +383,7 @@ Feature: Prune Properties rule
       | id | name           | dependencies | operator info                                                                                                                                                                        |
       | 12 | Aggregate      | 13           |                                                                                                                                                                                      |
       | 13 | BiInnerJoin    | 15, 11       |                                                                                                                                                                                      |
-      | 15 | Project        | 5            |                                                                                                                                                                                      |
-      | 5  | AppendVertices | 4            | {  "props": "[{\"props\":[\"name\", \"age\", \"_tag\"],\"tagId\":9}, {\"props\":[\"name\", \"speciality\", \"_tag\"],\"tagId\":8}, {\"props\":[\"name\", \"_tag\"],\"tagId\":10}]" } |
+      | 15 | Project        | 4            |                                                                                                                                                                                      |
       | 4  | Traverse       | 3            | { "vertexProps": "" }                                                                                                                                                                |
       | 3  | Traverse       | 14           | {  "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]" }                                                                                                                           |
       | 14 | IndexScan      | 2            |                                                                                                                                                                                      |