Use vertex key only when use_vertex_key is on (#4716)

* Revert "insert vertex key when only vertex and flag is set (#4685)"

This reverts commit fff82a6.

* Revert "only write vertex key when flag is set or explictly insert vertex (#4680)"

This reverts commit a5bed33.

* Revert "fix bug (#4675)"

This reverts commit cccc014.

* Revert "Rollback vertex key (#4629)"

This reverts commit 545137a.

* wtf

* tck format

docker/README.md

@@ -5,4 +5,4 @@ Following docker images will be ready in production.
 - [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd`
 - [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad`
 - [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged`
-- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump and meta_dump
+- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader

src/common/utils/NebulaKeyUtils.cpp

@@ -253,6 +253,7 @@ std::vector<std::string> NebulaKeyUtils::snapshotPrefix(PartitionID partId) {
   if (partId == 0) {
     result.emplace_back("");
   } else {
+    result.emplace_back(vertexPrefix(partId));
     result.emplace_back(tagPrefix(partId));
     result.emplace_back(edgePrefix(partId));
     result.emplace_back(IndexKeyUtils::indexPrefix(partId));

src/graph/service/GraphFlags.cpp

@@ -103,5 +103,3 @@ DEFINE_uint32(
     gc_worker_size,
     0,
     "Background garbage clean workers, default number is 0 which means using hardware core size.");
-
-DEFINE_bool(graph_use_vertex_key, false, "whether allow insert or query the vertex key");

src/graph/service/GraphFlags.h

@@ -65,7 +65,4 @@ DECLARE_int32(max_job_size);
 
 DECLARE_bool(enable_async_gc);
 DECLARE_uint32(gc_worker_size);
-
-DECLARE_bool(graph_use_vertex_key);
-
 #endif  // GRAPH_GRAPHFLAGS_H_

src/graph/validator/MutateValidator.cpp

@@ -46,9 +46,6 @@ Status InsertVerticesValidator::check() {
   }
 
   auto tagItems = sentence->tagItems();
-  if (!FLAGS_graph_use_vertex_key && tagItems.empty()) {
-    return Status::SemanticError("Insert vertex is forbidden, please speicify the tag");
-  }
 
   schemas_.reserve(tagItems.size());
 

src/storage/exec/GetPropNode.h

@@ -42,8 +42,10 @@ class GetTagPropNode : public QueryNode<VertexID> {
       return ret;
     }
 
-    // If none of the tag node valid, will check vertex key if use_vertex_key is true,
-    // do not emplace the row if the flag is false
+    // If none of the tag node valid, will check if vertex exists:
+    // 1. if use_vertex_key is true, check it by vertex key
+    // 2. if use_vertex_key is false, check it by scanning vertex prefix
+    // If vertex does not exists, do not emplace the row.
     if (!std::any_of(tagNodes_.begin(), tagNodes_.end(), [](const auto& tagNode) {
           return tagNode->valid();
         })) {
@@ -58,7 +60,16 @@ class GetTagPropNode : public QueryNode<VertexID> {
           return ret;
         }
       } else {
-        return nebula::cpp2::ErrorCode::SUCCEEDED;
+        // check if vId has any valid tag by prefix scan
+        std::unique_ptr<kvstore::KVIterator> iter;
+        auto tagPrefix = NebulaKeyUtils::tagPrefix(context_->vIdLen(), partId, vId);
+        ret = context_->env()->kvstore_->prefix(context_->spaceId(), partId, tagPrefix, &iter);
+        if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
+          return ret;
+        } else if (!iter->valid()) {
+          return nebula::cpp2::ErrorCode::SUCCEEDED;
+        }
+        // if has any tag, will emplace a row with vId
       }
     }
 

src/storage/exec/UpdateNode.h

@@ -488,6 +488,9 @@ class UpdateTagNode : public UpdateNode<VertexID> {
       }
     }
     // step 3, insert new vertex data
+    if (FLAGS_use_vertex_key) {
+      batchHolder->put(NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId), "");
+    }
     batchHolder->put(std::move(key_), std::move(nVal));
     return encodeBatchValue(batchHolder->getBatch());
   }

src/storage/mutate/AddVerticesProcessor.cpp

@@ -62,7 +62,6 @@ void AddVerticesProcessor::process(const cpp2::AddVerticesRequest& req) {
 void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
   const auto& partVertices = req.get_parts();
   const auto& propNamesMap = req.get_prop_names();
-  bool onlyVertex = propNamesMap.empty();
   for (auto& part : partVertices) {
     auto partId = part.first;
     const auto& vertices = part.second;
@@ -82,7 +81,7 @@ void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
         code = nebula::cpp2::ErrorCode::E_INVALID_VID;
         break;
       }
-      if (onlyVertex && FLAGS_use_vertex_key) {
+      if (FLAGS_use_vertex_key) {
         data.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid), "");
       }
       for (auto& newTag : newTags) {
@@ -140,7 +139,6 @@ void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
 void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& req) {
   const auto& partVertices = req.get_parts();
   const auto& propNamesMap = req.get_prop_names();
-  bool onlyVertex = propNamesMap.empty();
 
   for (const auto& part : partVertices) {
     auto partId = part.first;
@@ -163,7 +161,8 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re
         code = nebula::cpp2::ErrorCode::E_INVALID_VID;
         break;
       }
-      if (onlyVertex && FLAGS_use_vertex_key) {
+
+      if (FLAGS_use_vertex_key) {
         verticeData.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid));
       }
       for (const auto& newTag : newTags) {

src/tools/CMakeLists.txt

@@ -69,3 +69,4 @@ nebula_add_subdirectory(simple-kv-verify)
 endif()
 nebula_add_subdirectory(meta-dump)
 nebula_add_subdirectory(db-dump)
+nebula_add_subdirectory(db-upgrade)

src/tools/db-upgrade/CMakeLists.txt

@@ -0,0 +1,33 @@
+# Copyright (c) 2022 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+
+nebula_add_executable(
+    NAME
+        db_upgrader
+    SOURCES
+        DbUpgraderTool.cpp
+        NebulaKeyUtilsV1.cpp
+        NebulaKeyUtilsV2.cpp
+        NebulaKeyUtilsV3.cpp
+        DbUpgrader.cpp
+    OBJECTS
+        ${tools_test_deps}
+    LIBRARIES
+        ${ROCKSDB_LIBRARIES}
+        ${THRIFT_LIBRARIES}
+        ${PROXYGEN_LIBRARIES}
+)
+
+install(
+    TARGETS
+        db_upgrader
+    PERMISSIONS
+        OWNER_EXECUTE OWNER_WRITE OWNER_READ
+        GROUP_EXECUTE GROUP_READ
+        WORLD_EXECUTE WORLD_READ
+    DESTINATION
+        bin
+    COMPONENT
+        tool
+)