Introduce JSON_EXTRACT function (#4743)

* Introduce JSON_EXTRACT function

close: #3513
Note, we don't support the path argument in this phase

* address jievince's review commit

removed the unecessary interface of construct Map from Value

* Type handling

Only primitive types are supported

* Support depth1 nested

* lint: fmt

* ut: ctest, fixed wrong expression of Map

* fix ut errors

* tck: case for json_extract added

Co-authored-by: Sophie <[email protected]>

src/common/datatypes/Map.cpp

@@ -6,6 +6,7 @@
 #include "common/datatypes/Map.h"
 
 #include <folly/String.h>
+#include <folly/json.h>
 
 #include <sstream>
 
@@ -44,6 +45,49 @@ folly::dynamic Map::getMetaData() const {
   return mapMetadataObj;
 }
 
+// Map constructor to covert from folly::dynamic object
+// Called by function: json_extract()
+
+// TODO(wey-gu) support Datetime, deeper nested Map/Datatypes
+Map::Map(const folly::dynamic& obj) {
+  DCHECK(obj.isObject());
+  for (auto& kv : obj.items()) {
+    if (kv.second.isString()) {
+      kvs.emplace(kv.first.asString(), Value(kv.second.asString()));
+    } else if (kv.second.isInt()) {
+      kvs.emplace(kv.first.asString(), Value(kv.second.asInt()));
+    } else if (kv.second.isDouble()) {
+      kvs.emplace(kv.first.asString(), Value(kv.second.asDouble()));
+    } else if (kv.second.isBool()) {
+      kvs.emplace(kv.first.asString(), Value(kv.second.asBool()));
+    } else if (kv.second.isNull()) {
+      kvs.emplace(kv.first.asString(), Value());
+    } else if (kv.second.isObject()) {
+      std::unordered_map<std::string, Value> values;
+      for (auto& nkv : kv.second.items()) {
+        if (nkv.second.isString()) {
+          values.emplace(nkv.first.asString(), Value(nkv.second.asString()));
+        } else if (nkv.second.isInt()) {
+          values.emplace(nkv.first.asString(), Value(nkv.second.asInt()));
+        } else if (nkv.second.isDouble()) {
+          values.emplace(nkv.first.asString(), Value(nkv.second.asDouble()));
+        } else if (nkv.second.isBool()) {
+          values.emplace(nkv.first.asString(), Value(nkv.second.asBool()));
+        } else {
+          LOG(WARNING) << "JSON_EXTRACT nested layer 1: Map can be populated only by "
+                          "Bool, Double, Int, String value and null, now trying to parse from: "
+                       << nkv.second.typeName();
+        }
+      }
+      kvs.emplace(kv.first.asString(), Value(Map(std::move(values))));
+    } else {
+      LOG(WARNING) << "JSON_EXTRACT Only Bool, Double, Int, String value, null and Map(depth==1) "
+                      "are supported, now trying to parse from: "
+                   << kv.second.typeName();
+    }
+  }
+}
+
 }  // namespace nebula
 
 namespace std {

src/common/datatypes/Map.h

@@ -8,6 +8,7 @@
 
 #include <unordered_map>
 
+#include "common/base/Logging.h"
 #include "common/datatypes/Value.h"
 
 namespace nebula {
@@ -22,6 +23,8 @@ struct Map {
     kvs = std::move(values);
   }
 
+  explicit Map(const folly::dynamic& obj);
+
   Map& operator=(const Map& rhs) {
     if (this == &rhs) {
       return *this;

src/common/function/FunctionManager.cpp

@@ -5,6 +5,8 @@
 
 #include "FunctionManager.h"
 
+#include <folly/json.h>
+
 #include <boost/algorithm/string/replace.hpp>
 
 #include "common/base/Base.h"
@@ -421,6 +423,9 @@ std::unordered_map<std::string, std::vector<TypeSignature>> FunctionManager::typ
       TypeSignature({Value::Type::MAP}, Value::Type::DURATION)}},
     {"extract", {TypeSignature({Value::Type::STRING, Value::Type::STRING}, Value::Type::LIST)}},
     {"_nodeid", {TypeSignature({Value::Type::PATH, Value::Type::INT}, Value::Type::INT)}},
+    {"json_extract",
+     {TypeSignature({Value::Type::STRING}, Value::Type::MAP),
+      TypeSignature({Value::Type::STRING}, Value::Type::NULLVALUE)}},
 };
 
 // static
@@ -2766,6 +2771,34 @@ FunctionManager::FunctionManager() {
       }
     };
   }
+  {
+    auto &attr = functions_["json_extract"];
+    // note, we don't support second argument(path) like MySQL JSON_EXTRACT for now
+    attr.minArity_ = 1;
+    attr.maxArity_ = 1;
+    attr.isAlwaysPure_ = true;
+    attr.body_ = [](const auto &args) -> Value {
+      if (!args[0].get().isStr()) {
+        return Value::kNullBadType;
+      }
+      auto json = args[0].get().getStr();
+
+      // invalid string to json will be caught and returned as null
+      try {
+        auto obj = folly::parseJson(json);
+        if (!obj.isObject()) {
+          return Value::kNullBadData;
+        }
+        // if obj is empty, i.e. "{}", return empty map
+        if (obj.empty()) {
+          return Map();
+        }
+        return Map(obj);
+      } catch (const std::exception &e) {
+        return Value::kNullBadData;
+      }
+    };
+  }
 }  // NOLINT
 
 // static

src/common/function/test/FunctionManagerTest.cpp

@@ -132,7 +132,11 @@ std::unordered_map<std::string, std::vector<Value>> FunctionManagerTest::args_ =
     {"date", {Date(1984, 10, 11)}},
     {"datetime", {DateTime(1984, 10, 11, 12, 31, 14, 341)}},
     {"edge", {Edge("1", "2", -1, "e1", 0, {{"e1", 1}, {"e2", 2}})}},
-};
+    {"json_extract0", {"{\"a\": 1, \"b\": 0.2}"}},
+    {"json_extract1", {"{\"a\": 1, \"b\": 0.2, \"c\": {\"d\": true}}"}},
+    {"json_extract2", {"_"}},
+    {"json_extract3", {"{a: 1, \"b\": 0.2}"}},
+    {"json_extract4", {"{\"a\": \"foo\", \"b\": 0.2, \"c\": {\"d\": {\"e\": 0.1}}}"}}};
 
 #define TEST_FUNCTION(expr, ...)                   \
   do {                                             \
@@ -385,6 +389,21 @@ TEST_F(FunctionManagerTest, functionCall) {
   { TEST_FUNCTION(rand32, args_["empty"]); }
   { TEST_FUNCTION(now, args_["empty"]); }
   { TEST_FUNCTION(hash, args_["string"]); }
+  {
+    TEST_FUNCTION(
+        json_extract, args_["json_extract0"], Value(Map({{"a", Value(1)}, {"b", Value(0.2)}})));
+    TEST_FUNCTION(
+        json_extract,
+        args_["json_extract1"],
+        Value(Map({{"a", Value(1)}, {"b", Value(0.2)}, {"c", Value(Map({{"d", Value(true)}}))}})));
+    // invalid json string
+    TEST_FUNCTION(json_extract, args_["json_extract2"], Value::kNullBadData);
+    TEST_FUNCTION(json_extract, args_["json_extract3"], Value::kNullBadData);
+    // when there is nested Map in depth >= 2, the value will be dropped as empty Map()
+    TEST_FUNCTION(json_extract,
+                  args_["json_extract4"],
+                  Value(Map({{"a", Value("foo")}, {"b", Value(0.2)}, {"c", Value(Map())}})));
+  }
   {
     auto result = FunctionManager::get("hash", 1);
     ASSERT_TRUE(result.ok());

tests/tck/features/function/coalesce.feature

@@ -1,4 +1,4 @@
-Feature: Fetch Int Vid Edges
+Feature: Coalesce Function
 
   Background:
     Test coalesce function

tests/tck/features/function/json_extract.feature

@@ -0,0 +1,59 @@
+Feature: json_extract Function
+
+  Background:
+    Test json_extract function
+
+  Scenario: Test Positive Cases
+    When executing query:
+      """
+      YIELD JSON_EXTRACT('{"a": "foo", "b": 0.2, "c": true}') AS result;
+      """
+    Then the result should be, in any order:
+      | result                      |
+      | {a: "foo", b: 0.2, c: true} |
+    When executing query:
+      """
+      YIELD JSON_EXTRACT('{"a": 1, "b": {}, "c": {"d": true}}') AS result;
+      """
+    Then the result should be, in any order:
+      | result                      |
+      | {a: 1, b: {}, c: {d: true}} |
+    When executing query:
+      """
+      YIELD JSON_EXTRACT('{}') AS result;
+      """
+    Then the result should be, in any order:
+      | result |
+      | {}     |
+
+  Scenario: Test Cases With Invalid JSON String
+    When executing query:
+      """
+      YIELD JSON_EXTRACT('fuzz') AS result;
+      """
+    Then the result should be, in any order:
+      | result   |
+      | BAD_DATA |
+    When executing query:
+      """
+      YIELD JSON_EXTRACT(3.1415926) AS result;
+      """
+    Then a SemanticError should be raised at runtime: `JSON_EXTRACT(3.1415926)' is not a valid expression : Parameter's type error
+
+  Scenario: Test Cases Hitting Limitations
+    # Here nested Map depth is 2, the nested item is omitted:
+    When executing query:
+      """
+      YIELD JSON_EXTRACT('{"a": "foo", "b": false, "c": {"d": {"e": 0.1}}}') AS result;
+      """
+    Then the result should be, in any order:
+      | result                      |
+      | {a: "foo", b: false, c: {}} |
+    # Here List is not yet supported, the encounted value is omitted:
+    When executing query:
+      """
+      YIELD JSON_EXTRACT('{"a": "foo", "b": false, "c": [1, 2, 3]}') AS result;
+      """
+    Then the result should be, in any order:
+      | result               |
+      | {a: "foo", b: false} |