feat: Add JSON scalar (#2254)

## Relevant issue(s) Resolves #2243 ## Description This PR adds a JSON scalar type to the schema system ## Tasks - [x] I made sure the code is well commented, particularly hard-to-understand areas. - [x] I made sure the repository-held documentation is changed accordingly. - [x] I made sure the pull request title adheres to the conventional commit style (the subset used in the project can be found in [tools/configs/chglog/config.yml](tools/configs/chglog/config.yml)). - [x] I made sure to discuss its limitations such as threats to validity, vulnerability to mistake and misuse, robustness to invalidation of assumptions, resource requirements, ... ## How has this been tested? make test Specify the platform(s) on which this was tested: - MacOS --------- Co-authored-by: Shahzad Lone <[email protected]>
sourcenetwork · Jan 24, 2024 · ba79b3b · ba79b3b
1 parent 59b4652
commit ba79b3b
Show file tree

Hide file tree

Showing 8 changed files with 284 additions and 26 deletions.
diff --git a/client/descriptions.go b/client/descriptions.go
@@ -183,6 +183,8 @@ func (f FieldKind) String() string {
 		return "[String!]"
 	case FieldKind_BLOB:
 		return "Blob"
+	case FieldKind_JSON:
+		return "JSON"
 	default:
 		return fmt.Sprint(uint8(f))
 	}
@@ -204,7 +206,7 @@ const (
 	FieldKind_STRING       FieldKind = 11
 	FieldKind_STRING_ARRAY FieldKind = 12
 	FieldKind_BLOB         FieldKind = 13
-	_                      FieldKind = 14 // safe to repurpose (was never used)
+	FieldKind_JSON         FieldKind = 14
 	_                      FieldKind = 15 // safe to repurpose (was never used)
 
 	// Embedded object, but accessed via foreign keys
@@ -242,6 +244,7 @@ var FieldKindStringToEnumMapping = map[string]FieldKind{
 	"[String]":   FieldKind_NILLABLE_STRING_ARRAY,
 	"[String!]":  FieldKind_STRING_ARRAY,
 	"Blob":       FieldKind_BLOB,
+	"JSON":       FieldKind_JSON,
 }
 
 // RelationType describes the type of relation between two types.

diff --git a/client/document.go b/client/document.go
@@ -177,7 +177,7 @@ func NewDocsFromJSON(obj []byte, sd SchemaDescription) ([]*Document, error) {
 // the typed value again as an interface.
 func validateFieldSchema(val any, field FieldDescription) (any, error) {
 	switch field.Kind {
-	case FieldKind_DocID, FieldKind_STRING, FieldKind_BLOB:
+	case FieldKind_DocID, FieldKind_STRING, FieldKind_BLOB, FieldKind_JSON:
 		return getString(val)
 
 	case FieldKind_STRING_ARRAY:

diff --git a/request/graphql/schema/collection.go b/request/graphql/schema/collection.go
@@ -415,6 +415,7 @@ func astTypeToKind(t ast.Type) (client.FieldKind, error) {
 		typeDateTime string = "DateTime"
 		typeString   string = "String"
 		typeBlob     string = "Blob"
+		typeJSON     string = "JSON"
 	)
 
 	switch astTypeVal := t.(type) {
@@ -465,6 +466,8 @@ func astTypeToKind(t ast.Type) (client.FieldKind, error) {
 			return client.FieldKind_STRING, nil
 		case typeBlob:
 			return client.FieldKind_BLOB, nil
+		case typeJSON:
+			return client.FieldKind_JSON, nil
 		default:
 			return client.FieldKind_FOREIGN_OBJECT, nil
 		}

diff --git a/request/graphql/schema/descriptions.go b/request/graphql/schema/descriptions.go
@@ -34,6 +34,7 @@ var (
 		&gql.List{}:   client.FieldKind_FOREIGN_OBJECT_ARRAY,
 		// Custom scalars
 		schemaTypes.BlobScalarType: client.FieldKind_BLOB,
+		schemaTypes.JSONScalarType: client.FieldKind_JSON,
 		// More custom ones to come
 		// - JSON
 		// - Counters
@@ -55,6 +56,7 @@ var (
 		client.FieldKind_STRING_ARRAY:          gql.NewList(gql.NewNonNull(gql.String)),
 		client.FieldKind_NILLABLE_STRING_ARRAY: gql.NewList(gql.String),
 		client.FieldKind_BLOB:                  schemaTypes.BlobScalarType,
+		client.FieldKind_JSON:                  schemaTypes.JSONScalarType,
 	}
 
 	// This map is fine to use
@@ -74,6 +76,7 @@ var (
 		client.FieldKind_STRING_ARRAY:          client.LWW_REGISTER,
 		client.FieldKind_NILLABLE_STRING_ARRAY: client.LWW_REGISTER,
 		client.FieldKind_BLOB:                  client.LWW_REGISTER,
+		client.FieldKind_JSON:                  client.LWW_REGISTER,
 		client.FieldKind_FOREIGN_OBJECT:        client.LWW_REGISTER,
 		client.FieldKind_FOREIGN_OBJECT_ARRAY:  client.NONE_CRDT,
 	}

diff --git a/request/graphql/schema/types/scalars.go b/request/graphql/schema/types/scalars.go
@@ -16,6 +16,7 @@ import (
 
 	"github.com/sourcenetwork/graphql-go"
 	"github.com/sourcenetwork/graphql-go/language/ast"
+	"github.com/valyala/fastjson"
 )
 
 // BlobPattern is a regex for validating blob hex strings
@@ -63,3 +64,55 @@ var BlobScalarType = graphql.NewScalar(graphql.ScalarConfig{
 		}
 	},
 })
+
+// coerceJSON converts the given value into a valid json string.
+// If the value cannot be converted nil is returned.
+func coerceJSON(value any) any {
+	switch value := value.(type) {
+	case []byte:
+		err := fastjson.ValidateBytes(value)
+		if err != nil {
+			// ignore this error because the value
+			// cannot be converted to a json string
+			return nil
+		}
+		return string(value)
+
+	case *[]byte:
+		return coerceJSON(*value)
+
+	case string:
+		err := fastjson.Validate(value)
+		if err != nil {
+			// ignore this error because the value
+			// cannot be converted to a json string
+			return nil
+		}
+		return value
+
+	case *string:
+		return coerceJSON(*value)
+
+	default:
+		return nil
+	}
+}
+
+var JSONScalarType = graphql.NewScalar(graphql.ScalarConfig{
+	Name:        "JSON",
+	Description: "The `JSON` scalar type represents a JSON string.",
+	// Serialize converts the value to a json string
+	Serialize: coerceJSON,
+	// ParseValue converts the value to a json string
+	ParseValue: coerceJSON,
+	// ParseLiteral converts the ast value to a json string
+	ParseLiteral: func(valueAST ast.Value) any {
+		switch valueAST := valueAST.(type) {
+		case *ast.StringValue:
+			return coerceJSON(valueAST.Value)
+		default:
+			// return nil if the value cannot be parsed
+			return nil
+		}
+	},
+})
diff --git a/request/graphql/schema/types/scalars_test.go b/request/graphql/schema/types/scalars_test.go
@@ -86,3 +86,86 @@ func TestBlobScalarTypeParseLiteral(t *testing.T) {
 		assert.Equal(t, c.expect, result)
 	}
 }
+
+func TestJSONScalarTypeParseAndSerialize(t *testing.T) {
+	validString := `"hello"`
+	validBytes := []byte(`"hello"`)
+
+	boolString := "true"
+	boolBytes := []byte("true")
+
+	intString := "0"
+	intBytes := []byte("0")
+
+	floatString := "3.14"
+	floatBytes := []byte("3.14")
+
+	objectString := `{"name": "Bob"}`
+	objectBytes := []byte(`{"name": "Bob"}`)
+
+	invalidString := "invalid"
+	invalidBytes := []byte("invalid")
+
+	cases := []struct {
+		input  any
+		expect any
+	}{
+		{validString, `"hello"`},
+		{&validString, `"hello"`},
+		{validBytes, `"hello"`},
+		{&validBytes, `"hello"`},
+		{boolString, "true"},
+		{&boolString, "true"},
+		{boolBytes, "true"},
+		{&boolBytes, "true"},
+		{[]byte("true"), "true"},
+		{[]byte("false"), "false"},
+		{intString, "0"},
+		{&intString, "0"},
+		{intBytes, "0"},
+		{&intBytes, "0"},
+		{floatString, "3.14"},
+		{&floatString, "3.14"},
+		{floatBytes, "3.14"},
+		{&floatBytes, "3.14"},
+		{invalidString, nil},
+		{&invalidString, nil},
+		{invalidBytes, nil},
+		{&invalidBytes, nil},
+		{objectString, `{"name": "Bob"}`},
+		{&objectString, `{"name": "Bob"}`},
+		{objectBytes, `{"name": "Bob"}`},
+		{&objectBytes, `{"name": "Bob"}`},
+		{nil, nil},
+		{0, nil},
+		{false, nil},
+	}
+	for _, c := range cases {
+		parsed := JSONScalarType.ParseValue(c.input)
+		assert.Equal(t, c.expect, parsed)
+
+		serialized := JSONScalarType.Serialize(c.input)
+		assert.Equal(t, c.expect, serialized)
+	}
+}
+
+func TestJSONScalarTypeParseLiteral(t *testing.T) {
+	cases := []struct {
+		input  ast.Value
+		expect any
+	}{
+		{&ast.StringValue{Value: "0"}, "0"},
+		{&ast.StringValue{Value: "invalid"}, nil},
+		{&ast.IntValue{}, nil},
+		{&ast.BooleanValue{}, nil},
+		{&ast.NullValue{}, nil},
+		{&ast.EnumValue{}, nil},
+		{&ast.FloatValue{}, nil},
+		{&ast.ListValue{}, nil},
+		{&ast.ObjectValue{}, nil},
+	}
+	for _, c := range cases {
+		result := JSONScalarType.ParseLiteral(c.input)
+		assert.Equal(t, c.expect, result)
+	}
+}
diff --git a/tests/integration/schema/updates/add/field/kind/invalid_test.go b/tests/integration/schema/updates/add/field/kind/invalid_test.go
@@ -64,30 +64,6 @@ func TestSchemaUpdatesAddFieldKind9(t *testing.T) {
 	testUtils.ExecuteTestCase(t, test)
 }
 
-func TestSchemaUpdatesAddFieldKind14(t *testing.T) {
-	test := testUtils.TestCase{
-		Description: "Test schema update, add field with kind deprecated (14)",
-		Actions: []any{
-			testUtils.SchemaUpdate{
-				Schema: `
-					type Users {
-						name: String
-					}
-				`,
-			},
-			testUtils.SchemaPatch{
-				Patch: `
-					[
-						{ "op": "add", "path": "/Users/Fields/-", "value": {"Name": "foo", "Kind": 14} }
-					]
-				`,
-				ExpectedError: "no type found for given name. Type: 14",
-			},
-		},
-	}
-	testUtils.ExecuteTestCase(t, test)
-}
-
 func TestSchemaUpdatesAddFieldKind15(t *testing.T) {
 	test := testUtils.TestCase{
 		Description: "Test schema update, add field with kind deprecated (15)",

diff --git a/tests/integration/schema/updates/add/field/kind/json_test.go b/tests/integration/schema/updates/add/field/kind/json_test.go
@@ -0,0 +1,137 @@
+// Copyright 2023 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package kind
+
+import (
+	"testing"
+
+	testUtils "github.com/sourcenetwork/defradb/tests/integration"
+)
+
+func TestSchemaUpdatesAddFieldKindJSON(t *testing.T) {
+	test := testUtils.TestCase{
+		Description: "Test schema update, add field with kind json (14)",
+		Actions: []any{
+			testUtils.SchemaUpdate{
+				Schema: `
+					type Users {
+						name: String
+					}
+				`,
+			},
+			testUtils.SchemaPatch{
+				Patch: `
+					[
+						{ "op": "add", "path": "/Users/Fields/-", "value": {"Name": "foo", "Kind": 14} }
+					]
+				`,
+			},
+			testUtils.Request{
+				Request: `query {
+					Users {
+						name
+						foo
+					}
+				}`,
+				Results: []map[string]any{},
+			},
+		},
+	}
+	testUtils.ExecuteTestCase(t, test)
+}
+
+func TestSchemaUpdatesAddFieldKindJSONWithCreate(t *testing.T) {
+	test := testUtils.TestCase{
+		Description: "Test schema update, add field with kind json (14) with create",
+		Actions: []any{
+			testUtils.SchemaUpdate{
+				Schema: `
+					type Users {
+						name: String
+					}
+				`,
+			},
+			testUtils.SchemaPatch{
+				Patch: `
+					[
+						{ "op": "add", "path": "/Users/Fields/-", "value": {"Name": "foo", "Kind": 14} }
+					]
+				`,
+			},
+			testUtils.CreateDoc{
+				CollectionID: 0,
+				Doc: `{
+					"name": "John",
+					"foo": "{}"
+				}`,
+			},
+			testUtils.Request{
+				Request: `query {
+					Users {
+						name
+						foo
+					}
+				}`,
+				Results: []map[string]any{
+					{
+						"name": "John",
+						"foo":  "{}",
+					},
+				},
+			},
+		},
+	}
+	testUtils.ExecuteTestCase(t, test)
+}
+
+func TestSchemaUpdatesAddFieldKindJSONSubstitutionWithCreate(t *testing.T) {
+	test := testUtils.TestCase{
+		Description: "Test schema update, add field with kind json substitution with create",
+		Actions: []any{
+			testUtils.SchemaUpdate{
+				Schema: `
+					type Users {
+						name: String
+					}
+				`,
+			},
+			testUtils.SchemaPatch{
+				Patch: `
+					[
+						{ "op": "add", "path": "/Users/Fields/-", "value": {"Name": "foo", "Kind": "JSON"} }
+					]
+				`,
+			},
+			testUtils.CreateDoc{
+				CollectionID: 0,
+				Doc: `{
+					"name": "John",
+					"foo": "{}"
+				}`,
+			},
+			testUtils.Request{
+				Request: `query {
+					Users {
+						name
+						foo
+					}
+				}`,
+				Results: []map[string]any{
+					{
+						"name": "John",
+						"foo":  "{}",
+					},
+				},
+			},
+		},
+	}
+	testUtils.ExecuteTestCase(t, test)
+}