WIP - Add support for adding fields to schema

client/db.go

@@ -21,6 +21,7 @@ import (
 
 type DB interface {
 	AddSchema(context.Context, string) error
+	PatchSchema(context.Context, string) error
 
 	CreateCollection(context.Context, CollectionDescription) (Collection, error)
 	CreateCollectionTxn(context.Context, datastore.Txn, CollectionDescription) (Collection, error)

db/collection.go

@@ -199,6 +199,59 @@ func (db *db) CreateCollectionTxn(
 	return col, nil
 }
 
+func (db *db) UpdateCollection( //todo - add to DB interface?  Could be unsafe atm.  Move validation into here?
+	ctx context.Context,
+	txn datastore.Txn,
+	desc client.CollectionDescription,
+) (client.Collection, error) {
+	for i, field := range desc.Schema.Fields {
+		if field.ID == client.FieldID(0) {
+			// This is not wonderful and will probably break when we add the ability
+			// to delete fields. Location is also not great
+			field.ID = client.FieldID(i)
+			desc.Schema.Fields[i] = field
+		}
+	}
+
+	globalSchemaBuf, err := json.Marshal(desc.Schema)
+	if err != nil {
+		return nil, err
+	}
+
+	buf, err := json.Marshal(desc)
+	if err != nil {
+		return nil, err
+	}
+
+	cid, err := core.NewSHA256CidV1(globalSchemaBuf)
+	if err != nil {
+		return nil, err
+	}
+	schemaVersionID := cid.String()
+
+	collectionSchemaVersionKey := core.NewCollectionSchemaVersionKey(schemaVersionID)
+	// Whilst the schemaVersionKey is global, the data persisted at the key's location
+	// is local to the node (the global only elements are not useful beyond key generation).
+	err = txn.Systemstore().Put(ctx, collectionSchemaVersionKey.ToDS(), buf)
+	if err != nil {
+		return nil, err
+	}
+
+	collectionSchemaKey := core.NewCollectionSchemaKey(desc.Schema.SchemaID)
+	err = txn.Systemstore().Put(ctx, collectionSchemaKey.ToDS(), []byte(schemaVersionID))
+	if err != nil {
+		return nil, err
+	}
+
+	collectionKey := core.NewCollectionKey(desc.Name)
+	err = txn.Systemstore().Put(ctx, collectionKey.ToDS(), []byte(schemaVersionID))
+	if err != nil {
+		return nil, err
+	}
+
+	return db.GetCollectionByNameTxn(ctx, txn, desc.Name)
+}
+
 // getCollectionByVersionId returns the [*collection] at the given [schemaVersionId] version.
 //
 // Will return an error if the given key is empty, or not found.

db/schema.go

@@ -12,9 +12,13 @@ package db
 
 import (
 	"context"
+	"encoding/json"
+	"strings"
 
+	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/sourcenetwork/defradb/client"
 	"github.com/sourcenetwork/defradb/datastore"
+	"github.com/sourcenetwork/defradb/errors"
 )
 
 // AddSchema takes the provided schema in SDL format, and applies it to the database,
@@ -86,3 +90,119 @@ func (db *db) getCollectionDescriptions(ctx context.Context, txn datastore.Txn)
 
 	return descriptions, nil
 }
+
+func (db *db) PatchSchema(ctx context.Context, patchString string) error {
+	txn, err := db.NewTxn(ctx, false)
+	if err != nil {
+		return err
+	}
+	defer txn.Discard(ctx)
+
+	collectionsByName, err := db.getCollectionsByName(ctx, txn)
+	if err != nil {
+		return err
+	}
+
+	err = validateSchemaPatch(collectionsByName, patchString)
+	if err != nil {
+		return err
+	}
+
+	patch, err := jsonpatch.DecodePatch([]byte(patchString))
+	if err != nil {
+		return err
+	}
+
+	existingDescriptionJson, err := json.Marshal(collectionsByName)
+	if err != nil {
+		return err
+	}
+
+	newDescriptionJson, err := patch.Apply(existingDescriptionJson)
+	if err != nil {
+		return err
+	}
+
+	var newDescriptionsByName map[string]client.CollectionDescription
+	err = json.Unmarshal([]byte(newDescriptionJson), &newDescriptionsByName)
+	if err != nil {
+		return err
+	}
+
+	newDescriptions := []client.CollectionDescription{}
+	for _, desc := range newDescriptionsByName {
+		newDescriptions = append(newDescriptions, desc)
+	}
+
+	err = db.parser.SetSchema(ctx, txn, newDescriptions)
+	if err != nil {
+		return err
+	}
+
+	for _, desc := range newDescriptions {
+		if _, err := db.UpdateCollection(ctx, txn, desc); err != nil { //todo - test transactional, validation in here required! Can skip in PatchSchema
+			return err
+		}
+	}
+
+	return txn.Commit(ctx)
+}
+
+func (db *db) getCollectionsByName(ctx context.Context, txn datastore.Txn) (map[string]client.CollectionDescription, error) {
+	collections, err := db.GetAllCollectionsTxn(ctx, txn)
+	if err != nil {
+		return nil, err
+	}
+
+	collectionsByName := map[string]client.CollectionDescription{}
+	for _, collection := range collections {
+		collectionsByName[collection.Name()] = collection.Description()
+	}
+
+	return collectionsByName, nil
+}
+
+func validateSchemaPatch(collectionsByName map[string]client.CollectionDescription, patchString string) error {
+	var patch []schemaPatchItem
+	err := json.Unmarshal([]byte(patchString), &patch)
+	if err != nil {
+		return err
+	}
+
+	for _, patchItem := range patch {
+		switch patchItem.Op {
+		case "add":
+			// no-op, operation is supported
+
+		default:
+			return errors.New("op not supported", errors.NewKV("op", patchItem.Op))
+		}
+
+		path := strings.TrimPrefix(patchItem.Path, "/")
+		pathItems := strings.Split(path, "/")
+
+		if len(pathItems) != 4 {
+			return errors.New("path not supported", errors.NewKV("path", patchItem.Path))
+		}
+
+		if _, isCollectionName := collectionsByName[pathItems[0]]; !isCollectionName {
+			return errors.New("unknown collection", errors.NewKV("name", pathItems[0]))
+		}
+
+		if pathItems[1] != "Schema" {
+			return errors.New("path not supported", errors.NewKV("path", patchItem.Path))
+		}
+
+		if pathItems[2] != "Fields" {
+			return errors.New("path not supported", errors.NewKV("path", patchItem.Path))
+		}
+	}
+
+	return nil
+}
+
+type schemaPatchItem struct {
+	Op    string
+	Path  string
+	Value client.FieldDescription
+}

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/bxcodec/faker v2.0.1+incompatible
 	github.com/dgraph-io/badger/v3 v3.2103.4
+	github.com/evanphx/json-patch v0.5.2
 	github.com/fxamacker/cbor/v2 v2.4.0
 	github.com/go-chi/chi/v5 v5.0.7
 	github.com/go-chi/cors v1.2.1

go.sum

@@ -202,6 +202,8 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
+github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=

tests/integration/schema/updates/add/field/create_test.go

@@ -0,0 +1,119 @@
+// Copyright 2023 Democratized Data Foundation
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package field
+
+import (
+	"testing"
+
+	testUtils "github.com/sourcenetwork/defradb/tests/integration"
+)
+
+func TestSchemaUpdatesAddFieldWithCreate(t *testing.T) {
+	test := testUtils.TestCase{
+		Description: "Test schema update, add field with create",
+		Actions: []any{
+			testUtils.SchemaUpdate{
+				Schema: `
+					type Users {
+						Name: String
+					}
+				`,
+			},
+			testUtils.CreateDoc{
+				CollectionID: 0,
+				Doc: `{
+					"Name": "John"
+				}`,
+			},
+			testUtils.SchemaPatch{
+				Patch: `
+					[
+						{ "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "Email", "Kind": 11} }
+					]
+				`,
+			},
+			testUtils.Request{
+				Request: `query {
+					Users {
+						_key
+						Name
+						Email
+					}
+				}`,
+				Results: []map[string]any{
+					{
+						"_key":  "bae-43deba43-f2bc-59f4-9056-fef661b22832",
+						"Name":  "John",
+						"Email": nil,
+					},
+				},
+			},
+		},
+	}
+	testUtils.ExecuteTestCase(t, []string{"Users"}, test)
+}
+
+func TestSchemaUpdatesAddFieldWithCreateAfterSchemaUpdate(t *testing.T) {
+	test := testUtils.TestCase{
+		Description: "Test schema update, add field with create after schema update",
+		Actions: []any{
+			testUtils.SchemaUpdate{
+				Schema: `
+					type Users {
+						Name: String
+					}
+				`,
+			},
+			testUtils.CreateDoc{
+				CollectionID: 0,
+				Doc: `{
+					"Name": "John"
+				}`,
+			},
+			testUtils.SchemaPatch{
+				Patch: `
+					[
+						{ "op": "add", "path": "/Users/Schema/Fields/-", "value": {"Name": "Email", "Kind": 11} }
+					]
+				`,
+			},
+			testUtils.CreateDoc{
+				CollectionID: 0,
+				Doc: `{
+					"Name": "Shahzad",
+					"Email": "[email protected]"
+				}`,
+			},
+			testUtils.Request{
+				Request: `query {
+					Users {
+						_key
+						Name
+						Email
+					}
+				}`,
+				Results: []map[string]any{
+					{
+						"_key":  "bae-43deba43-f2bc-59f4-9056-fef661b22832",
+						"Name":  "John",
+						"Email": nil,
+					},
+					{
+						"_key":  "bae-68926881-2eed-519b-b4eb-883b4a6624a6",
+						"Name":  "Shahzad",
+						"Email": "[email protected]",
+					},
+				},
+			},
+		},
+	}
+	testUtils.ExecuteTestCase(t, []string{"Users"}, test)
+}