From be403c18b76bec3b037ffe7a542095a8b971f531 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Tue, 14 May 2024 10:11:11 -0400 Subject: [PATCH] Rework relation field kinds --- client/db.go | 3 + client/definitions.go | 170 ++++ client/errors.go | 9 + client/normal_value_test.go | 4 +- client/request/consts.go | 15 + client/schema_field_description.go | 256 ++++-- internal/core/key.go | 65 ++ internal/db/backup.go | 19 +- internal/db/collection.go | 19 +- internal/db/collection_define.go | 80 +- internal/db/collection_id.go | 127 +++ internal/db/collection_update.go | 10 +- internal/db/definition_validation.go | 238 +++--- internal/db/description/collection.go | 49 ++ internal/db/description/schema.go | 23 +- internal/db/errors.go | 9 + internal/db/schema.go | 311 ++++---- internal/db/schema_id.go | 393 ++++++++++ internal/planner/mapper/mapper.go | 7 +- internal/request/graphql/schema/collection.go | 17 +- internal/request/graphql/schema/generate.go | 11 +- tests/gen/gen_auto.go | 30 +- tests/gen/gen_auto_configurator.go | 105 ++- tests/gen/gen_auto_test.go | 22 +- tests/gen/schema_parser.go | 39 +- .../updates/replace/id_test.go | 2 +- .../updates/replace/name_one_many_test.go | 102 ++- tests/integration/schema/one_many_test.go | 18 +- tests/integration/schema/one_one_test.go | 6 +- tests/integration/schema/self_ref_test.go | 737 ++++++++++++++++++ .../field/kind/foreign_object_array_test.go | 2 +- .../schema/updates/remove/simple_test.go | 4 +- tests/predefined/gen_predefined.go | 106 +-- tests/predefined/gen_predefined_test.go | 17 +- 34 files changed, 2479 insertions(+), 546 deletions(-) create mode 100644 internal/db/collection_id.go create mode 100644 internal/db/schema_id.go create mode 100644 tests/integration/schema/self_ref_test.go diff --git a/client/db.go b/client/db.go index ad2229cdb0..e77dd6cb87 100644 --- a/client/db.go +++ b/client/db.go @@ -283,6 +283,9 @@ type CollectionFetchOptions struct { // If provided, only collections with schemas of this root will be returned. SchemaRoot immutable.Option[string] + // If provided, only collections with this root will be returned. + Root immutable.Option[uint32] + // If provided, only collections with this name will be returned. Name immutable.Option[string] diff --git a/client/definitions.go b/client/definitions.go index c32fd41b4f..610c797bd5 100644 --- a/client/definitions.go +++ b/client/definitions.go @@ -11,9 +11,15 @@ package client import ( + "context" + "errors" + "fmt" "strings" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client/request" + "github.com/sourcenetwork/defradb/datastore" ) // CollectionDefinition contains the metadata defining what a Collection is. @@ -195,3 +201,167 @@ func (f FieldDefinition) GetSecondaryRelationField(c CollectionDefinition) (Fiel secondary, valid := c.GetFieldByName(strings.TrimSuffix(f.Name, request.RelatedObjectID)) return secondary, valid && !secondary.IsPrimaryRelation } + +// DefinitionCache is an object providing easy access to cached collection definitions. +type DefinitionCache struct { + // The full set of [CollectionDefinition]s within this cache + Definitions []CollectionDefinition + + // The cached Definitions mapped by the Root of their [SchemaDescription] + DefinitionsBySchemaRoot map[string]CollectionDefinition + + // The cached Definitions mapped by the Root of their [CollectionDescription] + DefinitionsByCollectionRoot map[uint32]CollectionDefinition +} + +// NewDefinitionCache creates a new [DefinitionCache] populated with the given [CollectionDefinition]s. +func NewDefinitionCache(definitions []CollectionDefinition) DefinitionCache { + definitionsBySchemaRoot := make(map[string]CollectionDefinition, len(definitions)) + definitionsByCollectionRoot := make(map[uint32]CollectionDefinition, len(definitions)) + + for _, def := range definitions { + definitionsBySchemaRoot[def.Schema.Root] = def + definitionsByCollectionRoot[def.Description.RootID] = def + } + + return DefinitionCache{ + Definitions: definitions, + DefinitionsBySchemaRoot: definitionsBySchemaRoot, + DefinitionsByCollectionRoot: definitionsByCollectionRoot, + } +} + +// GetDefinition returns the definition that the given [FieldKind] points to, if it is found in the +// given [DefinitionCache]. +// +// If the related definition is not found, default and false will be returned. +func GetDefinition( + cache DefinitionCache, + host CollectionDefinition, + kind FieldKind, +) (CollectionDefinition, bool) { + switch typedKind := kind.(type) { + case *NamedKind: + for _, def := range cache.Definitions { + if def.GetName() == typedKind.Name { + return def, true + } + } + + return CollectionDefinition{}, false + + case *SchemaKind: + def, ok := cache.DefinitionsBySchemaRoot[typedKind.Root] + return def, ok + + case *CollectionKind: + def, ok := cache.DefinitionsByCollectionRoot[typedKind.Root] + return def, ok + + case *SelfKind: + if host.Description.RootID != 0 { + return host, true + } + + if typedKind.RelativeID == "" { + return host, true + } + + hostIDBase := strings.Split(host.Schema.Root, "-")[0] + targetID := fmt.Sprintf("%s-%s", hostIDBase, typedKind.RelativeID) + + def, ok := cache.DefinitionsBySchemaRoot[targetID] + return def, ok + + default: + // no-op + } + + return CollectionDefinition{}, false +} + +// GetDefinitionUncached returns the definition that the given [FieldKind] points to, if it is found in the given store. +// +// If the related definition is not found, or an error occurs, default and false will be returned. +func GetDefinitionUncached( + ctx context.Context, + store Store, + host CollectionDefinition, + kind FieldKind, +) (CollectionDefinition, bool, error) { + switch typedKind := kind.(type) { + case *NamedKind: + col, err := store.GetCollectionByName(ctx, typedKind.Name) + if errors.Is(datastore.ErrNotFound, err) { + schemas, err := store.GetSchemas(ctx, SchemaFetchOptions{ + Name: immutable.Some(typedKind.Name), + }) + if len(schemas) == 0 || err != nil { + return CollectionDefinition{}, false, err + } + + return CollectionDefinition{ + // todo - returning the first is a temporary simplification until + // https://github.com/sourcenetwork/defradb/issues/2934 + Schema: schemas[0], + }, true, nil + } else if err != nil { + return CollectionDefinition{}, false, err + } + + return col.Definition(), true, nil + + case *SchemaKind: + schemas, err := store.GetSchemas(ctx, SchemaFetchOptions{ + Root: immutable.Some(typedKind.Root), + }) + if len(schemas) == 0 || err != nil { + return CollectionDefinition{}, false, err + } + + return CollectionDefinition{ + // todo - returning the first is a temporary simplification until + // https://github.com/sourcenetwork/defradb/issues/2934 + Schema: schemas[0], + }, true, nil + + case *CollectionKind: + cols, err := store.GetCollections(ctx, CollectionFetchOptions{ + Root: immutable.Some(typedKind.Root), + }) + + if len(cols) == 0 || err != nil { + return CollectionDefinition{}, false, err + } + + return cols[0].Definition(), true, nil + + case *SelfKind: + if host.Description.RootID != 0 { + return host, true, nil + } + + if typedKind.RelativeID == "" { + return host, true, nil + } + + hostIDBase := strings.Split(host.Schema.Root, "-")[0] + targetID := fmt.Sprintf("%s-%s", hostIDBase, typedKind.RelativeID) + + cols, err := store.GetCollections(ctx, CollectionFetchOptions{ + SchemaRoot: immutable.Some(targetID), + }) + if len(cols) == 0 || err != nil { + return CollectionDefinition{}, false, err + } + def := cols[0].Definition() + def.Description = CollectionDescription{} + + return def, true, nil + + default: + // no-op + } + + return CollectionDefinition{}, false, nil +} diff --git a/client/errors.go b/client/errors.go index dac8ebcc87..46b598b52c 100644 --- a/client/errors.go +++ b/client/errors.go @@ -32,6 +32,7 @@ const ( errCanNotNormalizeValue string = "can not normalize value" errCanNotTurnNormalValueIntoArray string = "can not turn normal value into array" errCanNotMakeNormalNilFromFieldKind string = "can not make normal nil from field kind" + errFailedToParseKind string = "failed to parse kind" ) // Errors returnable from this package. @@ -57,6 +58,7 @@ var ( ErrCanNotTurnNormalValueIntoArray = errors.New(errCanNotTurnNormalValueIntoArray) ErrCanNotMakeNormalNilFromFieldKind = errors.New(errCanNotMakeNormalNilFromFieldKind) ErrCollectionNotFound = errors.New(errCollectionNotFound) + ErrFailedToParseKind = errors.New(errFailedToParseKind) ) // NewErrFieldNotExist returns an error indicating that the given field does not exist. @@ -165,3 +167,10 @@ func NewErrCRDTKindMismatch(cType, kind string) error { func NewErrInvalidJSONPaylaod(payload string) error { return errors.New(errInvalidJSONPayload, errors.NewKV("Payload", payload)) } + +func NewErrFailedToParseKind(kind []byte) error { + return errors.New( + errCRDTKindMismatch, + errors.NewKV("Kind", kind), + ) +} diff --git a/client/normal_value_test.go b/client/normal_value_test.go index 73e9def5d6..ce454a55b4 100644 --- a/client/normal_value_test.go +++ b/client/normal_value_test.go @@ -1393,8 +1393,8 @@ func TestNormalValue_NewNormalNil(t *testing.T) { for _, kind := range FieldKindStringToEnumMapping { fieldKinds = append(fieldKinds, kind) } - fieldKinds = append(fieldKinds, ObjectKind("Object")) - fieldKinds = append(fieldKinds, ObjectArrayKind("ObjectArr")) + fieldKinds = append(fieldKinds, NewCollectionKind(1, false)) + fieldKinds = append(fieldKinds, NewCollectionKind(1, true)) for _, kind := range fieldKinds { if kind.IsNillable() { diff --git a/client/request/consts.go b/client/request/consts.go index 8567bb6952..8b98199827 100644 --- a/client/request/consts.go +++ b/client/request/consts.go @@ -72,6 +72,12 @@ const ( DeltaArgPriority = "Priority" DeltaArgDocID = "DocID" + // SelfTypeName is the name given to relation field types that reference the host type. + // + // For example, when a `User` collection contains a relation to the `User` collection the field + // will be of type [SelfTypeName]. + SelfTypeName = "Self" + LinksNameFieldName = "name" LinksCidFieldName = "cid" @@ -85,6 +91,15 @@ var ( string(DESC): DESC, } + // ReservedTypeNames is the set of type names reserved by the system. + // + // Users cannot define types using these names. + // + // For example, collections and schemas may not be defined using these names. + ReservedTypeNames = map[string]struct{}{ + SelfTypeName: {}, + } + ReservedFields = map[string]struct{}{ TypeNameFieldName: {}, VersionFieldName: {}, diff --git a/client/schema_field_description.go b/client/schema_field_description.go index 4c8f0f72d0..cc5690b72c 100644 --- a/client/schema_field_description.go +++ b/client/schema_field_description.go @@ -12,8 +12,10 @@ package client import ( "encoding/json" + "fmt" "strconv" - "strings" + + "github.com/sourcenetwork/defradb/client/request" ) // FieldKind describes the type of a field. @@ -21,12 +23,6 @@ type FieldKind interface { // String returns the string representation of this FieldKind. String() string - // Underlying returns the underlying Kind as a string. - // - // If this is an array, it will return the element kind, else it will return the same as - // [String()]. - Underlying() string - // IsNillable returns true if this kind supports nil values. IsNillable() bool @@ -62,16 +58,63 @@ type ScalarKind uint8 // ScalarArrayKind represents arrays of simple scalar field kinds, such as `[Int]`. type ScalarArrayKind uint8 -// ObjectKind represents singular objects (foreign and embedded), such as `User`. -type ObjectKind string +// CollectionKind represents a relationship with a [CollectionDescription]. +type CollectionKind struct { + // If true, this side of the relationship points to many related records. + Array bool + + // The root ID of the related [CollectionDescription]. + Root uint32 +} -// ObjectKind represents arrays of objects (foreign and embedded), such as `[User]`. -type ObjectArrayKind string +// SchemaKind represents a relationship with a [SchemaDescription]. +type SchemaKind struct { + // If true, this side of the relationship points to many related records. + Array bool + + // The root ID of the related [SchemaDescription]. + Root string +} + +// NamedKind represents a temporary declaration of a relationship to another +// [CollectionDefinition]. +// +// This is used only to temporarily describe a relationship, this kind will +// never be persisted in the store and instead will be converted to one of +// [CollectionKind], [SchemaKind] or [SelfKind] first. +type NamedKind struct { + // The current name of the related [CollectionDefinition]. + Name string + + // If true, this side of the relationship points to many related records. + Array bool +} + +// SelfKind represents a relationship with the host. +// +// This includes any other schema that formed a circular dependency with the +// host at the point at which they were created. +// +// For example: the relations in User=>Dog=>User form a circle, and would be +// defined using [SelfKind] instead of [SchemaKind]. +// +// This is because schema IDs are content IDs and cannot be generated for a +// single element within a circular dependency tree. +type SelfKind struct { + // The relative ID to the related type. If this points at its host this + // will be empty. + RelativeID string + + // If true, this side of the relationship points to many related records. + Array bool +} var _ FieldKind = ScalarKind(0) var _ FieldKind = ScalarArrayKind(0) -var _ FieldKind = ObjectKind("") -var _ FieldKind = ObjectArrayKind("") +var _ FieldKind = (*CollectionKind)(nil) +var _ FieldKind = (*SchemaKind)(nil) +var _ FieldKind = (*SelfKind)(nil) +var _ FieldKind = (*NamedKind)(nil) func (k ScalarKind) String() string { switch k { @@ -96,10 +139,6 @@ func (k ScalarKind) String() string { } } -func (k ScalarKind) Underlying() string { - return k.String() -} - func (k ScalarKind) IsNillable() bool { return true } @@ -135,10 +174,6 @@ func (k ScalarArrayKind) String() string { } } -func (k ScalarArrayKind) Underlying() string { - return strings.Trim(k.String(), "[]") -} - func (k ScalarArrayKind) IsNillable() bool { return true } @@ -151,48 +186,115 @@ func (k ScalarArrayKind) IsArray() bool { return true } -func (k ObjectKind) String() string { - return string(k) +func NewCollectionKind(root uint32, isArray bool) *CollectionKind { + return &CollectionKind{ + Root: root, + Array: isArray, + } } -func (k ObjectKind) Underlying() string { - return k.String() +func (k *CollectionKind) String() string { + if k.Array { + return fmt.Sprintf("[%v]", k.Root) + } + return strconv.FormatInt(int64(k.Root), 10) } -func (k ObjectKind) IsNillable() bool { +func (k *CollectionKind) IsNillable() bool { return true } -func (k ObjectKind) IsObject() bool { +func (k *CollectionKind) IsObject() bool { return true } -func (k ObjectKind) IsArray() bool { - return false +func (k *CollectionKind) IsArray() bool { + return k.Array +} + +func NewSchemaKind(root string, isArray bool) *SchemaKind { + return &SchemaKind{ + Root: root, + Array: isArray, + } +} + +func (k *SchemaKind) String() string { + if k.Array { + return fmt.Sprintf("[%v]", k.Root) + } + return k.Root +} + +func (k *SchemaKind) IsNillable() bool { + return true } -func (k ObjectArrayKind) String() string { - return "[" + string(k) + "]" +func (k *SchemaKind) IsObject() bool { + return true +} + +func (k *SchemaKind) IsArray() bool { + return k.Array } -func (k ObjectArrayKind) Underlying() string { - return strings.Trim(k.String(), "[]") +func NewSelfKind(relativeID string, isArray bool) *SelfKind { + return &SelfKind{ + RelativeID: relativeID, + Array: isArray, + } +} + +func (k *SelfKind) String() string { + var relativeName string + if k.RelativeID != "" { + relativeName = fmt.Sprintf("%s-%s", request.SelfTypeName, k.RelativeID) + } else { + relativeName = request.SelfTypeName + } + + if k.Array { + return fmt.Sprintf("[%s]", relativeName) + } + return relativeName } -func (k ObjectArrayKind) IsNillable() bool { +func (k *SelfKind) IsNillable() bool { return true } -func (k ObjectArrayKind) IsObject() bool { +func (k *SelfKind) IsObject() bool { return true } -func (k ObjectArrayKind) IsArray() bool { +func (k *SelfKind) IsArray() bool { + return k.Array +} + +func NewNamedKind(name string, isArray bool) *NamedKind { + return &NamedKind{ + Name: name, + Array: isArray, + } +} + +func (k *NamedKind) String() string { + if k.Array { + return fmt.Sprintf("[%v]", k.Name) + } + return k.Name +} + +func (k *NamedKind) IsNillable() bool { return true } -func (k ObjectArrayKind) MarshalJSON() ([]byte, error) { - return []byte(`"` + k.String() + `"`), nil +func (k *NamedKind) IsObject() bool { + return true +} + +func (k *NamedKind) IsArray() bool { + return k.Array } // Note: These values are serialized and persisted in the database, avoid modifying existing values. @@ -229,22 +331,24 @@ const ( // in the future. They currently roughly correspond to the GQL field types, but this // equality is not guaranteed. var FieldKindStringToEnumMapping = map[string]FieldKind{ - "ID": FieldKind_DocID, - "Boolean": FieldKind_NILLABLE_BOOL, - "[Boolean]": FieldKind_NILLABLE_BOOL_ARRAY, - "[Boolean!]": FieldKind_BOOL_ARRAY, - "Int": FieldKind_NILLABLE_INT, - "[Int]": FieldKind_NILLABLE_INT_ARRAY, - "[Int!]": FieldKind_INT_ARRAY, - "DateTime": FieldKind_NILLABLE_DATETIME, - "Float": FieldKind_NILLABLE_FLOAT, - "[Float]": FieldKind_NILLABLE_FLOAT_ARRAY, - "[Float!]": FieldKind_FLOAT_ARRAY, - "String": FieldKind_NILLABLE_STRING, - "[String]": FieldKind_NILLABLE_STRING_ARRAY, - "[String!]": FieldKind_STRING_ARRAY, - "Blob": FieldKind_NILLABLE_BLOB, - "JSON": FieldKind_NILLABLE_JSON, + "ID": FieldKind_DocID, + "Boolean": FieldKind_NILLABLE_BOOL, + "[Boolean]": FieldKind_NILLABLE_BOOL_ARRAY, + "[Boolean!]": FieldKind_BOOL_ARRAY, + "Int": FieldKind_NILLABLE_INT, + "[Int]": FieldKind_NILLABLE_INT_ARRAY, + "[Int!]": FieldKind_INT_ARRAY, + "DateTime": FieldKind_NILLABLE_DATETIME, + "Float": FieldKind_NILLABLE_FLOAT, + "[Float]": FieldKind_NILLABLE_FLOAT_ARRAY, + "[Float!]": FieldKind_FLOAT_ARRAY, + "String": FieldKind_NILLABLE_STRING, + "[String]": FieldKind_NILLABLE_STRING_ARRAY, + "[String!]": FieldKind_STRING_ARRAY, + "Blob": FieldKind_NILLABLE_BLOB, + "JSON": FieldKind_NILLABLE_JSON, + request.SelfTypeName: NewSelfKind("", false), + fmt.Sprintf("[%s]", request.SelfTypeName): NewSelfKind("", true), } // IsRelation returns true if this field is a relation. @@ -279,11 +383,40 @@ func (f *SchemaFieldDescription) UnmarshalJSON(bytes []byte) error { return nil } +// objectKind is a private type used to facilitate the unmarshalling +// of json to a [FieldKind]. +type objectKind struct { + Array bool + Root any + RelativeID string +} + func parseFieldKind(bytes json.RawMessage) (FieldKind, error) { if len(bytes) == 0 { return FieldKind_None, nil } + if bytes[0] == '{' { + var objKind objectKind + err := json.Unmarshal(bytes, &objKind) + if err != nil { + return nil, err + } + + if objKind.Root == nil { + return NewSelfKind(objKind.RelativeID, objKind.Array), nil + } + + switch root := objKind.Root.(type) { + case float64: + return NewCollectionKind(uint32(root), objKind.Array), nil + case string: + return NewSchemaKind(root, objKind.Array), nil + default: + return nil, NewErrFailedToParseKind(bytes) + } + } + if bytes[0] != '"' { // If the Kind is not represented by a string, assume try to parse it to an int, as // that is the only other type we support. @@ -313,12 +446,13 @@ func parseFieldKind(bytes json.RawMessage) (FieldKind, error) { return kind, nil } - // If we don't find the string representation of this type in the - // scalar mapping, assume it is an object - if it is not, validation - // will catch this later. If it is unknown we have no way of telling - // as to whether the user thought it was a scalar or an object anyway. - if strKind[0] == '[' { - return ObjectArrayKind(strings.Trim(strKind, "[]")), nil + isArray := strKind[0] == '[' + if isArray { + // Strip the brackets + strKind = strKind[1 : len(strKind)-1] } - return ObjectKind(strKind), nil + + // This is used by patch schema/collection, where new fields added + // by users will be initially added as [NamedKind]s. + return NewNamedKind(strKind, isArray), nil } diff --git a/internal/core/key.go b/internal/core/key.go index 8f0ab3fd4e..ecbe3fd0d7 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -47,6 +47,7 @@ const ( COLLECTION_ID = "/collection/id" COLLECTION_NAME = "/collection/name" COLLECTION_SCHEMA_VERSION = "/collection/version" + COLLECTION_ROOT = "/collection/root" COLLECTION_INDEX = "/collection/index" SCHEMA_VERSION = "/schema/version/v" SCHEMA_VERSION_ROOT = "/schema/version/r" @@ -142,6 +143,17 @@ type CollectionSchemaVersionKey struct { var _ Key = (*CollectionSchemaVersionKey)(nil) +// CollectionRootKey points to nil, but the keys/prefix can be used +// to get collections that are of a given RootID. +// +// It is stored in the format `/collection/root/[RootID]/[CollectionID]`. +type CollectionRootKey struct { + RootID uint32 + CollectionID uint32 +} + +var _ Key = (*CollectionRootKey)(nil) + // CollectionIndexKey to a stored description of an index type CollectionIndexKey struct { // CollectionID is the id of the collection that the index is on @@ -288,6 +300,37 @@ func NewCollectionSchemaVersionKeyFromString(key string) (CollectionSchemaVersio }, nil } +func NewCollectionRootKey(rootID uint32, collectionID uint32) CollectionRootKey { + return CollectionRootKey{ + RootID: rootID, + CollectionID: collectionID, + } +} + +// NewCollectionRootKeyFromString creates a new [CollectionRootKey]. +// +// It expects the key to be in the format `/collection/root/[RootID]/[CollectionID]`. +func NewCollectionRootKeyFromString(key string) (CollectionRootKey, error) { + keyArr := strings.Split(key, "/") + if len(keyArr) != 5 || keyArr[1] != COLLECTION || keyArr[2] != "root" { + return CollectionRootKey{}, ErrInvalidKey + } + rootID, err := strconv.Atoi(keyArr[3]) + if err != nil { + return CollectionRootKey{}, err + } + + collectionID, err := strconv.Atoi(keyArr[4]) + if err != nil { + return CollectionRootKey{}, err + } + + return CollectionRootKey{ + RootID: uint32(rootID), + CollectionID: uint32(collectionID), + }, nil +} + // NewCollectionIndexKey creates a new CollectionIndexKey from a collection name and index name. func NewCollectionIndexKey(colID immutable.Option[uint32], indexName string) CollectionIndexKey { return CollectionIndexKey{CollectionID: colID, IndexName: indexName} @@ -588,6 +631,28 @@ func (k CollectionSchemaVersionKey) ToDS() ds.Key { return ds.NewKey(k.ToString()) } +func (k CollectionRootKey) ToString() string { + result := COLLECTION_ROOT + + if k.RootID != 0 { + result = fmt.Sprintf("%s/%s", result, strconv.Itoa(int(k.RootID))) + } + + if k.CollectionID != 0 { + result = fmt.Sprintf("%s/%s", result, strconv.Itoa(int(k.CollectionID))) + } + + return result +} + +func (k CollectionRootKey) Bytes() []byte { + return []byte(k.ToString()) +} + +func (k CollectionRootKey) ToDS() ds.Key { + return ds.NewKey(k.ToString()) +} + func (k SchemaVersionKey) ToString() string { result := SCHEMA_VERSION diff --git a/internal/db/backup.go b/internal/db/backup.go index 1353376f34..e41a29178d 100644 --- a/internal/db/backup.go +++ b/internal/db/backup.go @@ -134,10 +134,12 @@ func (db *db) basicExport(ctx context.Context, config *client.BackupConfig) (err cols = append(cols, col) } } - colNameCache := map[string]struct{}{} + + definitions := make([]client.CollectionDefinition, 0, len(cols)) for _, col := range cols { - colNameCache[col.Name().Value()] = struct{}{} + definitions = append(definitions, col.Definition()) } + definitionCache := client.NewDefinitionCache(definitions) tempFile := config.Filepath + ".temp" f, err := os.Create(tempFile) @@ -213,9 +215,6 @@ func (db *db) basicExport(ctx context.Context, config *client.BackupConfig) (err // replace any foreign key if it needs to be changed for _, field := range col.Schema().Fields { if field.Kind.IsObject() && !field.Kind.IsArray() { - if _, ok := colNameCache[field.Kind.Underlying()]; !ok { - continue - } if foreignKey, err := doc.Get(field.Name + request.RelatedObjectID); err == nil { if newKey, ok := keyChangeCache[foreignKey.(string)]; ok { err := doc.Set(field.Name+request.RelatedObjectID, newKey) @@ -227,10 +226,14 @@ func (db *db) basicExport(ctx context.Context, config *client.BackupConfig) (err refFieldName = field.Name + request.RelatedObjectID } } else { - foreignCol, err := db.getCollectionByName(ctx, field.Kind.Underlying()) - if err != nil { - return NewErrFailedToGetCollection(field.Kind.Underlying(), err) + foreignDef, ok := client.GetDefinition(definitionCache, col.Definition(), field.Kind) + if !ok { + // If the collection is not in the cache the backup was not configured to + // handle this collection. + continue } + foreignCol := db.newCollection(foreignDef.Description, foreignDef.Schema) + foreignDocID, err := client.NewDocIDFromString(foreignKey.(string)) if err != nil { return err diff --git a/internal/db/collection.go b/internal/db/collection.go index 088e5075fa..c66eb6d4b9 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -129,6 +129,13 @@ func (db *db) getCollections( var cols []client.CollectionDescription switch { + case options.Root.HasValue(): + var err error + cols, err = description.GetCollectionsByRoot(ctx, txn, options.Root.Value()) + if err != nil { + return nil, err + } + case options.Name.HasValue(): col, err := description.GetCollectionByName(ctx, txn, options.Name.Value()) if err != nil { @@ -173,6 +180,13 @@ func (db *db) getCollections( continue } } + + if options.Root.HasValue() { + if col.RootID != options.Root.Value() { + continue + } + } + // By default, we don't return inactive collections unless a specific version is requested. if !options.IncludeInactive.Value() && !col.Name.HasValue() && !options.SchemaVersionID.HasValue() { continue @@ -745,11 +759,12 @@ func (c *collection) validateOneToOneLinkDoesntAlreadyExist( return nil } - otherCol, err := c.db.getCollectionByName(ctx, objFieldDescription.Kind.Underlying()) + otherCol, _, err := client.GetDefinitionUncached(ctx, c.db, c.Definition(), objFieldDescription.Kind) if err != nil { return err } - otherObjFieldDescription, _ := otherCol.Description().GetFieldByRelation( + + otherObjFieldDescription, _ := otherCol.Description.GetFieldByRelation( fieldDescription.RelationName, c.Name().Value(), objFieldDescription.Name, diff --git a/internal/db/collection_define.go b/internal/db/collection_define.go index a31165e314..357ab07d61 100644 --- a/internal/db/collection_define.go +++ b/internal/db/collection_define.go @@ -20,8 +20,6 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/client/request" - "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/db/description" ) @@ -36,69 +34,24 @@ func (db *db) createCollections( return nil, err } - txn := mustGetContextTxn(ctx) - + newSchemas := make([]client.SchemaDescription, len(newDefinitions)) for i, def := range newDefinitions { - schemaByName := map[string]client.SchemaDescription{} - for _, existingDefinition := range existingDefinitions { - schemaByName[existingDefinition.Schema.Name] = existingDefinition.Schema - } - for _, newDefinition := range newDefinitions { - schemaByName[newDefinition.Schema.Name] = newDefinition.Schema - } - - schema, err := description.CreateSchemaVersion(ctx, txn, def.Schema) - if err != nil { - return nil, err - } - newDefinitions[i].Description.SchemaVersionID = schema.VersionID - newDefinitions[i].Schema = schema + newSchemas[i] = def.Schema } - for i, def := range newDefinitions { - if len(def.Description.Fields) == 0 { - // This is a schema-only definition, we should not create a collection for it - continue - } - - colSeq, err := db.getSequence(ctx, core.CollectionIDSequenceKey{}) - if err != nil { - return nil, err - } - colID, err := colSeq.next(ctx) - if err != nil { - return nil, err - } - - fieldSeq, err := db.getSequence(ctx, core.NewFieldIDSequenceKey(uint32(colID))) - if err != nil { - return nil, err - } + err = setSchemaIDs(newSchemas) + if err != nil { + return nil, err + } - newDefinitions[i].Description.ID = uint32(colID) - newDefinitions[i].Description.RootID = newDefinitions[i].Description.ID - - for _, localField := range def.Description.Fields { - var fieldID uint64 - if localField.Name == request.DocIDFieldName { - // There is no hard technical requirement for this, we just think it looks nicer - // if the doc id is at the zero index. It makes it look a little nicer in commit - // queries too. - fieldID = 0 - } else { - fieldID, err = fieldSeq.next(ctx) - if err != nil { - return nil, err - } - } + for i := range newDefinitions { + newDefinitions[i].Description.SchemaVersionID = newSchemas[i].VersionID + newDefinitions[i].Schema = newSchemas[i] + } - for j := range def.Description.Fields { - if newDefinitions[i].Description.Fields[j].Name == localField.Name { - newDefinitions[i].Description.Fields[j].ID = client.FieldID(fieldID) - break - } - } - } + err = db.setCollectionIDs(ctx, newDefinitions) + if err != nil { + return nil, err } err = db.validateNewCollection( @@ -116,7 +69,14 @@ func (db *db) createCollections( return nil, err } + txn := mustGetContextTxn(ctx) + for _, def := range newDefinitions { + _, err := description.CreateSchemaVersion(ctx, txn, def.Schema) + if err != nil { + return nil, err + } + if len(def.Description.Fields) == 0 { // This is a schema-only definition, we should not create a collection for it returnDescriptions = append(returnDescriptions, def) diff --git a/internal/db/collection_id.go b/internal/db/collection_id.go new file mode 100644 index 0000000000..e635a4477f --- /dev/null +++ b/internal/db/collection_id.go @@ -0,0 +1,127 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package db + +import ( + "context" + + "github.com/sourcenetwork/immutable" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/client/request" + "github.com/sourcenetwork/defradb/internal/core" +) + +// setCollectionIDs sets the IDs on a collection description, including field IDs, mutating the input set. +func (db *db) setCollectionIDs(ctx context.Context, newCollections []client.CollectionDefinition) error { + err := db.setCollectionID(ctx, newCollections) + if err != nil { + return err + } + + return db.setFieldIDs(ctx, newCollections) +} + +// setCollectionID sets the IDs directly on a collection description, excluding stuff like field IDs, +// mutating the input set. +func (db *db) setCollectionID(ctx context.Context, newCollections []client.CollectionDefinition) error { + colSeq, err := db.getSequence(ctx, core.CollectionIDSequenceKey{}) + if err != nil { + return err + } + + for i := range newCollections { + if len(newCollections[i].Description.Fields) == 0 { + // This is a schema-only definition, we should not create a collection for it + continue + } + + colID, err := colSeq.next(ctx) + if err != nil { + return err + } + + // Unlike schema, collections can be mutated and thus we need to make sure this function + // does not assign new IDs to existing collections. + if newCollections[i].Description.ID == 0 { + newCollections[i].Description.ID = uint32(colID) + } + + if newCollections[i].Description.RootID == 0 { + newCollections[i].Description.RootID = uint32(colID) + } + } + + return nil +} + +// setFieldIDs sets the field IDs hosted on the given collections, mutating the input set. +func (db *db) setFieldIDs(ctx context.Context, definitions []client.CollectionDefinition) error { + collectionsByName := map[string]client.CollectionDescription{} + schemasByName := map[string]client.SchemaDescription{} + for _, def := range definitions { + if def.Description.Name.HasValue() { + collectionsByName[def.Description.Name.Value()] = def.Description + } + schemasByName[def.Schema.Name] = def.Schema + } + + for i := range definitions { + fieldSeq, err := db.getSequence(ctx, core.NewFieldIDSequenceKey(definitions[i].Description.RootID)) + if err != nil { + return err + } + + for j := range definitions[i].Description.Fields { + var fieldID client.FieldID + if definitions[i].Description.Fields[j].ID != client.FieldID(0) { + fieldID = definitions[i].Description.Fields[j].ID + } else if definitions[i].Description.Fields[j].Name == request.DocIDFieldName { + // There is no hard technical requirement for this, we just think it looks nicer + // if the doc id is at the zero index. It makes it look a little nicer in commit + // queries too. + fieldID = 0 + } else { + nextID, err := fieldSeq.next(ctx) + if err != nil { + return err + } + fieldID = client.FieldID(nextID) + } + + if definitions[i].Description.Fields[j].Kind.HasValue() { + switch kind := definitions[i].Description.Fields[j].Kind.Value().(type) { + case *client.NamedKind: + var newKind client.FieldKind + if kind.Name == definitions[i].Description.Name.Value() { + newKind = client.NewSelfKind("", kind.IsArray()) + } else if otherCol, ok := collectionsByName[kind.Name]; ok { + newKind = client.NewCollectionKind(otherCol.RootID, kind.IsArray()) + } else if otherSchema, ok := schemasByName[kind.Name]; ok { + newKind = client.NewSchemaKind(otherSchema.Root, kind.IsArray()) + } else { + // Continue, and let the validation stage return user friendly errors + // if appropriate + continue + } + + definitions[i].Description.Fields[j].Kind = immutable.Some(newKind) + default: + // no-op + } + } + + definitions[i].Description.Fields[j].ID = fieldID + } + } + + return nil +} diff --git a/internal/db/collection_update.go b/internal/db/collection_update.go index d29e562977..8a4073b60f 100644 --- a/internal/db/collection_update.go +++ b/internal/db/collection_update.go @@ -151,12 +151,12 @@ func (c *collection) patchPrimaryDoc( return err } - primaryCol, err := c.db.getCollectionByName(ctx, relationFieldDescription.Kind.Underlying()) + primaryDef, _, err := client.GetDefinitionUncached(ctx, c.db, c.Definition(), relationFieldDescription.Kind) if err != nil { return err } - primaryField, ok := primaryCol.Description().GetFieldByRelation( + primaryField, ok := primaryDef.Description.GetFieldByRelation( relationFieldDescription.RelationName, secondaryCollectionName, relationFieldDescription.Name, @@ -165,11 +165,12 @@ func (c *collection) patchPrimaryDoc( return client.NewErrFieldNotExist(relationFieldDescription.RelationName) } - primaryIDField, ok := primaryCol.Definition().GetFieldByName(primaryField.Name + request.RelatedObjectID) + primaryIDField, ok := primaryDef.GetFieldByName(primaryField.Name + request.RelatedObjectID) if !ok { return client.NewErrFieldNotExist(primaryField.Name + request.RelatedObjectID) } + primaryCol := c.db.newCollection(primaryDef.Description, primaryDef.Schema) doc, err := primaryCol.Get( ctx, primaryDocID, @@ -185,8 +186,7 @@ func (c *collection) patchPrimaryDoc( return nil } - pc := c.db.newCollection(primaryCol.Description(), primaryCol.Schema()) - err = pc.validateOneToOneLinkDoesntAlreadyExist( + err = primaryCol.validateOneToOneLinkDoesntAlreadyExist( ctx, primaryDocID.String(), primaryIDField, diff --git a/internal/db/definition_validation.go b/internal/db/definition_validation.go index 9e28624982..69ec8402c9 100644 --- a/internal/db/definition_validation.go +++ b/internal/db/definition_validation.go @@ -30,44 +30,62 @@ type definitionState struct { schemaByName map[string]client.SchemaDescription definitionsByName map[string]client.CollectionDefinition + definitionCache client.DefinitionCache } -// newDefinitionState creates a new definitionState object given the provided +// newColOnlyDefinitionState creates a new definitionState object given the provided // descriptions. -func newDefinitionState( +func newColOnlyDefinitionState( collections []client.CollectionDescription, - schemasByID map[string]client.SchemaDescription, ) *definitionState { collectionsByID := map[uint32]client.CollectionDescription{} definitionsByName := map[string]client.CollectionDefinition{} + definitions := []client.CollectionDefinition{} schemaByName := map[string]client.SchemaDescription{} - schemaVersionsAdded := map[string]struct{}{} for _, col := range collections { if len(col.Fields) == 0 { continue } - schema := schemasByID[col.SchemaVersionID] definition := client.CollectionDefinition{ Description: col, - Schema: schema, } definitionsByName[definition.GetName()] = definition - schemaVersionsAdded[schema.VersionID] = struct{}{} + definitions = append(definitions, definition) collectionsByID[col.ID] = col } - for _, schema := range schemasByID { - schemaByName[schema.Name] = schema + return &definitionState{ + collections: collections, + collectionsByID: collectionsByID, + schemaByID: map[string]client.SchemaDescription{}, + schemaByName: schemaByName, + definitionsByName: definitionsByName, + definitionCache: client.NewDefinitionCache(definitions), + } +} - if _, ok := schemaVersionsAdded[schema.VersionID]; ok { - continue - } +// newDefinitionState creates a new definitionState object given the provided +// descriptions. +func newDefinitionState( + definitions []client.CollectionDefinition, +) *definitionState { + collectionsByID := map[uint32]client.CollectionDescription{} + schemasByID := map[string]client.SchemaDescription{} + definitionsByName := map[string]client.CollectionDefinition{} + collections := []client.CollectionDescription{} + schemaByName := map[string]client.SchemaDescription{} - definitionsByName[schema.Name] = client.CollectionDefinition{ - Schema: schema, + for _, def := range definitions { + definitionsByName[def.GetName()] = def + schemasByID[def.Schema.VersionID] = def.Schema + schemaByName[def.Schema.Name] = def.Schema + + if len(def.Description.Fields) != 0 { + collectionsByID[def.Description.ID] = def.Description + collections = append(collections, def.Description) } } @@ -77,6 +95,7 @@ func newDefinitionState( schemaByID: schemasByID, schemaByName: schemaByName, definitionsByName: definitionsByName, + definitionCache: client.NewDefinitionCache(definitions), } } @@ -93,7 +112,7 @@ type definitionValidator = func( // they will not be executed for updates to existing records. var createOnlyValidators = []definitionValidator{} -// createOnlyValidators are executed on the update of existing descriptions only +// updateOnlyValidators are executed on the update of existing descriptions only // they will not be executed for new records. var updateOnlyValidators = []definitionValidator{ validateSourcesNotRedefined, @@ -102,10 +121,7 @@ var updateOnlyValidators = []definitionValidator{ validatePolicyNotModified, validateIDNotZero, validateIDUnique, - validateIDExists, validateRootIDNotMutated, - validateSchemaVersionIDNotMutated, - validateCollectionNotRemoved, validateSingleVersionActive, validateSchemaNotAdded, validateSchemaFieldNotDeleted, @@ -113,6 +129,27 @@ var updateOnlyValidators = []definitionValidator{ validateFieldNotMoved, } +var schemaUpdateValidators = append( + append( + []definitionValidator{}, + updateOnlyValidators..., + ), + globalValidators..., +) + +var collectionUpdateValidators = append( + append( + append( + []definitionValidator{}, + updateOnlyValidators..., + ), + validateIDExists, + validateSchemaVersionIDNotMutated, + validateCollectionNotRemoved, + ), + globalValidators..., +) + // globalValidators are run on create and update of records. var globalValidators = []definitionValidator{ validateCollectionNameUnique, @@ -126,32 +163,23 @@ var globalValidators = []definitionValidator{ validateTypeSupported, validateTypeAndKindCompatible, validateFieldNotDuplicated, + validateSelfReferences, } -var updateValidators = append( - append([]definitionValidator{}, updateOnlyValidators...), - globalValidators..., -) - var createValidators = append( append([]definitionValidator{}, createOnlyValidators...), globalValidators..., ) -func (db *db) validateCollectionChanges( +func (db *db) validateSchemaUpdate( ctx context.Context, - oldCols []client.CollectionDescription, - newColsByID map[uint32]client.CollectionDescription, + oldDefinitions []client.CollectionDefinition, + newDefinitions []client.CollectionDefinition, ) error { - newCols := make([]client.CollectionDescription, 0, len(newColsByID)) - for _, col := range newColsByID { - newCols = append(newCols, col) - } - - newState := newDefinitionState(newCols, map[string]client.SchemaDescription{}) - oldState := newDefinitionState(oldCols, map[string]client.SchemaDescription{}) + newState := newDefinitionState(newDefinitions) + oldState := newDefinitionState(oldDefinitions) - for _, validator := range updateValidators { + for _, validator := range schemaUpdateValidators { err := validator(ctx, db, newState, oldState) if err != nil { return err @@ -161,38 +189,20 @@ func (db *db) validateCollectionChanges( return nil } -func (db *db) validateNewCollection( +func (db *db) validateCollectionChanges( ctx context.Context, - newDefinitions []client.CollectionDefinition, - oldDefinitions []client.CollectionDefinition, + oldCols []client.CollectionDescription, + newColsByID map[uint32]client.CollectionDescription, ) error { - newCollections := []client.CollectionDescription{} - newSchemasByID := map[string]client.SchemaDescription{} - - for _, def := range newDefinitions { - if len(def.Description.Fields) != 0 { - newCollections = append(newCollections, def.Description) - } - - newSchemasByID[def.Schema.VersionID] = def.Schema - } - - newState := newDefinitionState(newCollections, newSchemasByID) - - oldCollections := []client.CollectionDescription{} - oldSchemasByID := map[string]client.SchemaDescription{} - - for _, def := range oldDefinitions { - if len(def.Description.Fields) != 0 { - oldCollections = append(oldCollections, def.Description) - } - - oldSchemasByID[def.Schema.VersionID] = def.Schema + newCols := make([]client.CollectionDescription, 0, len(newColsByID)) + for _, col := range newColsByID { + newCols = append(newCols, col) } - oldState := newDefinitionState(oldCollections, oldSchemasByID) + newState := newColOnlyDefinitionState(newCols) + oldState := newColOnlyDefinitionState(oldCols) - for _, validator := range createValidators { + for _, validator := range collectionUpdateValidators { err := validator(ctx, db, newState, oldState) if err != nil { return err @@ -202,24 +212,15 @@ func (db *db) validateNewCollection( return nil } -func (db *db) validateSchemaUpdate( +func (db *db) validateNewCollection( ctx context.Context, - newSchemaByName map[string]client.SchemaDescription, - oldSchemaByName map[string]client.SchemaDescription, + newDefinitions []client.CollectionDefinition, + oldDefinitions []client.CollectionDefinition, ) error { - newSchemaByID := make(map[string]client.SchemaDescription, len(newSchemaByName)) - oldSchemaByID := make(map[string]client.SchemaDescription, len(oldSchemaByName)) - for _, schema := range newSchemaByName { - newSchemaByID[schema.VersionID] = schema - } - for _, schema := range oldSchemaByName { - oldSchemaByID[schema.VersionID] = schema - } + newState := newDefinitionState(newDefinitions) + oldState := newDefinitionState(oldDefinitions) - newState := newDefinitionState([]client.CollectionDescription{}, newSchemaByID) - oldState := newDefinitionState([]client.CollectionDescription{}, oldSchemaByID) - - for _, validator := range updateValidators { + for _, validator := range createValidators { err := validator(ctx, db, newState, oldState) if err != nil { return err @@ -245,10 +246,10 @@ func validateRelationPointsToValidKind( continue } - underlying := field.Kind.Value().Underlying() - _, ok := newState.definitionsByName[underlying] + definition := newState.definitionsByName[newCollection.Name.Value()] + _, ok := client.GetDefinition(newState.definitionCache, definition, field.Kind.Value()) if !ok { - return NewErrFieldKindNotFound(field.Name, underlying) + return NewErrFieldKindNotFound(field.Name, field.Kind.Value().String()) } } } @@ -259,10 +260,9 @@ func validateRelationPointsToValidKind( continue } - underlying := field.Kind.Underlying() - _, ok := newState.definitionsByName[underlying] + _, ok := client.GetDefinition(newState.definitionCache, client.CollectionDefinition{Schema: schema}, field.Kind) if !ok { - return NewErrFieldKindNotFound(field.Name, underlying) + return NewErrFieldKindNotFound(field.Name, field.Kind.String()) } } } @@ -305,8 +305,7 @@ func validateSecondaryFieldsPairUp( continue } - underlying := field.Kind.Value().Underlying() - otherDef, ok := newState.definitionsByName[underlying] + otherDef, ok := client.GetDefinition(newState.definitionCache, definition, field.Kind.Value()) if !ok { continue } @@ -322,13 +321,13 @@ func validateSecondaryFieldsPairUp( field.Name, ) if !ok { - return NewErrRelationMissingField(underlying, field.RelationName.Value()) + return NewErrRelationMissingField(otherDef.GetName(), field.RelationName.Value()) } _, ok = otherDef.Schema.GetFieldByName(otherField.Name) if !ok { // This secondary is paired with another secondary, which is invalid - return NewErrRelationMissingField(underlying, field.RelationName.Value()) + return NewErrRelationMissingField(otherDef.GetName(), field.RelationName.Value()) } } } @@ -367,8 +366,7 @@ func validateSingleSidePrimary( continue } - underlying := field.Kind.Underlying() - otherDef, ok := newState.definitionsByName[underlying] + otherDef, ok := client.GetDefinition(newState.definitionCache, definition, field.Kind) if !ok { continue } @@ -609,7 +607,11 @@ func validateRootIDNotMutated( } for _, newSchema := range newState.schemaByName { - oldSchema := oldState.schemaByName[newSchema.Name] + oldSchema, ok := oldState.schemaByName[newSchema.Name] + if !ok { + continue + } + if newSchema.Root != oldSchema.Root { return NewErrSchemaRootDoesntMatch( newSchema.Name, @@ -843,6 +845,58 @@ func validateFieldNotDuplicated( return nil } +func validateSelfReferences( + ctx context.Context, + db *db, + newState *definitionState, + oldState *definitionState, +) error { + for _, schema := range newState.schemaByName { + for _, field := range schema.Fields { + if _, ok := field.Kind.(*client.SelfKind); ok { + continue + } + + otherDef, ok := client.GetDefinition( + newState.definitionCache, + client.CollectionDefinition{Schema: schema}, + field.Kind, + ) + if !ok { + continue + } + + if otherDef.Schema.Root == schema.Root { + return NewErrSelfReferenceWithoutSelf(field.Name) + } + } + } + + for _, col := range newState.collections { + for _, field := range col.Fields { + if !field.Kind.HasValue() { + continue + } + + if _, ok := field.Kind.Value().(*client.SelfKind); ok { + continue + } + + definition := newState.definitionsByName[col.Name.Value()] + otherDef, ok := client.GetDefinition(newState.definitionCache, definition, field.Kind.Value()) + if !ok { + continue + } + + if otherDef.Description.RootID == col.RootID { + return NewErrSelfReferenceWithoutSelf(field.Name) + } + } + } + + return nil +} + func validateSecondaryNotOnSchema( ctx context.Context, db *db, @@ -896,6 +950,12 @@ func validateSchemaNotAdded( oldState *definitionState, ) error { for _, newSchema := range newState.schemaByName { + if newSchema.Name == "" { + // continue, and allow a more appropriate rule to return a nicer error + // for the user + continue + } + if _, exists := oldState.schemaByName[newSchema.Name]; !exists { return NewErrAddSchemaWithPatch(newSchema.Name) } diff --git a/internal/db/description/collection.go b/internal/db/description/collection.go index 90ef594a39..20f652888e 100644 --- a/internal/db/description/collection.go +++ b/internal/db/description/collection.go @@ -97,6 +97,12 @@ func SaveCollection( return client.CollectionDescription{}, err } + rootKey := core.NewCollectionRootKey(desc.RootID, desc.ID) + err = txn.Systemstore().Put(ctx, rootKey.ToDS(), []byte{}) + if err != nil { + return client.CollectionDescription{}, err + } + return desc, nil } @@ -143,6 +149,49 @@ func GetCollectionByName( return GetCollectionByID(ctx, txn, id) } +func GetCollectionsByRoot( + ctx context.Context, + txn datastore.Txn, + root uint32, +) ([]client.CollectionDescription, error) { + rootKey := core.NewCollectionRootKey(root, 0) + + rootQuery, err := txn.Systemstore().Query(ctx, query.Query{ + Prefix: rootKey.ToString(), + KeysOnly: true, + }) + if err != nil { + return nil, NewErrFailedToCreateCollectionQuery(err) + } + + cols := []client.CollectionDescription{} + for res := range rootQuery.Next() { + if res.Error != nil { + if err := rootQuery.Close(); err != nil { + return nil, NewErrFailedToCloseSchemaQuery(err) + } + return nil, err + } + + rootKey, err := core.NewCollectionRootKeyFromString(string(res.Key)) + if err != nil { + if err := rootQuery.Close(); err != nil { + return nil, NewErrFailedToCloseSchemaQuery(err) + } + return nil, err + } + + col, err := GetCollectionByID(ctx, txn, rootKey.CollectionID) + if err != nil { + return nil, err + } + + cols = append(cols, col) + } + + return cols, nil +} + // GetCollectionsBySchemaVersionID returns all collections that use the given // schemaVersionID. // diff --git a/internal/db/description/schema.go b/internal/db/description/schema.go index 6f5a782ec7..f9d5935770 100644 --- a/internal/db/description/schema.go +++ b/internal/db/description/schema.go @@ -19,7 +19,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/core" - "github.com/sourcenetwork/defradb/internal/core/cid" ) // CreateSchemaVersion creates and saves to the store a new schema version. @@ -35,31 +34,13 @@ func CreateSchemaVersion( return client.SchemaDescription{}, err } - scid, err := cid.NewSHA256CidV1(buf) - if err != nil { - return client.SchemaDescription{}, err - } - versionID := scid.String() - isNew := desc.Root == "" - - desc.VersionID = versionID - if isNew { - // If this is a new schema, the Root will match the version ID - desc.Root = versionID - } - - // Rebuild the json buffer to include the newly set ID properties - buf, err = json.Marshal(desc) - if err != nil { - return client.SchemaDescription{}, err - } - - key := core.NewSchemaVersionKey(versionID) + key := core.NewSchemaVersionKey(desc.VersionID) err = txn.Systemstore().Put(ctx, key.ToDS(), buf) if err != nil { return client.SchemaDescription{}, err } + isNew := desc.Root == desc.VersionID if !isNew { // We don't need to add a root key if this is the first version schemaVersionHistoryKey := core.NewSchemaRootKey(desc.Root, desc.VersionID) diff --git a/internal/db/errors.go b/internal/db/errors.go index e8a835f3f2..71f7978a1b 100644 --- a/internal/db/errors.go +++ b/internal/db/errors.go @@ -101,6 +101,7 @@ const ( errReplicatorCollections string = "failed to get collections for replicator" errReplicatorNotFound string = "replicator not found" errCanNotEncryptBuiltinField string = "can not encrypt build-in field" + errSelfReferenceWithoutSelf string = "must specify 'Self' kind for self referencing relations" ) var ( @@ -141,6 +142,7 @@ var ( ErrReplicatorCollections = errors.New(errReplicatorCollections) ErrReplicatorNotFound = errors.New(errReplicatorNotFound) ErrCanNotEncryptBuiltinField = errors.New(errCanNotEncryptBuiltinField) + ErrSelfReferenceWithoutSelf = errors.New(errSelfReferenceWithoutSelf) ) // NewErrFailedToGetHeads returns a new error indicating that the heads of a document @@ -650,3 +652,10 @@ func NewErrReplicatorDocID(inner error, kv ...errors.KV) error { func NewErrReplicatorCollections(inner error, kv ...errors.KV) error { return errors.Wrap(errReplicatorCollections, inner, kv...) } + +func NewErrSelfReferenceWithoutSelf(fieldName string) error { + return errors.New( + errSelfReferenceWithoutSelf, + errors.NewKV("Field", fieldName), + ) +} diff --git a/internal/db/schema.go b/internal/db/schema.go index c9a2875d27..d9b9a4055c 100644 --- a/internal/db/schema.go +++ b/internal/db/schema.go @@ -23,7 +23,6 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/db/description" ) @@ -131,18 +130,15 @@ func (db *db) patchSchema( return err } - for _, schema := range newSchemaByName { - err := db.updateSchema( - ctx, - existingSchemaByName, - newSchemaByName, - schema, - migration, - setAsDefaultVersion, - ) - if err != nil { - return err - } + err = db.updateSchema( + ctx, + existingSchemaByName, + newSchemaByName, + migration, + setAsDefaultVersion, + ) + if err != nil { + return err } return db.loadSchema(ctx) @@ -332,192 +328,202 @@ func (db *db) updateSchema( ctx context.Context, existingSchemaByName map[string]client.SchemaDescription, proposedDescriptionsByName map[string]client.SchemaDescription, - schema client.SchemaDescription, migration immutable.Option[model.Lens], setAsActiveVersion bool, ) error { - previousSchema := existingSchemaByName[schema.Name] + newSchemas := []client.SchemaDescription{} + for _, schema := range proposedDescriptionsByName { + previousSchema := existingSchemaByName[schema.Name] - areEqual := areSchemasEqual(schema, previousSchema) - if areEqual { - return nil - } - - err := db.validateSchemaUpdate(ctx, proposedDescriptionsByName, existingSchemaByName) - if err != nil { - return err - } + previousFieldNames := make(map[string]struct{}, len(previousSchema.Fields)) + for _, field := range previousSchema.Fields { + previousFieldNames[field.Name] = struct{}{} + } - for _, field := range schema.Fields { - if field.Kind.IsObject() && !field.Kind.IsArray() { - idFieldName := field.Name + "_id" - if _, ok := schema.GetFieldByName(idFieldName); !ok { - schema.Fields = append(schema.Fields, client.SchemaFieldDescription{ - Name: idFieldName, - Kind: client.FieldKind_DocID, - }) + for i, field := range schema.Fields { + if _, existed := previousFieldNames[field.Name]; !existed && field.Typ == client.NONE_CRDT { + // If no CRDT Type has been provided, default to LWW_REGISTER. + schema.Fields[i].Typ = client.LWW_REGISTER } } - } - previousFieldNames := make(map[string]struct{}, len(previousSchema.Fields)) - for _, field := range previousSchema.Fields { - previousFieldNames[field.Name] = struct{}{} - } - - for i, field := range schema.Fields { - if _, existed := previousFieldNames[field.Name]; !existed && field.Typ == client.NONE_CRDT { - // If no CRDT Type has been provided, default to LWW_REGISTER. - field.Typ = client.LWW_REGISTER - schema.Fields[i] = field + for _, field := range schema.Fields { + if field.Kind.IsObject() && !field.Kind.IsArray() { + idFieldName := field.Name + "_id" + if _, ok := schema.GetFieldByName(idFieldName); !ok { + schema.Fields = append(schema.Fields, client.SchemaFieldDescription{ + Name: idFieldName, + Kind: client.FieldKind_DocID, + }) + } + } } - } - txn := mustGetContextTxn(ctx) - previousVersionID := schema.VersionID - schema, err = description.CreateSchemaVersion(ctx, txn, schema) - if err != nil { - return err + newSchemas = append(newSchemas, schema) } - // After creating the new schema version, we need to create new collection versions for - // any collection using the previous version. These will be inactive unless [setAsActiveVersion] - // is true. - - cols, err := description.GetCollectionsBySchemaVersionID(ctx, txn, previousVersionID) + err := setSchemaIDs(newSchemas) if err != nil { return err } - existingCols, err := description.GetCollectionsBySchemaVersionID(ctx, txn, schema.VersionID) - if err != nil { - return err + for _, schema := range newSchemas { + proposedDescriptionsByName[schema.Name] = schema } - colSeq, err := db.getSequence(ctx, core.CollectionIDSequenceKey{}) - if err != nil { - return err - } + for _, schema := range proposedDescriptionsByName { + previousSchema := existingSchemaByName[schema.Name] - for _, col := range cols { - previousID := col.ID - - // The collection version may exist before the schema version was created locally. This is - // because migrations for the globally known schema version may have been registered locally - // (typically to handle documents synced over P2P at higher versions) before the local schema - // was updated. We need to check for them now, and update them instead of creating new ones - // if they exist. - var isExistingCol bool - existingColLoop: - for _, existingCol := range existingCols { - sources := existingCol.CollectionSources() - for _, source := range sources { - // Make sure that this collection is the parent of the current [col], and not part of - // another collection set that happens to be using the same schema. - if source.SourceCollectionID == previousID { - if existingCol.RootID == client.OrphanRootID { - existingCol.RootID = col.RootID - } + areEqual := areSchemasEqual(schema, previousSchema) + if areEqual { + continue + } - fieldSeq, err := db.getSequence(ctx, core.NewFieldIDSequenceKey(existingCol.RootID)) - if err != nil { - return err - } + txn := mustGetContextTxn(ctx) + schema, err = description.CreateSchemaVersion(ctx, txn, schema) + if err != nil { + return err + } - for _, globalField := range schema.Fields { - var fieldID client.FieldID - // We must check the source collection if the field already exists, and take its ID - // from there, otherwise the field must be generated by the sequence. - existingField, ok := col.GetFieldByName(globalField.Name) - if ok { - fieldID = existingField.ID - } else { - nextFieldID, err := fieldSeq.next(ctx) - if err != nil { - return err + // After creating the new schema version, we need to create new collection versions for + // any collection using the previous version. These will be inactive unless [setAsActiveVersion] + // is true. + + previousVersionID := existingSchemaByName[schema.Name].VersionID + cols, err := description.GetCollectionsBySchemaVersionID(ctx, txn, previousVersionID) + if err != nil { + return err + } + + existingCols, err := description.GetCollectionsBySchemaVersionID(ctx, txn, schema.VersionID) + if err != nil { + return err + } + + definitions := make([]client.CollectionDefinition, 0, len(cols)) + + for _, col := range cols { + previousID := col.ID + + // The collection version may exist before the schema version was created locally. This is + // because migrations for the globally known schema version may have been registered locally + // (typically to handle documents synced over P2P at higher versions) before the local schema + // was updated. We need to check for them now, and update them instead of creating new ones + // if they exist. + var isExistingCol bool + existingColLoop: + for _, existingCol := range existingCols { + sources := existingCol.CollectionSources() + for _, source := range sources { + // Make sure that this collection is the parent of the current [col], and not part of + // another collection set that happens to be using the same schema. + if source.SourceCollectionID == previousID { + if existingCol.RootID == client.OrphanRootID { + existingCol.RootID = col.RootID + } + + for _, globalField := range schema.Fields { + var fieldID client.FieldID + // We must check the source collection if the field already exists, and take its ID + // from there, otherwise the field must be generated by the sequence. + existingField, ok := col.GetFieldByName(globalField.Name) + if ok { + fieldID = existingField.ID } - fieldID = client.FieldID(nextFieldID) + + existingCol.Fields = append( + existingCol.Fields, + client.CollectionFieldDescription{ + Name: globalField.Name, + ID: fieldID, + }, + ) } - existingCol.Fields = append( - existingCol.Fields, + definitions = append(definitions, client.CollectionDefinition{ + Description: existingCol, + Schema: schema, + }) + + isExistingCol = true + break existingColLoop + } + } + } + + if !isExistingCol { + // Create any new collections without a name (inactive), if [setAsActiveVersion] is true + // they will be activated later along with any existing collection versions. + col.ID = 0 + col.Name = immutable.None[string]() + col.SchemaVersionID = schema.VersionID + col.Sources = []any{ + &client.CollectionSource{ + SourceCollectionID: previousID, + Transform: migration, + }, + } + + for _, globalField := range schema.Fields { + _, exists := col.GetFieldByName(globalField.Name) + if !exists { + col.Fields = append( + col.Fields, client.CollectionFieldDescription{ Name: globalField.Name, - ID: fieldID, }, ) } - existingCol, err = description.SaveCollection(ctx, txn, existingCol) - if err != nil { - return err - } - isExistingCol = true - break existingColLoop } - } - } - if !isExistingCol { - colID, err := colSeq.next(ctx) - if err != nil { - return err + definitions = append(definitions, client.CollectionDefinition{ + Description: col, + Schema: schema, + }) } + } - fieldSeq, err := db.getSequence(ctx, core.NewFieldIDSequenceKey(col.RootID)) - if err != nil { - return err - } + err = db.setCollectionIDs(ctx, definitions) + if err != nil { + return err + } - // Create any new collections without a name (inactive), if [setAsActiveVersion] is true - // they will be activated later along with any existing collection versions. - col.Name = immutable.None[string]() - col.ID = uint32(colID) - col.SchemaVersionID = schema.VersionID - col.Sources = []any{ - &client.CollectionSource{ - SourceCollectionID: previousID, - Transform: migration, - }, - } + allExistingCols, err := db.getCollections(ctx, client.CollectionFetchOptions{}) + if err != nil { + return err + } - for _, globalField := range schema.Fields { - _, exists := col.GetFieldByName(globalField.Name) - if !exists { - fieldID, err := fieldSeq.next(ctx) - if err != nil { - return err - } + oldDefs := make([]client.CollectionDefinition, 0, len(allExistingCols)) + for _, col := range allExistingCols { + oldDefs = append(oldDefs, col.Definition()) + } - col.Fields = append( - col.Fields, - client.CollectionFieldDescription{ - Name: globalField.Name, - ID: client.FieldID(fieldID), - }, - ) - } - } + err = db.validateSchemaUpdate(ctx, oldDefs, definitions) + if err != nil { + return err + } - _, err = description.SaveCollection(ctx, txn, col) + for _, def := range definitions { + _, err = description.SaveCollection(ctx, txn, def.Description) if err != nil { return err } if migration.HasValue() { - err = db.LensRegistry().SetMigration(ctx, col.ID, migration.Value()) + err = db.LensRegistry().SetMigration(ctx, def.Description.ID, migration.Value()) if err != nil { return err } } } - } - if setAsActiveVersion { - // activate collection versions using the new schema ID. This call must be made after - // all new collection versions have been saved. - err = db.setActiveSchemaVersion(ctx, schema.VersionID) - if err != nil { - return err + if setAsActiveVersion { + // activate collection versions using the new schema ID. This call must be made after + // all new collection versions have been saved. + err = db.setActiveSchemaVersion(ctx, schema.VersionID) + if err != nil { + return err + } } } @@ -536,6 +542,5 @@ func areSchemasEqual(this client.SchemaDescription, that client.SchemaDescriptio } return this.Name == that.Name && - this.Root == that.Root && - this.VersionID == that.VersionID + this.Root == that.Root } diff --git a/internal/db/schema_id.go b/internal/db/schema_id.go new file mode 100644 index 0000000000..094774cbeb --- /dev/null +++ b/internal/db/schema_id.go @@ -0,0 +1,393 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package db + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/internal/core/cid" +) + +const schemaSetDeliminator string = "-" + +// setSchemaIDs sets all ID fields on a schema description, mutating the input parameter. +// +// This includes RootID (if not already set), VersionID, and relational fields. +func setSchemaIDs(newSchemas []client.SchemaDescription) error { + // We need to group the inputs and then mutate them, so we temporarily + // map them to pointers. + newSchemaPtrs := make([]*client.SchemaDescription, len(newSchemas)) + for i := range newSchemas { + schema := newSchemas[i] + newSchemaPtrs[i] = &schema + } + + schemaSets := getSchemaSets(newSchemaPtrs) + + for _, schemaSet := range schemaSets { + setID, err := generateSetID(schemaSet) + if err != nil { + return err + } + + setIDs(setID, schemaSet) + } + + for i := range newSchemaPtrs { + newSchemas[i] = *newSchemaPtrs[i] + } + + substituteRelationFieldKinds(newSchemas) + + return nil +} + +// schemaRelations is a trimmed down [client.SchemaDescription] containing +// only the useful information to the functions in this file. +type schemaRelations struct { + // The name of this schema + name string + + // The schema names of the primary relations from this schema. + relations []string +} + +// getSchemaSets groups schemas into sets. +// +// Most sets will contain a single schema, however if a circular dependency chain is found +// all elements within that chain will be grouped together into a single set. +// +// For example if User contains a relation *to* Dog, and Dog contains a relationship *to* +// User, they will be grouped into the same set. +func getSchemaSets(newSchemas []*client.SchemaDescription) [][]*client.SchemaDescription { + schemasWithRelations := map[string]schemaRelations{} + for _, schema := range newSchemas { + relations := []string{} + for _, field := range schema.Fields { + switch kind := field.Kind.(type) { + case *client.NamedKind: + // We only need to worry about use provided `NamedKind` relations in this scope. + // Other relation kinds can either not be circular, or are relative to the host. + relations = append(relations, kind.Name) + default: + // no-op + } + } + + if len(relations) == 0 { + // If a schema is defined with no relations, then it is not relevant to this function + // and can be skipped. + continue + } + + schemasWithRelations[schema.Name] = schemaRelations{ + name: schema.Name, + relations: relations, + } + } + + changedInLoop := true + for changedInLoop { + // This loop strips out schemas from `schemasWithRelations` that do not form circular + // schema sets (e.g. User=>Dog=>User). This allows later logic that figures out the + // exact path that circles forms to operate on a minimal set of data, reducing its cost + // and complexity. + // + // Some non circular relations may still remain after this first pass, for example + // one-directional relations between two circles. + changedInLoop = false + for _, schema := range schemasWithRelations { + i := 0 + relation := "" + deleteI := false + for i, relation = range schema.relations { + if _, ok := schemasWithRelations[relation]; !ok { + // If the related schema is not in `schemasWithRelations` it must have been removed + // in a previous iteration of the schemasWithRelations loop, this will have been + // done because it had no relevant remaining relations and thus could not be part + // of a circular schema set. If this is the case, this `relation` is also irrelevant + // here and can be removed as it too cannot form part of a circular schema set. + changedInLoop = true + deleteI = true + break + } + } + + if deleteI { + old := schema.relations + schema.relations = make([]string, len(schema.relations)-1) + if i > 0 { + copy(schema.relations, old[:i-1]) + } + copy(schema.relations[i:], old[i+1:]) + schemasWithRelations[schema.name] = schema + } + + if len(schema.relations) == 0 { + // If there are no relevant relations from this schema, remove the schema from + // `schemasWithRelations` as the schema cannot form part of a circular schema + // set. + changedInLoop = true + delete(schemasWithRelations, schema.name) + break + } + } + } + + // If len(schemasWithRelations) > 0 here there are circular relations. + // We then need to traverse them all to break the remaing set down into + // sub sets of non-overlapping circles - we want this as the self-referencing + // set must be as small as possible, so that users providing multiple SDL/schema operations + // will result in the same IDs as a single large operation, provided that the individual schema + // declarations remain the same. + + circularSchemaNames := make([]string, len(schemasWithRelations)) + for name := range schemasWithRelations { + circularSchemaNames = append(circularSchemaNames, name) + } + // The order in which ID indexes are assigned must be deterministic, so + // we must loop through a sorted slice instead of the map. + slices.Sort(circularSchemaNames) + + var i int + schemaSetIds := map[string]int{} + schemasHit := map[string]struct{}{} + for _, name := range circularSchemaNames { + schema := schemasWithRelations[name] + mapSchemaSetIDs(&i, schema, schemaSetIds, schemasWithRelations, schemasHit) + } + + schemaSetsByID := map[int][]*client.SchemaDescription{} + for _, schema := range newSchemas { + schemaSetId, ok := schemaSetIds[schema.Name] + if !ok { + // In most cases, if a schema does not form a circular set then it will not be in + // schemaSetIds, and we can assign it a new, unused setID + i++ + schemaSetId = i + } + + schemaSet, ok := schemaSetsByID[schemaSetId] + if !ok { + schemaSet = make([]*client.SchemaDescription, 0, 1) + } + + schemaSet = append(schemaSet, schema) + schemaSetsByID[schemaSetId] = schemaSet + } + + schemaSets := [][]*client.SchemaDescription{} + for _, schemaSet := range schemaSetsByID { + schemaSets = append(schemaSets, schemaSet) + } + + return schemaSets +} + +// mapSchemaSetIDs recursively scans through a schema and its relations, assigning each schema to a temporary setID. +// +// If a set of schemas form a circular dependency, all involved schemas will be assigned the same setID. Assigned setIDs +// will be added to the input param `schemaSetIds`. +// +// This function will return when all descendents of the initial schema have been processed. +// +// Parameters: +// - i: The largest setID so far assigned. This parameter is mutated by this function. +// - schema: The current schema to process +// - schemaSetIds: The set of already assigned setIDs mapped by schema name - this parameter will be mutated by this +// function +// - schemasRelationsBySchemaName: The full set of relevant schemas/relations mapped by schema name +// - schemasFullyProcessed: The set of schema names that have already been completely processed. If `schema` is in +// this set the function will return. This parameter is mutated by this function. +func mapSchemaSetIDs( + i *int, + schema schemaRelations, + schemaSetIds map[string]int, + schemasRelationsBySchemaName map[string]schemaRelations, + schemasFullyProcessed map[string]struct{}, +) { + if _, ok := schemasFullyProcessed[schema.name]; ok { + // we've circled all the way through and already processed this schema + return + } + schemasFullyProcessed[schema.name] = struct{}{} + + for _, relation := range schema.relations { + // if more than one relation, need to find out if the relation loops back here! It might connect to a separate circle + circlesBackHere := circlesBack(schema.name, relation, schemasRelationsBySchemaName, map[string]struct{}{}) + + var circleID int + if circlesBackHere { + if id, ok := schemaSetIds[relation]; ok { + // If this schema has already been assigned a setID, use that + circleID = id + } else { + schemaSetId, ok := schemaSetIds[schema.name] + if !ok { + // If this schema has not already been assigned a setID, it must be + // the first discovered node in a new circle. Assign it a new setID, + // this will be picked up by its circle-forming descendents. + *i = *i + 1 + schemaSetId = *i + } + schemaSetIds[schema.name] = schemaSetId + circleID = schemaSetId + } + } else { + // If this schema and its relations does not circle back to itself, we + // increment `i` and assign the new value to this schema *only* + *i = *i + 1 + circleID = *i + } + + schemaSetIds[relation] = circleID + mapSchemaSetIDs( + i, + schemasRelationsBySchemaName[relation], + schemaSetIds, + schemasRelationsBySchemaName, + schemasFullyProcessed, + ) + } +} + +// circlesBack returns true if any path from this schema through it's relations (and their relations) circles +// back to this schema. +// +// Parameters: +// - originalSchemaName: The original start schema of this recursive check - this will not change as this function +// recursively checks the relations on `currentSchemaName`. +// - currentSchemaName: The current schema to process. +// - schemasWithRelations: The full set of relevant schemas that may be referenced by this schema or its descendents. +// - schemasFullyProcessed: The set of schema names that have already been completely processed. If `schema` is in +// this set the function will return. This parameter is mutated by this function. +func circlesBack( + originalSchemaName string, + currentSchemaName string, + schemasWithRelations map[string]schemaRelations, + schemasFullyProcessed map[string]struct{}, +) bool { + if _, ok := schemasFullyProcessed[currentSchemaName]; ok { + // we've circled all the way through and not found the original + return false + } + + if currentSchemaName == originalSchemaName { + return true + } + + schemasFullyProcessed[currentSchemaName] = struct{}{} + + for _, relation := range schemasWithRelations[currentSchemaName].relations { + ciclesBackToOriginal := circlesBack(originalSchemaName, relation, schemasWithRelations, schemasFullyProcessed) + if ciclesBackToOriginal { + return true + } + } + + return false +} + +func generateSetID(schemaSet []*client.SchemaDescription) (string, error) { + // The schemas within each set must be in a deterministic order to ensure that + // their IDs are deterministic. + slices.SortFunc(schemaSet, func(a, b *client.SchemaDescription) int { + return strings.Compare(a.Name, b.Name) + }) + + var cidComponents any + if len(schemaSet) == 1 { + cidComponents = schemaSet[0] + } else { + cidComponents = schemaSet + } + + buf, err := json.Marshal(cidComponents) + if err != nil { + return "", err + } + + scid, err := cid.NewSHA256CidV1(buf) + if err != nil { + return "", err + } + return scid.String(), nil +} + +func setIDs(baseID string, schemaSet []*client.SchemaDescription) { + if len(schemaSet) == 1 { + schemaSet[0].VersionID = baseID + if schemaSet[0].Root == "" { + // Schema Root remains constant through all versions, if it is set at this point + // do not update it. + schemaSet[0].Root = baseID + } + return + } + + for i := range schemaSet { + id := fmt.Sprintf("%s%v%v", baseID, schemaSetDeliminator, i) + + schemaSet[i].VersionID = id + if schemaSet[i].Root == "" { + // Schema Root remains constant through all versions, if it is set at this point + // do not update it. + schemaSet[i].Root = id + } + } +} + +// substituteRelationFieldKinds substitutes relations defined using [NamedKind]s to their long-term +// types. +// +// Using names to reference other types is unsuitable as the names may change over time. +func substituteRelationFieldKinds(schemas []client.SchemaDescription) { + schemasByName := map[string]client.SchemaDescription{} + for _, schema := range schemas { + schemasByName[schema.Name] = schema + } + + for i := range schemas { + rootComponents := strings.Split(schemas[i].Root, schemaSetDeliminator) + rootBase := rootComponents[0] + + for j := range schemas[i].Fields { + switch kind := schemas[i].Fields[j].Kind.(type) { + case *client.NamedKind: + relationSchema, ok := schemasByName[kind.Name] + if !ok { + // Continue, and let the validation step pick up whatever went wrong later + continue + } + + relationRootComponents := strings.Split(relationSchema.Root, schemaSetDeliminator) + if relationRootComponents[0] == rootBase { + if len(relationRootComponents) == 2 { + schemas[i].Fields[j].Kind = client.NewSelfKind(relationRootComponents[1], kind.IsArray()) + } else { + // If the relation root is simple and does not contain a relative index, then this relation + // must point to the host schema (self-reference, e.g. User=>User). + schemas[i].Fields[j].Kind = client.NewSelfKind("", kind.IsArray()) + } + } else { + schemas[i].Fields[j].Kind = client.NewSchemaKind(relationSchema.Root, kind.IsArray()) + } + + default: + // no-op + } + } + } +} diff --git a/internal/planner/mapper/mapper.go b/internal/planner/mapper/mapper.go index 7542449727..69c87d82fc 100644 --- a/internal/planner/mapper/mapper.go +++ b/internal/planner/mapper/mapper.go @@ -821,9 +821,14 @@ func getCollectionName( hostFieldDesc, parentHasField := parentCollection.Definition().GetFieldByName(selectRequest.Name) if parentHasField && hostFieldDesc.Kind.IsObject() { + def, found, err := client.GetDefinitionUncached(ctx, store, parentCollection.Definition(), hostFieldDesc.Kind) + if !found { + return "", NewErrTypeNotFound(hostFieldDesc.Kind.String()) + } + // If this field exists on the parent, and it is a child object // then this collection name is the collection name of the child. - return hostFieldDesc.Kind.Underlying(), nil + return def.GetName(), err } } diff --git a/internal/request/graphql/schema/collection.go b/internal/request/graphql/schema/collection.go index be361e0ec2..5f8d121b62 100644 --- a/internal/request/graphql/schema/collection.go +++ b/internal/request/graphql/schema/collection.go @@ -371,8 +371,8 @@ func fieldsFromAST( schemaFieldDescriptions := []client.SchemaFieldDescription{} collectionFieldDescriptions := []client.CollectionFieldDescription{} - if kind.IsObject() { - relationName, err := getRelationshipName(field, hostObjectName, kind.Underlying()) + if namedKind, ok := kind.(*client.NamedKind); ok { + relationName, err := getRelationshipName(field, hostObjectName, namedKind.Name) if err != nil { return nil, nil, err } @@ -544,7 +544,7 @@ func astTypeToKind(t ast.Type) (client.FieldKind, error) { case typeString: return client.FieldKind_NILLABLE_STRING_ARRAY, nil default: - return client.ObjectArrayKind(astTypeVal.Type.(*ast.Named).Name.Value), nil + return client.NewNamedKind(astTypeVal.Type.(*ast.Named).Name.Value, true), nil } } @@ -567,7 +567,7 @@ func astTypeToKind(t ast.Type) (client.FieldKind, error) { case typeJSON: return client.FieldKind_NILLABLE_JSON, nil default: - return client.ObjectKind(astTypeVal.Name.Value), nil + return client.NewNamedKind(astTypeVal.Name.Value, false), nil } case *ast.NonNull: @@ -644,7 +644,12 @@ func finalizeRelations( } for _, field := range definition.Description.Fields { - if !field.Kind.HasValue() || !field.Kind.Value().IsObject() || field.Kind.Value().IsArray() { + if !field.Kind.HasValue() { + continue + } + + namedKind, ok := field.Kind.Value().(*client.NamedKind) + if !ok || namedKind.IsArray() { // We only need to process the primary side of a relation here, if the field is not a relation // or if it is an array, we can skip it. continue @@ -654,7 +659,7 @@ func finalizeRelations( for _, otherDef := range definitions { // Check the 'other' schema name, there can only be a one-one mapping in an SDL // appart from embedded, which will be schema only. - if otherDef.Schema.Name == field.Kind.Value().Underlying() { + if otherDef.Schema.Name == namedKind.Name { otherColDefinition = immutable.Some(otherDef) break } diff --git a/internal/request/graphql/schema/generate.go b/internal/request/graphql/schema/generate.go index 4d86a97204..476b1f954c 100644 --- a/internal/request/graphql/schema/generate.go +++ b/internal/request/graphql/schema/generate.go @@ -416,6 +416,8 @@ func (g *Generator) createExpandedFieldList( func (g *Generator) buildTypes( collections []client.CollectionDefinition, ) ([]*gql.Object, error) { + definitionCache := client.NewDefinitionCache(collections) + // @todo: Check for duplicate named defined types in the TypeMap // get all the defined types from the AST objs := make([]*gql.Object, 0) @@ -471,11 +473,10 @@ func (g *Generator) buildTypes( } var ttype gql.Type - if field.Kind.IsObject() { - var ok bool - ttype, ok = g.manager.schema.TypeMap()[field.Kind.Underlying()] + if otherDef, ok := client.GetDefinition(definitionCache, c, field.Kind); ok { + ttype, ok = g.manager.schema.TypeMap()[otherDef.GetName()] if !ok { - return nil, NewErrTypeNotFound(field.Kind.Underlying()) + return nil, NewErrTypeNotFound(field.Kind.String()) } if field.Kind.IsArray() { ttype = gql.NewList(ttype) @@ -484,7 +485,7 @@ func (g *Generator) buildTypes( var ok bool ttype, ok = fieldKindToGQLType[field.Kind] if !ok { - return nil, NewErrTypeNotFound(fmt.Sprint(field.Kind)) + return nil, NewErrTypeNotFound(field.Kind.String()) } } diff --git a/tests/gen/gen_auto.go b/tests/gen/gen_auto.go index 487558e934..5b00d5f5b2 100644 --- a/tests/gen/gen_auto.go +++ b/tests/gen/gen_auto.go @@ -38,7 +38,7 @@ func AutoGenerateFromSDL(gqlSDL string, options ...Option) ([]GeneratedDoc, erro if err != nil { return nil, err } - typeDefs, err := parseSDL(gqlSDL) + typeDefs, err := ParseSDL(gqlSDL) if err != nil { return nil, err } @@ -119,9 +119,15 @@ func (g *randomDocGenerator) getMaxTotalDemand() int { } // getNextPrimaryDocID returns the docID of the next primary document to be used as a relation. -func (g *randomDocGenerator) getNextPrimaryDocID(secondaryType string, field *client.FieldDefinition) string { +func (g *randomDocGenerator) getNextPrimaryDocID( + host client.CollectionDefinition, + secondaryType string, + field *client.FieldDefinition, +) string { ind := g.configurator.usageCounter.getNextTypeIndForField(secondaryType, field) - return g.generatedDocs[field.Kind.Underlying()][ind].docID + otherDef, _ := client.GetDefinition(g.configurator.definitionCache, host, field.Kind) + + return g.generatedDocs[otherDef.GetName()][ind].docID } func (g *randomDocGenerator) generateRandomDocs(order []string) error { @@ -141,9 +147,9 @@ func (g *randomDocGenerator) generateRandomDocs(order []string) error { if field.IsRelation() { if field.IsPrimaryRelation && field.Kind.IsObject() { if strings.HasSuffix(field.Name, request.RelatedObjectID) { - newDoc[field.Name] = g.getNextPrimaryDocID(typeName, &field) + newDoc[field.Name] = g.getNextPrimaryDocID(typeDef, typeName, &field) } else { - newDoc[field.Name+request.RelatedObjectID] = g.getNextPrimaryDocID(typeName, &field) + newDoc[field.Name+request.RelatedObjectID] = g.getNextPrimaryDocID(typeDef, typeName, &field) } } } else { @@ -210,7 +216,8 @@ func (g *randomDocGenerator) getValueGenerator(fieldKind client.FieldKind, field func validateDefinitions(definitions []client.CollectionDefinition) error { colIDs := make(map[uint32]struct{}) colNames := make(map[string]struct{}) - fieldRefs := []string{} + defCache := client.NewDefinitionCache(definitions) + for _, def := range definitions { if def.Description.Name.Value() == "" { return NewErrIncompleteColDefinition("description name is empty") @@ -226,17 +233,16 @@ func validateDefinitions(definitions []client.CollectionDefinition) error { return NewErrIncompleteColDefinition("field name is empty") } if field.Kind.IsObject() { - fieldRefs = append(fieldRefs, field.Kind.Underlying()) + _, found := client.GetDefinition(defCache, def, field.Kind) + if !found { + return NewErrIncompleteColDefinition("field schema references unknown collection") + } } } colNames[def.Description.Name.Value()] = struct{}{} colIDs[def.Description.ID] = struct{}{} } - for _, ref := range fieldRefs { - if _, ok := colNames[ref]; !ok { - return NewErrIncompleteColDefinition("field schema references unknown collection") - } - } + if len(colIDs) != len(definitions) { return NewErrIncompleteColDefinition("duplicate collection IDs") } diff --git a/tests/gen/gen_auto_configurator.go b/tests/gen/gen_auto_configurator.go index ec8c1ea881..ce567fd385 100644 --- a/tests/gen/gen_auto_configurator.go +++ b/tests/gen/gen_auto_configurator.go @@ -15,6 +15,8 @@ import ( "math/rand" "time" + "github.com/sourcenetwork/immutable" + "github.com/sourcenetwork/defradb/client" ) @@ -35,7 +37,9 @@ func (d typeDemand) getAverage() int { // demand for each type, setting up the relation usage counters, and setting up // the random seed. type docsGenConfigurator struct { - types map[string]client.CollectionDefinition + types map[string]client.CollectionDefinition + definitionCache client.DefinitionCache + config configsMap primaryGraph map[string][]string typesOrder []string @@ -44,6 +48,8 @@ type docsGenConfigurator struct { random *rand.Rand } +type collectionID = uint32 + // typeUsageCounters is a map of primary type to secondary type to field name to // relation usage. This is used to keep track of the usage of each relation. // Each foreign field has a tracker that keeps track of which and how many of primary @@ -51,13 +57,13 @@ type docsGenConfigurator struct { // number of documents generated for each primary type is within the range of the // demand for that type and to guarantee a uniform distribution of the documents. type typeUsageCounters struct { - m map[string]map[string]map[string]*relationUsage + m map[collectionID]map[string]map[string]*relationUsage random *rand.Rand } func newTypeUsageCounter(random *rand.Rand) typeUsageCounters { return typeUsageCounters{ - m: make(map[string]map[string]map[string]*relationUsage), + m: make(map[collectionID]map[string]map[string]*relationUsage), random: random, } } @@ -68,21 +74,35 @@ func (c *typeUsageCounters) addRelationUsage( field client.FieldDefinition, minPerDoc, maxPerDoc, numDocs int, ) { - primaryType := field.Kind.Underlying() - if _, ok := c.m[primaryType]; !ok { - c.m[primaryType] = make(map[string]map[string]*relationUsage) + var collectionRoot uint32 + switch kind := field.Kind.(type) { + case *client.CollectionKind: + collectionRoot = kind.Root + + default: + return + } + + if _, ok := c.m[collectionRoot]; !ok { + c.m[collectionRoot] = make(map[string]map[string]*relationUsage) } - if _, ok := c.m[primaryType][secondaryType]; !ok { - c.m[primaryType][secondaryType] = make(map[string]*relationUsage) + if _, ok := c.m[collectionRoot][secondaryType]; !ok { + c.m[collectionRoot][secondaryType] = make(map[string]*relationUsage) } - if _, ok := c.m[primaryType][secondaryType][field.Name]; !ok { - c.m[primaryType][secondaryType][field.Name] = newRelationUsage(minPerDoc, maxPerDoc, numDocs, c.random) + if _, ok := c.m[collectionRoot][secondaryType][field.Name]; !ok { + c.m[collectionRoot][secondaryType][field.Name] = newRelationUsage(minPerDoc, maxPerDoc, numDocs, c.random) } } // getNextTypeIndForField returns the next index to be used for a foreign field. func (c *typeUsageCounters) getNextTypeIndForField(secondaryType string, field *client.FieldDefinition) int { - current := c.m[field.Kind.Underlying()][secondaryType][field.Name] + var collectionRoot uint32 + switch kind := field.Kind.(type) { + case *client.CollectionKind: + collectionRoot = kind.Root + } + + current := c.m[collectionRoot][secondaryType][field.Name] return current.useNextDocIDIndex() } @@ -156,10 +176,16 @@ func (u *relationUsage) allocateIndexes() { } func newDocGenConfigurator(types map[string]client.CollectionDefinition, config configsMap) docsGenConfigurator { + defs := make([]client.CollectionDefinition, 0, len(types)) + for _, def := range types { + defs = append(defs, def) + } + return docsGenConfigurator{ - types: types, - config: config, - docsDemand: make(map[string]typeDemand), + types: types, + definitionCache: client.NewDefinitionCache(defs), + config: config, + docsDemand: make(map[string]typeDemand), } } @@ -185,7 +211,7 @@ func (g *docsGenConfigurator) Configure(options ...Option) error { g.usageCounter = newTypeUsageCounter(g.random) - g.primaryGraph = getRelationGraph(g.types) + g.primaryGraph = g.getRelationGraph(g.types) g.typesOrder = getTopologicalOrder(g.primaryGraph, g.types) if len(g.docsDemand) == 0 { @@ -252,7 +278,10 @@ func (g *docsGenConfigurator) allocateUsageCounterIndexes() { demand.min = max g.docsDemand[typeName] = demand } - for _, usage := range g.usageCounter.m[typeName] { + + def := g.types[typeName] + + for _, usage := range g.usageCounter.m[def.Description.RootID] { for _, field := range usage { if field.numAvailablePrimaryDocs == math.MaxInt { field.numAvailablePrimaryDocs = max @@ -272,8 +301,16 @@ func (g *docsGenConfigurator) getDemandForPrimaryType( primaryGraph map[string][]string, ) (typeDemand, error) { primaryTypeDef := g.types[primaryType] + secondaryTypeDef := g.types[secondaryType] + for _, field := range primaryTypeDef.GetFields() { - if field.Kind.IsObject() && field.Kind.Underlying() == secondaryType { + var otherRoot immutable.Option[uint32] + switch kind := field.Kind.(type) { + case *client.CollectionKind: + otherRoot = immutable.Some(kind.Root) + } + + if otherRoot.HasValue() && otherRoot.Value() == secondaryTypeDef.Description.RootID { primaryDemand := typeDemand{min: secondaryDemand.min, max: secondaryDemand.max} minPerDoc, maxPerDoc := 1, 1 @@ -312,7 +349,7 @@ func (g *docsGenConfigurator) getDemandForPrimaryType( return typeDemand{}, NewErrCanNotSupplyTypeDemand(primaryType) } g.docsDemand[primaryType] = primaryDemand - g.initRelationUsages(field.Kind.Underlying(), primaryType, minPerDoc, maxPerDoc) + g.initRelationUsages(secondaryTypeDef.GetName(), primaryType, minPerDoc, maxPerDoc) } } return secondaryDemand, nil @@ -344,7 +381,8 @@ func (g *docsGenConfigurator) calculateDemandForSecondaryTypes( newSecDemand := typeDemand{min: primaryDocDemand.min, max: primaryDocDemand.max} minPerDoc, maxPerDoc := 1, 1 - curSecDemand, hasSecDemand := g.docsDemand[field.Kind.Underlying()] + otherType, _ := client.GetDefinition(g.definitionCache, typeDef, field.Kind) + curSecDemand, hasSecDemand := g.docsDemand[otherType.GetName()] if field.Kind.IsArray() { fieldConf := g.config.ForField(typeName, field.Name) @@ -368,23 +406,23 @@ func (g *docsGenConfigurator) calculateDemandForSecondaryTypes( if hasSecDemand { if curSecDemand.min < newSecDemand.min || curSecDemand.max > newSecDemand.max { - return NewErrCanNotSupplyTypeDemand(field.Kind.Underlying()) + return NewErrCanNotSupplyTypeDemand(otherType.GetName()) } } else { - g.docsDemand[field.Kind.Underlying()] = newSecDemand + g.docsDemand[otherType.GetName()] = newSecDemand } - g.initRelationUsages(field.Kind.Underlying(), typeName, minPerDoc, maxPerDoc) + g.initRelationUsages(otherType.GetName(), typeName, minPerDoc, maxPerDoc) - err := g.calculateDemandForSecondaryTypes(field.Kind.Underlying(), primaryGraph) + err := g.calculateDemandForSecondaryTypes(otherType.GetName(), primaryGraph) if err != nil { return err } - for _, primaryTypeName := range primaryGraph[field.Kind.Underlying()] { + for _, primaryTypeName := range primaryGraph[otherType.GetName()] { if _, ok := g.docsDemand[primaryTypeName]; !ok { primaryDemand, err := g.getDemandForPrimaryType( primaryTypeName, - field.Kind.Underlying(), + otherType.GetName(), newSecDemand, primaryGraph, ) @@ -401,15 +439,22 @@ func (g *docsGenConfigurator) calculateDemandForSecondaryTypes( func (g *docsGenConfigurator) initRelationUsages(secondaryType, primaryType string, minPerDoc, maxPerDoc int) { secondaryTypeDef := g.types[secondaryType] + primaryTypeDef := g.types[primaryType] for _, secondaryTypeField := range secondaryTypeDef.GetFields() { - if secondaryTypeField.Kind.Underlying() == primaryType { + var otherRoot immutable.Option[uint32] + switch kind := secondaryTypeField.Kind.(type) { + case *client.CollectionKind: + otherRoot = immutable.Some(kind.Root) + } + + if otherRoot.HasValue() && otherRoot.Value() == primaryTypeDef.Description.RootID { g.usageCounter.addRelationUsage(secondaryType, secondaryTypeField, minPerDoc, maxPerDoc, g.docsDemand[primaryType].getAverage()) } } } -func getRelationGraph(types map[string]client.CollectionDefinition) map[string][]string { +func (g *docsGenConfigurator) getRelationGraph(types map[string]client.CollectionDefinition) map[string][]string { primaryGraph := make(map[string][]string) appendUnique := func(slice []string, val string) []string { @@ -424,10 +469,12 @@ func getRelationGraph(types map[string]client.CollectionDefinition) map[string][ for typeName, typeDef := range types { for _, field := range typeDef.GetFields() { if field.Kind.IsObject() { + otherDef, _ := client.GetDefinition(g.definitionCache, typeDef, field.Kind) + if field.IsPrimaryRelation { - primaryGraph[typeName] = appendUnique(primaryGraph[typeName], field.Kind.Underlying()) + primaryGraph[typeName] = appendUnique(primaryGraph[typeName], otherDef.GetName()) } else { - primaryGraph[field.Kind.Underlying()] = appendUnique(primaryGraph[field.Kind.Underlying()], typeName) + primaryGraph[otherDef.GetName()] = appendUnique(primaryGraph[otherDef.GetName()], typeName) } } } diff --git a/tests/gen/gen_auto_test.go b/tests/gen/gen_auto_test.go index 02cb45331b..612b244030 100644 --- a/tests/gen/gen_auto_test.go +++ b/tests/gen/gen_auto_test.go @@ -1209,7 +1209,7 @@ func TestAutoGenerate_IfCollectionDefinitionIsIncomplete_ReturnError(t *testing. }, { Name: "device", - Kind: immutable.Some[client.FieldKind](client.ObjectKind("Device")), + Kind: immutable.Some[client.FieldKind](client.NewNamedKind("Device", false)), }, }, }, @@ -1233,7 +1233,7 @@ func TestAutoGenerate_IfCollectionDefinitionIsIncomplete_ReturnError(t *testing. }, { Name: "owner", - Kind: immutable.Some[client.FieldKind](client.ObjectKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewNamedKind("User", false)), }, }, }, @@ -1246,7 +1246,7 @@ func TestAutoGenerate_IfCollectionDefinitionIsIncomplete_ReturnError(t *testing. }, { Name: "owner", - Kind: client.ObjectKind("User"), + Kind: client.NewNamedKind("User", false), }, }, }, @@ -1316,8 +1316,9 @@ func TestAutoGenerate_IfColDefinitionsAreValid_ShouldGenerate(t *testing.T) { defs := []client.CollectionDefinition{ { Description: client.CollectionDescription{ - Name: immutable.Some("User"), - ID: 0, + Name: immutable.Some("User"), + ID: 0, + RootID: 0, Fields: []client.CollectionFieldDescription{ { Name: "name", @@ -1330,7 +1331,7 @@ func TestAutoGenerate_IfColDefinitionsAreValid_ShouldGenerate(t *testing.T) { }, { Name: "devices", - Kind: immutable.Some[client.FieldKind](client.ObjectArrayKind("Device")), + Kind: immutable.Some[client.FieldKind](client.NewCollectionKind(1, true)), RelationName: immutable.Some("Device_owner"), }, }, @@ -1355,15 +1356,16 @@ func TestAutoGenerate_IfColDefinitionsAreValid_ShouldGenerate(t *testing.T) { }, { Description: client.CollectionDescription{ - Name: immutable.Some("Device"), - ID: 1, + Name: immutable.Some("Device"), + ID: 1, + RootID: 1, Fields: []client.CollectionFieldDescription{ { Name: "model", }, { Name: "owner", - Kind: immutable.Some[client.FieldKind](client.ObjectKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewCollectionKind(0, false)), RelationName: immutable.Some("Device_owner"), }, { @@ -1381,7 +1383,7 @@ func TestAutoGenerate_IfColDefinitionsAreValid_ShouldGenerate(t *testing.T) { }, { Name: "owner", - Kind: client.ObjectKind("User"), + Kind: client.NewNamedKind("User", false), Typ: client.LWW_REGISTER, }, { diff --git a/tests/gen/schema_parser.go b/tests/gen/schema_parser.go index fbd82bc47b..3e08212b5c 100644 --- a/tests/gen/schema_parser.go +++ b/tests/gen/schema_parser.go @@ -17,21 +17,48 @@ import ( "unicode" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/internal/request/graphql" + "github.com/sourcenetwork/defradb/node" ) -func parseSDL(gqlSDL string) (map[string]client.CollectionDefinition, error) { - parser, err := graphql.NewParser() +func ParseSDL(gqlSDL string) (map[string]client.CollectionDefinition, error) { + ctx := context.Background() + + // Spinning up a temporary in-memory node with all extras disabled is the + // most reliable and cheapest maintainance-cost-wise way to fully parse + // the SDL and correctly link all relations. + node, err := node.NewNode( + ctx, + node.WithBadgerInMemory(true), + node.WithDisableAPI(true), + node.WithDisableP2P(true), + ) if err != nil { return nil, err } - cols, err := parser.ParseSDL(context.Background(), gqlSDL) + + err = node.Start(ctx) + if err != nil { + return nil, err + } + + _, err = node.DB.AddSchema(ctx, gqlSDL) + if err != nil { + return nil, err + } + + cols, err := node.DB.GetCollections(ctx, client.CollectionFetchOptions{}) if err != nil { return nil, err } - result := make(map[string]client.CollectionDefinition) + + err = node.Close(ctx) + if err != nil { + return nil, err + } + + result := make(map[string]client.CollectionDefinition, len(cols)) for _, col := range cols { - result[col.Description.Name.Value()] = col + result[col.Definition().GetName()] = col.Definition() } return result, nil } diff --git a/tests/integration/collection_description/updates/replace/id_test.go b/tests/integration/collection_description/updates/replace/id_test.go index a89dad193b..7538f2161e 100644 --- a/tests/integration/collection_description/updates/replace/id_test.go +++ b/tests/integration/collection_description/updates/replace/id_test.go @@ -115,7 +115,7 @@ func TestColDescrUpdateReplaceID_WithExistingDifferentRoot_Errors(t *testing.T) { "op": "replace", "path": "/2/ID", "value": 1 } ] `, - ExpectedError: "collection root ID cannot be mutated.", + ExpectedError: "collection root ID cannot be mutated. CollectionID:", }, }, } diff --git a/tests/integration/collection_description/updates/replace/name_one_many_test.go b/tests/integration/collection_description/updates/replace/name_one_many_test.go index 3cb93a86b6..0993af66c0 100644 --- a/tests/integration/collection_description/updates/replace/name_one_many_test.go +++ b/tests/integration/collection_description/updates/replace/name_one_many_test.go @@ -50,7 +50,107 @@ func TestColDescrUpdateReplaceNameOneToMany_GivenExistingName(t *testing.T) { { "op": "replace", "path": "/1/Name", "value": "Writer" } ] `, - ExpectedError: `no type found for given name. Field: author, Kind: Author`, + }, + testUtils.Request{ + Request: `query { + Book { + name + author { + name + } + } + }`, + Results: map[string]any{ + "Book": []map[string]any{ + { + "name": "Painted House", + "author": map[string]any{ + "name": "John Grisham", + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestColDescrUpdateReplaceNameOneToMany_GivenExistingNameReplacedBeforeAndAfterCreate(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Author { + name: String + books: [Book] + } + + type Book { + name: String + author: Author + } + `, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "John Grisham", + }, + }, + testUtils.CreateDoc{ + CollectionID: 1, + DocMap: map[string]any{ + "name": "Painted House", + "author": testUtils.NewDocIndex(0, 0), + }, + }, + testUtils.PatchCollection{ + Patch: ` + [ + { "op": "replace", "path": "/1/Name", "value": "Writer" } + ] + `, + }, + testUtils.CreateDoc{ + DocMap: map[string]any{ + "name": "Cornelia Funke", + }, + }, + testUtils.CreateDoc{ + CollectionID: 1, + DocMap: map[string]any{ + "name": "Theif Lord", + "author": testUtils.NewDocIndex(0, 1), + }, + }, + testUtils.Request{ + Request: `query { + Book { + name + author { + name + } + } + }`, + // This test ensures that documents created before and after the collection rename + // are correctly fetched together + Results: map[string]any{ + "Book": []map[string]any{ + { + "name": "Painted House", + "author": map[string]any{ + "name": "John Grisham", + }, + }, + { + "name": "Theif Lord", + "author": map[string]any{ + "name": "Cornelia Funke", + }, + }, + }, + }, }, }, } diff --git a/tests/integration/schema/one_many_test.go b/tests/integration/schema/one_many_test.go index 40d73f58de..cd7a288f8b 100644 --- a/tests/integration/schema/one_many_test.go +++ b/tests/integration/schema/one_many_test.go @@ -44,7 +44,7 @@ func TestSchemaOneMany_Primary(t *testing.T) { { Name: "dogs", ID: 1, - Kind: immutable.Some[client.FieldKind](client.ObjectArrayKind("Dog")), + Kind: immutable.Some[client.FieldKind](client.NewCollectionKind(2, true)), RelationName: immutable.Some("dog_user"), }, { @@ -66,7 +66,7 @@ func TestSchemaOneMany_Primary(t *testing.T) { { Name: "owner", ID: 2, - Kind: immutable.Some[client.FieldKind](client.ObjectKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewCollectionKind(1, false)), RelationName: immutable.Some("dog_user"), }, { @@ -105,7 +105,7 @@ func TestSchemaOneMany_SelfReferenceOneFieldLexographicallyFirst(t *testing.T) { { Name: "a", ID: 1, - Kind: immutable.Some[client.FieldKind](client.ObjectKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewSelfKind("", false)), RelationName: immutable.Some("user_user"), }, { @@ -117,7 +117,7 @@ func TestSchemaOneMany_SelfReferenceOneFieldLexographicallyFirst(t *testing.T) { { Name: "b", ID: 3, - Kind: immutable.Some[client.FieldKind](client.ObjectArrayKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewSelfKind("", true)), RelationName: immutable.Some("user_user"), }, }, @@ -150,13 +150,13 @@ func TestSchemaOneMany_SelfReferenceManyFieldLexographicallyFirst(t *testing.T) { Name: "a", ID: 1, - Kind: immutable.Some[client.FieldKind](client.ObjectArrayKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewSelfKind("", true)), RelationName: immutable.Some("user_user"), }, { Name: "b", ID: 2, - Kind: immutable.Some[client.FieldKind](client.ObjectKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewSelfKind("", false)), RelationName: immutable.Some("user_user"), }, { @@ -200,7 +200,7 @@ func TestSchemaOneMany_SelfUsingActualName(t *testing.T) { { Name: "boss", ID: 1, - Kind: immutable.Some[client.FieldKind](client.ObjectKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewSelfKind("", false)), RelationName: immutable.Some("user_user"), }, { @@ -212,7 +212,7 @@ func TestSchemaOneMany_SelfUsingActualName(t *testing.T) { { Name: "minions", ID: 3, - Kind: immutable.Some[client.FieldKind](client.ObjectArrayKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewSelfKind("", true)), RelationName: immutable.Some("user_user"), }, }, @@ -232,7 +232,7 @@ func TestSchemaOneMany_SelfUsingActualName(t *testing.T) { }, { Name: "boss", - Kind: client.ObjectKind("User"), + Kind: client.NewSelfKind("", false), Typ: client.LWW_REGISTER, }, { diff --git a/tests/integration/schema/one_one_test.go b/tests/integration/schema/one_one_test.go index aea979c836..d0fea6b7a2 100644 --- a/tests/integration/schema/one_one_test.go +++ b/tests/integration/schema/one_one_test.go @@ -88,7 +88,7 @@ func TestSchemaOneOne_SelfUsingActualName(t *testing.T) { { Name: "boss", ID: 1, - Kind: immutable.Some[client.FieldKind](client.ObjectKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewSelfKind("", false)), RelationName: immutable.Some("user_user"), }, { @@ -100,7 +100,7 @@ func TestSchemaOneOne_SelfUsingActualName(t *testing.T) { { Name: "minion", ID: 3, - Kind: immutable.Some[client.FieldKind](client.ObjectKind("User")), + Kind: immutable.Some[client.FieldKind](client.NewSelfKind("", false)), RelationName: immutable.Some("user_user"), }, { @@ -126,7 +126,7 @@ func TestSchemaOneOne_SelfUsingActualName(t *testing.T) { }, { Name: "boss", - Kind: client.ObjectKind("User"), + Kind: client.NewSelfKind("", false), Typ: client.LWW_REGISTER, }, { diff --git a/tests/integration/schema/self_ref_test.go b/tests/integration/schema/self_ref_test.go new file mode 100644 index 0000000000..888b9db280 --- /dev/null +++ b/tests/integration/schema/self_ref_test.go @@ -0,0 +1,737 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package schema + +import ( + "testing" + + "github.com/sourcenetwork/defradb/client" + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestSchemaSelfReferenceSimple_SchemaHasSimpleSchemaID(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + boss: User + } + `, + }, + testUtils.IntrospectionRequest{ + Request: ` + query { + __type (name: "User") { + name + fields { + name + type { + name + kind + } + } + } + } + `, + ExpectedData: map[string]any{ + "__type": map[string]any{ + "name": "User", + "fields": DefaultFields.Append( + Field{ + "name": "boss_id", + "type": map[string]any{ + "kind": "SCALAR", + "name": "ID", + }, + }, + ).Append( + Field{ + "name": "boss", + "type": map[string]any{ + "kind": "OBJECT", + "name": "User", + }, + }, + ).Tidy(), + }, + }, + }, + testUtils.GetSchema{ + ExpectedResults: []client.SchemaDescription{ + { + Name: "User", + Root: "bafkreifchjktkdtha7vkcqt6itzsw6lnzfyp7ufws4s32e7vigu7akn2q4", + VersionID: "bafkreifchjktkdtha7vkcqt6itzsw6lnzfyp7ufws4s32e7vigu7akn2q4", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "boss", + Typ: client.LWW_REGISTER, + // Simple self kinds do not contain a base ID, as there is only one possible value + // that they could hold + Kind: client.NewSelfKind("", false), + }, + { + Name: "boss_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestSchemaSelfReferenceTwoTypes_SchemaHasComplexSchemaID(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + // The two primary relations form a circular two-collection self reference + Schema: ` + type User { + hosts: Dog @primary @relation(name:"hosts") + walks: Dog @relation(name:"walkies") + } + type Dog { + host: User @relation(name:"hosts") + walker: User @primary @relation(name:"walkies") + } + `, + }, + testUtils.IntrospectionRequest{ + Request: ` + query { + __type (name: "User") { + name + fields { + name + type { + name + kind + } + } + } + } + `, + ExpectedData: map[string]any{ + "__type": map[string]any{ + "name": "User", + "fields": DefaultFields.Append( + Field{ + "name": "hosts_id", + "type": map[string]any{ + "kind": "SCALAR", + "name": "ID", + }, + }, + ).Append( + Field{ + "name": "hosts", + "type": map[string]any{ + "kind": "OBJECT", + "name": "Dog", + }, + }, + ).Append( + Field{ + "name": "walks_id", + "type": map[string]any{ + "kind": "SCALAR", + "name": "ID", + }, + }, + ).Append( + Field{ + "name": "walks", + "type": map[string]any{ + "kind": "OBJECT", + "name": "Dog", + }, + }, + ).Tidy(), + }, + }, + }, + testUtils.IntrospectionRequest{ + Request: ` + query { + __type (name: "Dog") { + name + fields { + name + type { + name + kind + } + } + } + } + `, + ExpectedData: map[string]any{ + "__type": map[string]any{ + "name": "Dog", + "fields": DefaultFields.Append( + Field{ + "name": "host_id", + "type": map[string]any{ + "kind": "SCALAR", + "name": "ID", + }, + }, + ).Append( + Field{ + "name": "host", + "type": map[string]any{ + "kind": "OBJECT", + "name": "User", + }, + }, + ).Append( + Field{ + "name": "walker_id", + "type": map[string]any{ + "kind": "SCALAR", + "name": "ID", + }, + }, + ).Append( + Field{ + "name": "walker", + "type": map[string]any{ + "kind": "OBJECT", + "name": "User", + }, + }, + ).Tidy(), + }, + }, + }, + testUtils.GetSchema{ + ExpectedResults: []client.SchemaDescription{ + { + Name: "Dog", + // Note how Dog and User share the same base ID, but with a different index suffixed on + // the end. + Root: "bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-0", + VersionID: "bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-0", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "walker", + Typ: client.LWW_REGISTER, + // Because Dog and User form a circular dependency tree, the relation is declared + // as a SelfKind, with the index identifier of User being held in the relation kind. + Kind: client.NewSelfKind("1", false), + }, + { + Name: "walker_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "User", + // Note how Dog and User share the same base ID, but with a different index suffixed on + // the end. + Root: "bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-1", + VersionID: "bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-1", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "hosts", + Typ: client.LWW_REGISTER, + // Because Dog and User form a circular dependency tree, the relation is declared + // as a SelfKind, with the index identifier of User being held in the relation kind. + Kind: client.NewSelfKind("0", false), + }, + { + Name: "hosts_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestSchemaSelfReferenceTwoPairsOfTwoTypes_SchemasHaveDifferentComplexSchemaID(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + // - User and Dog form a circular dependency. + // - Cat and Mouse form a another circular dependency. + // - There is a relationship from Cat to User, this does not form a circular dependency + // between the two (User/Dog and Cat/Mouse) circles, this is included to ensure that + // the code does not incorrectly merge the User/Dog and Cat/Mouse circles into a single + // circle. + Schema: ` + type User { + hosts: Dog @primary @relation(name:"hosts") + walks: Dog @relation(name:"walkies") + toleratedBy: Cat @relation(name:"tolerates") + } + type Dog { + host: User @relation(name:"hosts") + walker: User @primary @relation(name:"walkies") + } + type Cat { + loves: Mouse @primary @relation(name:"loves") + hatedBy: Mouse @relation(name:"hates") + tolerates: User @primary @relation(name:"tolerates") + } + type Mouse { + lovedBy: Cat @relation(name:"loves") + hates: Cat @primary @relation(name:"hates") + } + `, + }, + testUtils.GetSchema{ + ExpectedResults: []client.SchemaDescription{ + { + Name: "Cat", + // Cat and Mouse share the same base ID, but with a different index suffixed on + // the end. This base must be different to the Dog/User base ID. + Root: "bafkreiacf7kjwlw32eiizyy6awdnfrnn7edaptp2chhfc5xktgxvrccqsa-0", + VersionID: "bafkreiacf7kjwlw32eiizyy6awdnfrnn7edaptp2chhfc5xktgxvrccqsa-0", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "loves", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("1", false), + }, + { + Name: "loves_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + { + Name: "tolerates", + Typ: client.LWW_REGISTER, + // This relationship reaches out of the Cat/Dog circle, and thus must be of type SchemaKind, + // specified with the full User ID (including the `-1` index suffixed). + Kind: client.NewSchemaKind("bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-1", false), + }, + { + Name: "tolerates_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "Mouse", + // Cat and Mouse share the same base ID, but with a different index suffixed on + // the end. This base must be different to the Dog/User base ID. + Root: "bafkreiacf7kjwlw32eiizyy6awdnfrnn7edaptp2chhfc5xktgxvrccqsa-1", + VersionID: "bafkreiacf7kjwlw32eiizyy6awdnfrnn7edaptp2chhfc5xktgxvrccqsa-1", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "hates", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("0", false), + }, + { + Name: "hates_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "Dog", + // Dog and User share the same base ID, but with a different index suffixed on + // the end. This base must be different to the Cat/Mouse base ID. + Root: "bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-0", + VersionID: "bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-0", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "walker", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("1", false), + }, + { + Name: "walker_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "User", + // Dog and User share the same base ID, but with a different index suffixed on + // the end. This base must be different to the Cat/Mouse base ID. + Root: "bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-1", + VersionID: "bafkreichlth4ajgalengyv3hnmqnxa4vhnv5f34a3gzwh2jaajqb2yxd4i-1", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "hosts", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("0", false), + }, + { + Name: "hosts_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestSchemaSelfReferenceTwoPairsOfTwoTypesJoinedByThirdCircle_SchemasAllHaveSameBaseSchemaID(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + // - User and Dog form a circular dependency. + // - Cat and Mouse form a another circular dependency. + // - User and Cat form a circular dependency - this circle overlaps with the two otherwise + // independent User/Dog and Cat/Mouse circles, causing the 4 types to be locked together in + // a larger circle (a relationship DAG cannot be formed) - all 4 types must thus share the + // same base ID. + Schema: ` + type User { + hosts: Dog @primary @relation(name:"hosts") + walks: Dog @relation(name:"walkies") + toleratedBy: Cat @relation(name:"tolerates") + feeds: Cat @primary @relation(name:"feeds") + } + type Dog { + host: User @relation(name:"hosts") + walker: User @primary @relation(name:"walkies") + } + type Cat { + loves: Mouse @primary @relation(name:"loves") + hatedBy: Mouse @relation(name:"hates") + tolerates: User @primary @relation(name:"tolerates") + fedBy: User @relation(name:"feeds") + } + type Mouse { + lovedBy: Cat @relation(name:"loves") + hates: Cat @primary @relation(name:"hates") + } + `, + }, + testUtils.GetSchema{ + ExpectedResults: []client.SchemaDescription{ + { + Name: "Cat", + Root: "bafkreibykyk7nm7hbh44rnyqc6glt7d73dpnn3ttwmichwdqydiajjh3ea-0", + VersionID: "bafkreibykyk7nm7hbh44rnyqc6glt7d73dpnn3ttwmichwdqydiajjh3ea-0", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "loves", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("2", false), + }, + { + Name: "loves_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + { + Name: "tolerates", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("3", false), + }, + { + Name: "tolerates_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "Dog", + Root: "bafkreibykyk7nm7hbh44rnyqc6glt7d73dpnn3ttwmichwdqydiajjh3ea-1", + VersionID: "bafkreibykyk7nm7hbh44rnyqc6glt7d73dpnn3ttwmichwdqydiajjh3ea-1", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "walker", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("3", false), + }, + { + Name: "walker_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "Mouse", + Root: "bafkreibykyk7nm7hbh44rnyqc6glt7d73dpnn3ttwmichwdqydiajjh3ea-2", + VersionID: "bafkreibykyk7nm7hbh44rnyqc6glt7d73dpnn3ttwmichwdqydiajjh3ea-2", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "hates", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("0", false), + }, + { + Name: "hates_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "User", + Root: "bafkreibykyk7nm7hbh44rnyqc6glt7d73dpnn3ttwmichwdqydiajjh3ea-3", + VersionID: "bafkreibykyk7nm7hbh44rnyqc6glt7d73dpnn3ttwmichwdqydiajjh3ea-3", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "feeds", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("0", false), + }, + { + Name: "feeds_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + { + Name: "hosts", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("1", false), + }, + { + Name: "hosts_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestSchemaSelfReferenceTwoPairsOfTwoTypesJoinedByThirdCircleAcrossAll_SchemasAllHaveSameBaseSchemaID(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + // - User and Dog form a circular dependency. + // - Cat and Mouse form a another circular dependency. + // - A larger circle is formed by bridging the two (User/Dog and Cat/Mouse) circles + // at different points in the same direction - this circle forms from + // User=>Dog=>Mouse=>Cat=>User=>etc. This test ensures that the two independent circles do not + // confuse the code into ignoring the larger circle. + Schema: ` + type User { + hosts: Dog @primary @relation(name:"hosts") + walks: Dog @relation(name:"walkies") + toleratedBy: Cat @relation(name:"tolerates") + } + type Dog { + host: User @relation(name:"hosts") + walker: User @primary @relation(name:"walkies") + licks: Mouse @primary @relation(name:"licks") + } + type Cat { + loves: Mouse @primary @relation(name:"loves") + hatedBy: Mouse @relation(name:"hates") + tolerates: User @primary @relation(name:"tolerates") + } + type Mouse { + lovedBy: Cat @relation(name:"loves") + hates: Cat @primary @relation(name:"hates") + lickedBy: Dog @relation(name:"licks") + } + `, + }, + testUtils.GetSchema{ + ExpectedResults: []client.SchemaDescription{ + { + Name: "Cat", + Root: "bafkreidetmki4jtod5jfmromvcz2vd75j6t6g3vnw3aenlv7znludye4ru-0", + VersionID: "bafkreidetmki4jtod5jfmromvcz2vd75j6t6g3vnw3aenlv7znludye4ru-0", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "loves", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("2", false), + }, + { + Name: "loves_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + { + Name: "tolerates", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("3", false), + }, + { + Name: "tolerates_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "Dog", + Root: "bafkreidetmki4jtod5jfmromvcz2vd75j6t6g3vnw3aenlv7znludye4ru-1", + VersionID: "bafkreidetmki4jtod5jfmromvcz2vd75j6t6g3vnw3aenlv7znludye4ru-1", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "licks", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("2", false), + }, + { + Name: "licks_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + { + Name: "walker", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("3", false), + }, + { + Name: "walker_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "Mouse", + Root: "bafkreidetmki4jtod5jfmromvcz2vd75j6t6g3vnw3aenlv7znludye4ru-2", + VersionID: "bafkreidetmki4jtod5jfmromvcz2vd75j6t6g3vnw3aenlv7znludye4ru-2", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "hates", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("0", false), + }, + { + Name: "hates_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + { + Name: "User", + Root: "bafkreidetmki4jtod5jfmromvcz2vd75j6t6g3vnw3aenlv7znludye4ru-3", + VersionID: "bafkreidetmki4jtod5jfmromvcz2vd75j6t6g3vnw3aenlv7znludye4ru-3", + Fields: []client.SchemaFieldDescription{ + { + Name: "_docID", + Typ: client.NONE_CRDT, + Kind: client.FieldKind_DocID, + }, + { + Name: "hosts", + Typ: client.LWW_REGISTER, + Kind: client.NewSelfKind("1", false), + }, + { + Name: "hosts_id", + Typ: client.LWW_REGISTER, + Kind: client.FieldKind_DocID, + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/schema/updates/add/field/kind/foreign_object_array_test.go b/tests/integration/schema/updates/add/field/kind/foreign_object_array_test.go index abeff648fd..c0330e2ecf 100644 --- a/tests/integration/schema/updates/add/field/kind/foreign_object_array_test.go +++ b/tests/integration/schema/updates/add/field/kind/foreign_object_array_test.go @@ -35,7 +35,7 @@ func TestSchemaUpdatesAddFieldKindForeignObjectArray_UnknownSchema(t *testing.T) }} ] `, - ExpectedError: "no type found for given name. Field: foo, Kind: Unknown", + ExpectedError: "no type found for given name. Field: foo, Kind: [Unknown]", }, }, } diff --git a/tests/integration/schema/updates/remove/simple_test.go b/tests/integration/schema/updates/remove/simple_test.go index 6628ccde86..44d331c98b 100644 --- a/tests/integration/schema/updates/remove/simple_test.go +++ b/tests/integration/schema/updates/remove/simple_test.go @@ -34,7 +34,7 @@ func TestSchemaUpdatesRemoveCollectionNameErrors(t *testing.T) { { "op": "remove", "path": "/Users/Name" } ] `, - ExpectedError: "SchemaRoot does not match existing. Name: ", + ExpectedError: "schema name can't be empty", }, }, } @@ -120,7 +120,7 @@ func TestSchemaUpdatesRemoveSchemaNameErrors(t *testing.T) { { "op": "remove", "path": "/Users/Name" } ] `, - ExpectedError: "SchemaRoot does not match existing. Name: ", + ExpectedError: "schema name can't be empty", }, }, } diff --git a/tests/predefined/gen_predefined.go b/tests/predefined/gen_predefined.go index 647d878f82..99c1862d48 100644 --- a/tests/predefined/gen_predefined.go +++ b/tests/predefined/gen_predefined.go @@ -11,31 +11,13 @@ package predefined import ( - "context" "strings" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/client/request" - "github.com/sourcenetwork/defradb/internal/request/graphql" "github.com/sourcenetwork/defradb/tests/gen" ) -func parseSDL(gqlSDL string) (map[string]client.CollectionDefinition, error) { - parser, err := graphql.NewParser() - if err != nil { - return nil, err - } - cols, err := parser.ParseSDL(context.Background(), gqlSDL) - if err != nil { - return nil, err - } - result := make(map[string]client.CollectionDefinition) - for _, col := range cols { - result[col.Description.Name.Value()] = col - } - return result, nil -} - // CreateFromSDL generates documents for GraphQL SDL from a predefined list // of docs that might include nested docs. // The SDL is parsed to get the list of fields, and the docs @@ -44,11 +26,21 @@ func parseSDL(gqlSDL string) (map[string]client.CollectionDefinition, error) { // fields, and create SDLs with different fields from it. func CreateFromSDL(gqlSDL string, docsList DocsList) ([]gen.GeneratedDoc, error) { resultDocs := make([]gen.GeneratedDoc, 0, len(docsList.Docs)) - typeDefs, err := parseSDL(gqlSDL) + typeDefsByName, err := gen.ParseSDL(gqlSDL) if err != nil { return nil, err } - generator := docGenerator{types: typeDefs} + + defs := make([]client.CollectionDefinition, 0, len(typeDefsByName)) + for _, def := range typeDefsByName { + defs = append(defs, def) + } + + generator := docGenerator{ + types: typeDefsByName, + definitionCache: client.NewDefinitionCache(defs), + } + for _, doc := range docsList.Docs { docs, err := generator.generateRelatedDocs(doc, docsList.ColName) if err != nil { @@ -87,7 +79,12 @@ func Create(defs []client.CollectionDefinition, docsList DocsList) ([]gen.Genera for _, col := range defs { typeDefs[col.Description.Name.Value()] = col } - generator := docGenerator{types: typeDefs} + + generator := docGenerator{ + types: typeDefs, + definitionCache: client.NewDefinitionCache(defs), + } + for _, doc := range docsList.Docs { docs, err := generator.generateRelatedDocs(doc, docsList.ColName) if err != nil { @@ -99,7 +96,8 @@ func Create(defs []client.CollectionDefinition, docsList DocsList) ([]gen.Genera } type docGenerator struct { - types map[string]client.CollectionDefinition + types map[string]client.CollectionDefinition + definitionCache client.DefinitionCache } // toRequestedDoc removes the fields that are not in the schema of the collection. @@ -132,31 +130,31 @@ func (this *docGenerator) generatePrimary( result := []gen.GeneratedDoc{} requestedSecondary := toRequestedDoc(secDocMap, secType) for _, secDocField := range secType.GetFields() { - if secDocField.IsRelation() { + if secDocField.IsRelation() && secDocField.IsPrimaryRelation { if secDocMapField, hasField := secDocMap[secDocField.Name]; hasField { - if secDocField.IsPrimaryRelation { - primType := this.types[secDocField.Kind.Underlying()] - primDocMap, subResult, err := this.generatePrimary( - secDocMap[secDocField.Name].(map[string]any), &primType) - if err != nil { - return nil, nil, NewErrFailedToGenerateDoc(err) - } - primDoc, err := client.NewDocFromMap(primDocMap, primType) - if err != nil { - return nil, nil, NewErrFailedToGenerateDoc(err) - } - docID := primDoc.ID().String() - requestedSecondary[secDocField.Name+request.RelatedObjectID] = docID - subResult = append(subResult, gen.GeneratedDoc{Col: &primType, Doc: primDoc}) - result = append(result, subResult...) + primaryDef, _ := client.GetDefinition(this.definitionCache, *secType, secDocField.Kind) + primType := this.types[primaryDef.GetName()] - secondaryDocs, err := this.generateSecondaryDocs( - secDocMapField.(map[string]any), docID, &primType, secType.Description.Name.Value()) - if err != nil { - return nil, nil, err - } - result = append(result, secondaryDocs...) + primDocMap, subResult, err := this.generatePrimary( + secDocMap[secDocField.Name].(map[string]any), &primType) + if err != nil { + return nil, nil, NewErrFailedToGenerateDoc(err) + } + primDoc, err := client.NewDocFromMap(primDocMap, primType) + if err != nil { + return nil, nil, NewErrFailedToGenerateDoc(err) } + docID := primDoc.ID().String() + requestedSecondary[secDocField.Name+request.RelatedObjectID] = docID + subResult = append(subResult, gen.GeneratedDoc{Col: &primType, Doc: primDoc}) + result = append(result, subResult...) + + secondaryDocs, err := this.generateSecondaryDocs( + secDocMapField.(map[string]any), docID, &primType, secType.Description.Name.Value()) + if err != nil { + return nil, nil, err + } + result = append(result, secondaryDocs...) } } } @@ -186,6 +184,7 @@ func (this *docGenerator) generateRelatedDocs(docMap map[string]any, typeName st return nil, err } result = append(result, secondaryDocs...) + return result, nil } @@ -197,15 +196,16 @@ func (this *docGenerator) generateSecondaryDocs( ) ([]gen.GeneratedDoc, error) { result := []gen.GeneratedDoc{} for _, field := range primaryType.GetFields() { - if field.IsRelation() { + if field.IsRelation() && !field.IsPrimaryRelation { if _, hasProp := primaryDocMap[field.Name]; hasProp { - if !field.IsPrimaryRelation && - (parentTypeName == "" || parentTypeName != field.Kind.Underlying()) { + otherDef, _ := client.GetDefinition(this.definitionCache, *primaryType, field.Kind) + if parentTypeName == "" || parentTypeName != otherDef.GetName() { docs, err := this.generateSecondaryDocsForField( - primaryDocMap, primaryType.Description.Name.Value(), &field, docID) + primaryDocMap, *primaryType, &field, docID) if err != nil { return nil, err } + result = append(result, docs...) } } @@ -217,15 +217,19 @@ func (this *docGenerator) generateSecondaryDocs( // generateSecondaryDocsForField generates secondary docs for the given field of a primary doc. func (this *docGenerator) generateSecondaryDocsForField( primaryDoc map[string]any, - primaryTypeName string, + primaryType client.CollectionDefinition, relField *client.FieldDefinition, primaryDocID string, ) ([]gen.GeneratedDoc, error) { result := []gen.GeneratedDoc{} - relTypeDef := this.types[relField.Kind.Underlying()] + + relTypeDef, _ := client.GetDefinition(this.definitionCache, primaryType, relField.Kind) + primaryPropName := "" for _, relDocField := range relTypeDef.GetFields() { - if relDocField.Kind.Underlying() == primaryTypeName && relDocField.IsPrimaryRelation { + relDocDef, _ := client.GetDefinition(this.definitionCache, relTypeDef, relDocField.Kind) + + if relDocDef.GetName() == primaryType.GetName() && relDocField.IsPrimaryRelation { primaryPropName = relDocField.Name + request.RelatedObjectID switch relVal := primaryDoc[relField.Name].(type) { case []map[string]any: diff --git a/tests/predefined/gen_predefined_test.go b/tests/predefined/gen_predefined_test.go index 30cd446697..a32c261ce7 100644 --- a/tests/predefined/gen_predefined_test.go +++ b/tests/predefined/gen_predefined_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client/request" + "github.com/sourcenetwork/defradb/tests/gen" ) func TestGeneratePredefinedFromSchema_Simple(t *testing.T) { @@ -36,7 +37,7 @@ func TestGeneratePredefinedFromSchema_Simple(t *testing.T) { docs, err := CreateFromSDL(schema, docsList) assert.NoError(t, err) - colDefMap, err := parseSDL(schema) + colDefMap, err := gen.ParseSDL(schema) require.NoError(t, err) errorMsg := assertDocs(mustAddDocIDsToDocs(docsList.Docs, colDefMap["User"]), docs) @@ -60,7 +61,7 @@ func TestGeneratePredefinedFromSchema_StripExcessiveFields(t *testing.T) { }) assert.NoError(t, err) - colDefMap, err := parseSDL(schema) + colDefMap, err := gen.ParseSDL(schema) require.NoError(t, err) errorMsg := assertDocs(mustAddDocIDsToDocs([]map[string]any{ @@ -102,7 +103,7 @@ func TestGeneratePredefinedFromSchema_OneToOne(t *testing.T) { }) assert.NoError(t, err) - colDefMap, err := parseSDL(schema) + colDefMap, err := gen.ParseSDL(schema) require.NoError(t, err) userDocs := mustAddDocIDsToDocs([]map[string]any{ @@ -157,7 +158,7 @@ func TestGeneratePredefinedFromSchema_OneToOnePrimary(t *testing.T) { }) assert.NoError(t, err) - colDefMap, err := parseSDL(schema) + colDefMap, err := gen.ParseSDL(schema) require.NoError(t, err) userDocs := mustAddDocIDsToDocs([]map[string]any{ @@ -213,7 +214,7 @@ func TestGeneratePredefinedFromSchema_OneToOneToOnePrimary(t *testing.T) { }) assert.NoError(t, err) - colDefMap, err := parseSDL(schema) + colDefMap, err := gen.ParseSDL(schema) require.NoError(t, err) specsDoc := mustAddDocIDToDoc(map[string]any{"OS": "iOS"}, colDefMap["Specs"]) @@ -264,7 +265,7 @@ func TestGeneratePredefinedFromSchema_TwoPrimaryToOneMiddle(t *testing.T) { }) assert.NoError(t, err) - colDefMap, err := parseSDL(schema) + colDefMap, err := gen.ParseSDL(schema) require.NoError(t, err) specsDoc := mustAddDocIDToDoc(map[string]any{"OS": "iOS"}, colDefMap["Specs"]) @@ -313,7 +314,7 @@ func TestGeneratePredefinedFromSchema_OneToTwoPrimary(t *testing.T) { }) assert.NoError(t, err) - colDefMap, err := parseSDL(schema) + colDefMap, err := gen.ParseSDL(schema) require.NoError(t, err) deviceDoc := mustAddDocIDToDoc(map[string]any{"model": "iPhone"}, colDefMap["Device"]) @@ -364,7 +365,7 @@ func TestGeneratePredefinedFromSchema_TwoPrimaryToOneRoot(t *testing.T) { }) assert.NoError(t, err) - colDefMap, err := parseSDL(schema) + colDefMap, err := gen.ParseSDL(schema) require.NoError(t, err) deviceDoc := mustAddDocIDToDoc(map[string]any{"model": "iPhone"}, colDefMap["Device"])