From e58f2d69478b9d860629e69f98cd6ad8e4638737 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 12 Jun 2024 12:44:10 +0200 Subject: [PATCH 01/42] Add encryption package --- internal/core/crdt/lwwreg.go | 13 +++++- internal/db/collection.go | 7 +++- internal/db/config.go | 7 ++++ internal/db/db.go | 2 + internal/encryption/aes.go | 73 +++++++++++++++++++++++++++++++++ internal/encryption/context.go | 20 +++++++++ internal/encryption/document.go | 18 ++++++++ internal/merkle/crdt/counter.go | 4 +- internal/merkle/crdt/field.go | 8 ++++ internal/merkle/crdt/lwwreg.go | 13 +++++- 10 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 internal/encryption/aes.go create mode 100644 internal/encryption/context.go create mode 100644 internal/encryption/document.go create mode 100644 internal/merkle/crdt/field.go diff --git a/internal/core/crdt/lwwreg.go b/internal/core/crdt/lwwreg.go index edfff9ca05..7419360089 100644 --- a/internal/core/crdt/lwwreg.go +++ b/internal/core/crdt/lwwreg.go @@ -20,6 +20,7 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/db/base" + "github.com/sourcenetwork/defradb/internal/encryption" ) // LWWRegDelta is a single delta operation for an LWWRegister @@ -111,7 +112,17 @@ func (reg LWWRegister) Merge(ctx context.Context, delta core.Delta) error { return ErrMismatchedMergeType } - return reg.setValue(ctx, d.Data, d.GetPriority()) + data := d.Data + + if cipher, ok := encryption.TryGetContextDocEnc(ctx); ok { + var err error + data, err = cipher.Decrypt(string(d.DocID), 0, data) + if err != nil { + return err + } + } + + return reg.setValue(ctx, data, d.GetPriority()) } func (reg LWWRegister) setValue(ctx context.Context, val []byte, priority uint64) error { diff --git a/internal/db/collection.go b/internal/db/collection.go index 7e20f0da8f..635bb8e242 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -33,6 +33,7 @@ import ( "github.com/sourcenetwork/defradb/internal/db/base" "github.com/sourcenetwork/defradb/internal/db/description" "github.com/sourcenetwork/defradb/internal/db/fetcher" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/lens" merklecrdt "github.com/sourcenetwork/defradb/internal/merkle/crdt" ) @@ -586,6 +587,10 @@ func (c *collection) save( doc.Clean() }) + if c.db.isEncrypted { + ctx = encryption.NewContext(ctx) + } + // New batch transaction/store (optional/todo) // Ensute/Set doc object marker // Loop through doc values @@ -657,7 +662,7 @@ func (c *collection) save( return cid.Undef, err } - link, _, err := merkleCRDT.Save(ctx, val) + link, _, err := merkleCRDT.Save(ctx, &merklecrdt.Field{FieldValue: val}) if err != nil { return cid.Undef, err } diff --git a/internal/db/config.go b/internal/db/config.go index 8ce725ebd0..83b2017a90 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -28,3 +28,10 @@ func WithMaxRetries(num int) Option { db.maxTxnRetries = immutable.Some(num) } } + +// WithDocEncryption enables document encryption. +func WithEnableDocEncryption(enable bool) Option { + return func(db *db) { + db.isEncrypted = enable + } +} diff --git a/internal/db/db.go b/internal/db/db.go index 197ab493d5..314bba8d7d 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -66,6 +66,8 @@ type db struct { // The maximum number of retries per transaction. maxTxnRetries immutable.Option[int] + + isEncrypted bool // The options used to init the database options []Option diff --git a/internal/encryption/aes.go b/internal/encryption/aes.go new file mode 100644 index 0000000000..43d4b6d48f --- /dev/null +++ b/internal/encryption/aes.go @@ -0,0 +1,73 @@ +package encryption + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "fmt" + "io" +) + +const nonceLength = 12 + +// EncryptAES encrypts data using AES-GCM with a provided key. +func EncryptAES(plainText, key []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + nonce := make([]byte, nonceLength) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + aesGCM, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + cipherText := aesGCM.Seal(nonce, nonce, plainText, nil) + + buf := make([]byte, base64.StdEncoding.EncodedLen(len(cipherText))) + base64.StdEncoding.Encode(buf, cipherText) + + return buf, nil +} + +// DecryptAES decrypts AES-GCM encrypted data with a provided key. +func DecryptAES(cipherTextBase64, key []byte) ([]byte, error) { + cipherText := make([]byte, base64.StdEncoding.DecodedLen(len(cipherTextBase64))) + n, err := base64.StdEncoding.Decode(cipherText, []byte(cipherTextBase64)) + + if err != nil { + return nil, err + } + + cipherText = cipherText[:n] + + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + if len(cipherText) < nonceLength { + return nil, fmt.Errorf("cipherText too short") + } + + nonce := cipherText[:nonceLength] + cipherText = cipherText[nonceLength:] + + aesGCM, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + + plainText, err := aesGCM.Open(nil, nonce, cipherText, nil) + if err != nil { + return nil, err + } + + return plainText, nil +} diff --git a/internal/encryption/context.go b/internal/encryption/context.go new file mode 100644 index 0000000000..19284635aa --- /dev/null +++ b/internal/encryption/context.go @@ -0,0 +1,20 @@ +package encryption + +import "context" + +// docEncContextKey is the key type for document encryption context values. +type docEncContextKey struct{} + +// TryGetContextDocEnc returns a document encryption and a bool indicating if +// it was retrieved from the given context. +func TryGetContextDocEnc(ctx context.Context) (*DocCipher, bool) { + d, ok := ctx.Value(docEncContextKey{}).(*DocCipher) + return d, ok +} + +// NewContext returns a new context with the document encryption value set. +// +// This will overwrite any previously set transaction value. +func NewContext(ctx context.Context) context.Context { + return context.WithValue(ctx, docEncContextKey{}, NewDocCipher()) +} diff --git a/internal/encryption/document.go b/internal/encryption/document.go new file mode 100644 index 0000000000..9c5a40fde9 --- /dev/null +++ b/internal/encryption/document.go @@ -0,0 +1,18 @@ +package encryption + +type DocCipher struct { +} + +const testKey = "examplekey1234567890examplekey12" + +func NewDocCipher() *DocCipher { + return &DocCipher{} +} + +func (d *DocCipher) Encrypt(docID string, fieldID int, plainText []byte) ([]byte, error) { + return EncryptAES(plainText, []byte(testKey)) +} + +func (d *DocCipher) Decrypt(docID string, fieldID int, cipherText []byte) ([]byte, error) { + return DecryptAES(cipherText, []byte(testKey)) +} diff --git a/internal/merkle/crdt/counter.go b/internal/merkle/crdt/counter.go index 2553dcfd2f..7038097483 100644 --- a/internal/merkle/crdt/counter.go +++ b/internal/merkle/crdt/counter.go @@ -49,11 +49,11 @@ func NewMerkleCounter( // Save the value of the Counter to the DAG. func (mc *MerkleCounter) Save(ctx context.Context, data any) (cidlink.Link, []byte, error) { - value, ok := data.(*client.FieldValue) + value, ok := data.(*Field) if !ok { return cidlink.Link{}, nil, NewErrUnexpectedValueType(mc.reg.CType(), &client.FieldValue{}, data) } - bytes, err := value.Bytes() + bytes, err := value.FieldValue.Bytes() if err != nil { return cidlink.Link{}, nil, err } diff --git a/internal/merkle/crdt/field.go b/internal/merkle/crdt/field.go new file mode 100644 index 0000000000..cd26e70d28 --- /dev/null +++ b/internal/merkle/crdt/field.go @@ -0,0 +1,8 @@ +package merklecrdt + +import "github.com/sourcenetwork/defradb/client" + +type Field struct { + DocID string + FieldValue *client.FieldValue +} diff --git a/internal/merkle/crdt/lwwreg.go b/internal/merkle/crdt/lwwreg.go index b8132ccad5..1d3a32bcd7 100644 --- a/internal/merkle/crdt/lwwreg.go +++ b/internal/merkle/crdt/lwwreg.go @@ -18,6 +18,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/internal/core" corecrdt "github.com/sourcenetwork/defradb/internal/core/crdt" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/merkle/clock" ) @@ -47,14 +48,22 @@ func NewMerkleLWWRegister( // Save the value of the register to the DAG. func (mlwwreg *MerkleLWWRegister) Save(ctx context.Context, data any) (cidlink.Link, []byte, error) { - value, ok := data.(*client.FieldValue) + value, ok := data.(*Field) if !ok { return cidlink.Link{}, nil, NewErrUnexpectedValueType(client.LWW_REGISTER, &client.FieldValue{}, data) } - bytes, err := value.Bytes() + bytes, err := value.FieldValue.Bytes() if err != nil { return cidlink.Link{}, nil, err } + + if cipher, ok := encryption.TryGetContextDocEnc(ctx); ok { + bytes, err = cipher.Encrypt(value.DocID, 0, bytes) + if err != nil { + return cidlink.Link{}, nil, err + } + } + // Set() call on underlying LWWRegister CRDT // persist/publish delta delta := mlwwreg.reg.Set(bytes) From 107a7f31abf684fb5799440eaedc398e09dc239a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 13 Jun 2024 18:34:56 +0200 Subject: [PATCH 02/42] Pass enc key upon doc creation --- cli/collection_create.go | 18 +++- cli/utils.go | 13 +++ internal/db/collection.go | 5 - internal/encryption/aes.go | 8 +- internal/encryption/context.go | 13 ++- internal/encryption/document.go | 11 ++- internal/encryption/nonce.go | 41 ++++++++ tests/integration/encryption/commit_test.go | 104 ++++++++++++++++++++ tests/integration/encryption/query_test.go | 58 +++++++++++ tests/integration/encryption/utils.go | 29 ++++++ tests/integration/test_case.go | 5 + tests/integration/utils2.go | 21 ++-- 12 files changed, 298 insertions(+), 28 deletions(-) create mode 100644 internal/encryption/nonce.go create mode 100644 tests/integration/encryption/commit_test.go create mode 100644 tests/integration/encryption/query_test.go create mode 100644 tests/integration/encryption/utils.go diff --git a/cli/collection_create.go b/cli/collection_create.go index 994911a14c..ace3ff023f 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -21,10 +21,20 @@ import ( func MakeCollectionCreateCommand() *cobra.Command { var file string + var encryptionKey string var cmd = &cobra.Command{ - Use: "create [-i --identity] ", + Use: "create [-i --identity] [-e --encryption] ", Short: "Create a new document.", Long: `Create a new document. + +Options: + -i, --identity + Marks the document as private and set the identity as the owner. The access to the document + and permissions are controlled by ACP (Access Control Policy). + + -e, --encryption + Encrypts the document with the encryption key. The encryption key is used to encrypt and decrypt + the document using symmetric AES-GCM encryption algorithm. Example: create from string: defradb client collection create --name User '{ "name": "Bob" }' @@ -69,6 +79,10 @@ Example: create from stdin: return cmd.Usage() } + if encryptionKey != "" { + setContextDocEncryptionKey(cmd, encryptionKey) + } + if client.IsJSONArray(docData) { docs, err := client.NewDocsFromJSON(docData, col.Definition()) if err != nil { @@ -84,6 +98,8 @@ Example: create from stdin: return col.Create(cmd.Context(), doc) }, } + cmd.PersistentFlags().StringVarP(&encryptionKey, "encryption", "e", "", + "Encryption key used to encrypt/decrypt the document") cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)") return cmd } diff --git a/cli/utils.go b/cli/utils.go index d1ee09962b..74e26d83bb 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -27,6 +27,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/internal/db" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/keyring" ) @@ -49,6 +50,8 @@ var ( // If a transaction exists, all operations will be executed // in the current transaction context. colContextKey = contextKey("col") + // docEncContextKey is the context key for the document encryption key. + docEncContextKey = contextKey("docEnc") ) // readPassword reads a user input password without echoing it to the terminal. @@ -160,6 +163,16 @@ func setContextIdentity(cmd *cobra.Command, privateKeyHex string) error { return nil } +// setContextIdentity sets the identity for the current command context. +func setContextDocEncryptionKey(cmd *cobra.Command, docEncryptionKey string) { + if docEncryptionKey == "" { + return + } + encryption.NewContext(cmd.Context()) + ctx := context.WithValue(cmd.Context(), docEncContextKey, docEncryptionKey) + cmd.SetContext(ctx) +} + // setContextRootDir sets the rootdir for the current command context. func setContextRootDir(cmd *cobra.Command) error { rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir") diff --git a/internal/db/collection.go b/internal/db/collection.go index 635bb8e242..e2408006cf 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -33,7 +33,6 @@ import ( "github.com/sourcenetwork/defradb/internal/db/base" "github.com/sourcenetwork/defradb/internal/db/description" "github.com/sourcenetwork/defradb/internal/db/fetcher" - "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/lens" merklecrdt "github.com/sourcenetwork/defradb/internal/merkle/crdt" ) @@ -587,10 +586,6 @@ func (c *collection) save( doc.Clean() }) - if c.db.isEncrypted { - ctx = encryption.NewContext(ctx) - } - // New batch transaction/store (optional/todo) // Ensute/Set doc object marker // Loop through doc values diff --git a/internal/encryption/aes.go b/internal/encryption/aes.go index 43d4b6d48f..b698806d27 100644 --- a/internal/encryption/aes.go +++ b/internal/encryption/aes.go @@ -3,14 +3,10 @@ package encryption import ( "crypto/aes" "crypto/cipher" - "crypto/rand" "encoding/base64" "fmt" - "io" ) -const nonceLength = 12 - // EncryptAES encrypts data using AES-GCM with a provided key. func EncryptAES(plainText, key []byte) ([]byte, error) { block, err := aes.NewCipher(key) @@ -18,8 +14,8 @@ func EncryptAES(plainText, key []byte) ([]byte, error) { return nil, err } - nonce := make([]byte, nonceLength) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + nonce, err := generateNonceFunc() + if err != nil { return nil, err } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index 19284635aa..3d94d0a1ae 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -12,9 +12,12 @@ func TryGetContextDocEnc(ctx context.Context) (*DocCipher, bool) { return d, ok } -// NewContext returns a new context with the document encryption value set. -// -// This will overwrite any previously set transaction value. -func NewContext(ctx context.Context) context.Context { - return context.WithValue(ctx, docEncContextKey{}, NewDocCipher()) +func SetDocEncContext(ctx context.Context, encryptionKey string) context.Context { + cipher, ok := TryGetContextDocEnc(ctx) + if !ok { + cipher = NewDocCipher() + ctx = context.WithValue(ctx, docEncContextKey{}, cipher) + } + cipher.setKey(encryptionKey) + return ctx } diff --git a/internal/encryption/document.go b/internal/encryption/document.go index 9c5a40fde9..bde2c5c332 100644 --- a/internal/encryption/document.go +++ b/internal/encryption/document.go @@ -1,18 +1,21 @@ package encryption type DocCipher struct { + encryptionKey string } -const testKey = "examplekey1234567890examplekey12" - func NewDocCipher() *DocCipher { return &DocCipher{} } +func (d *DocCipher) setKey(encryptionKey string) { + d.encryptionKey = encryptionKey +} + func (d *DocCipher) Encrypt(docID string, fieldID int, plainText []byte) ([]byte, error) { - return EncryptAES(plainText, []byte(testKey)) + return EncryptAES(plainText, []byte(d.encryptionKey)) } func (d *DocCipher) Decrypt(docID string, fieldID int, cipherText []byte) ([]byte, error) { - return DecryptAES(cipherText, []byte(testKey)) + return DecryptAES(cipherText, []byte(d.encryptionKey)) } diff --git a/internal/encryption/nonce.go b/internal/encryption/nonce.go new file mode 100644 index 0000000000..eda0d0be5a --- /dev/null +++ b/internal/encryption/nonce.go @@ -0,0 +1,41 @@ +package encryption + +import ( + "crypto/rand" + "errors" + "io" + "os" + "strings" +) + +const nonceLength = 12 + +var generateNonceFunc = generateNonce + +func generateNonce() ([]byte, error) { + nonce := make([]byte, nonceLength) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + return nonce, nil +} + +// generateTestNonce generates a deterministic nonce for testing. +func generateTestNonce() ([]byte, error) { + nonce := []byte("deterministic nonce for testing") + + if len(nonce) < nonceLength { + return nil, errors.New("nonce length is longer than available deterministic nonce") + } + + return nonce[:nonceLength], nil +} + +func init() { + arg := os.Args[0] + // If the binary is a test binary, use a deterministic nonce. + if strings.HasSuffix(arg, ".test") || strings.Contains(arg, "/defradb/tests/") { + generateNonceFunc = generateTestNonce + } +} diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go new file mode 100644 index 0000000000..2d86fa447c --- /dev/null +++ b/tests/integration/encryption/commit_test.go @@ -0,0 +1,104 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + "github.com/sourcenetwork/defradb/internal/encryption" + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/sourcenetwork/immutable" +) + +const encKey = "examplekey1234567890examplekey12" + +func encryptAES(key string, plaintext []byte) []byte { + val, _ := encryption.EncryptAES(plaintext, []byte(key)) + return val +} + +func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + updateUserCollectionSchema(), + testUtils.CreateDoc{ + Doc: `{ + "name": "John", + "age": 21 + }`, + EncryptionKey: immutable.Some(encKey), + }, + testUtils.Request{ + Request: ` + query { + commits { + cid + collectionID + delta + docID + fieldId + fieldName + height + links { + cid + name + } + } + } + `, + Results: []map[string]any{ + { + "cid": "bafyreicv422zhiuqefs32wp7glrqsbjpy76hgem4ivagm2ttuli43wluci", + "collectionID": int64(1), + "delta": encryptAES(encKey, testUtils.CBORValue(21)), + "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "fieldId": "1", + "fieldName": "age", + "height": int64(1), + "links": []map[string]any{}, + }, + { + "cid": "bafyreie6i4dw5jh6bp2anszqkmuwfslsemzatrflipetljhtpjhjn3zbum", + "collectionID": int64(1), + "delta": encryptAES(encKey, testUtils.CBORValue("John")), + "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "fieldId": "2", + "fieldName": "name", + "height": int64(1), + "links": []map[string]any{}, + }, + { + "cid": "bafyreia747gvxxbowag2mob2up34zwh364olc7ocab3nunj2ikdxq7srom", + "collectionID": int64(1), + "delta": nil, + "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "fieldId": "C", + "fieldName": nil, + "height": int64(1), + "links": []map[string]any{ + { + "cid": "bafyreicv422zhiuqefs32wp7glrqsbjpy76hgem4ivagm2ttuli43wluci", + "name": "age", + }, + { + "cid": "bafyreie6i4dw5jh6bp2anszqkmuwfslsemzatrflipetljhtpjhjn3zbum", + "name": "name", + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + diff --git a/tests/integration/encryption/query_test.go b/tests/integration/encryption/query_test.go new file mode 100644 index 0000000000..8e1514f427 --- /dev/null +++ b/tests/integration/encryption/query_test.go @@ -0,0 +1,58 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/sourcenetwork/immutable" +) + +func TestDocEncryption_WithEncryption_ShouldFetchDecrypted(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + age: Int + } + `}, + testUtils.CreateDoc{ + Doc: `{ + "name": "John", + "age": 21 + }`, + EncryptionKey: immutable.Some("examplekey1234567890examplekey12"), + }, + testUtils.Request{ + Request: ` + query { + Users { + _docID + name + age + } + }`, + Results: []map[string]any{ + { + "_docID": "bae-0b2f15e5-bfe7-5cb7-8045-471318d7dbc3", + "name": "John", + "age": int64(21), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/utils.go b/tests/integration/encryption/utils.go new file mode 100644 index 0000000000..6bcb7acc83 --- /dev/null +++ b/tests/integration/encryption/utils.go @@ -0,0 +1,29 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +const userCollectionGQLSchema = (` + type Users { + name: String + age: Int + verified: Boolean + } +`) + +func updateUserCollectionSchema() testUtils.SchemaUpdate { + return testUtils.SchemaUpdate{ + Schema: userCollectionGQLSchema, + } +} diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index 4536c0cd0a..deaef1d04d 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -230,6 +230,11 @@ type CreateDoc struct { // created document(s) will be owned by this Identity. Identity immutable.Option[acpIdentity.Identity] + // The encryption key to use for the document encryption. Optional. + // + // If an EncryptionKey is not provided the document will not be encrypted. + EncryptionKey immutable.Option[string] + // The collection in which this document should be created. CollectionID int diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index fab8cc5ed9..7a1f8be98b 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -34,6 +34,7 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/internal/db" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/request/graphql" "github.com/sourcenetwork/defradb/net" changeDetector "github.com/sourcenetwork/defradb/tests/change_detector" @@ -882,7 +883,8 @@ func refreshDocuments( continue } - ctx := db.SetContextIdentity(s.ctx, action.Identity) + ctx := makeContextForDocCreate(s.ctx, &action) + // The document may have been mutated by other actions, so to be sure we have the latest // version without having to worry about the individual update mechanics we fetch it. doc, err = collection.Get(ctx, doc.ID(), false) @@ -1226,12 +1228,19 @@ func createDocViaColSave( txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := db.SetContextTxn(s.ctx, txn) - ctx = db.SetContextIdentity(ctx, action.Identity) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) return doc, collections[action.CollectionID].Save(ctx, doc) } +func makeContextForDocCreate(ctx context.Context, action *CreateDoc) context.Context { + ctx = db.SetContextIdentity(ctx, action.Identity) + if action.EncryptionKey.HasValue() { + ctx = encryption.SetDocEncContext(ctx, action.EncryptionKey.Value()) + } + return ctx +} + func createDocViaColCreate( s *state, action CreateDoc, @@ -1251,8 +1260,7 @@ func createDocViaColCreate( txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := db.SetContextTxn(s.ctx, txn) - ctx = db.SetContextIdentity(ctx, action.Identity) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) return doc, collections[action.CollectionID].Create(ctx, doc) } @@ -1286,8 +1294,7 @@ func createDocViaGQL( txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := db.SetContextTxn(s.ctx, txn) - ctx = db.SetContextIdentity(ctx, action.Identity) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) result := node.ExecRequest( ctx, From 3bc618b37577dd6ea11256b0d813f5d264da16a1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 14 Jun 2024 10:53:08 +0200 Subject: [PATCH 03/42] Add license --- internal/encryption/aes.go | 10 ++++++++++ internal/encryption/context.go | 10 ++++++++++ internal/encryption/document.go | 10 ++++++++++ internal/encryption/nonce.go | 10 ++++++++++ 4 files changed, 40 insertions(+) diff --git a/internal/encryption/aes.go b/internal/encryption/aes.go index b698806d27..e3a7feb563 100644 --- a/internal/encryption/aes.go +++ b/internal/encryption/aes.go @@ -1,3 +1,13 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package encryption import ( diff --git a/internal/encryption/context.go b/internal/encryption/context.go index 3d94d0a1ae..f214fae4a0 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -1,3 +1,13 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package encryption import "context" diff --git a/internal/encryption/document.go b/internal/encryption/document.go index bde2c5c332..979d79dbdb 100644 --- a/internal/encryption/document.go +++ b/internal/encryption/document.go @@ -1,3 +1,13 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package encryption type DocCipher struct { diff --git a/internal/encryption/nonce.go b/internal/encryption/nonce.go index eda0d0be5a..8f67c8d958 100644 --- a/internal/encryption/nonce.go +++ b/internal/encryption/nonce.go @@ -1,3 +1,13 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package encryption import ( From 09c538ef747fc00add6d0aacbeb86a3e6d9cca9f Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 14 Jun 2024 11:55:53 +0200 Subject: [PATCH 04/42] Slight restructure --- cli/utils.go | 5 +-- internal/core/crdt/lwwreg.go | 10 ++--- internal/encryption/context.go | 42 +++++++++++++++---- .../encryption/{document.go => encryptor.go} | 12 +++--- internal/merkle/crdt/lwwreg.go | 8 ++-- tests/integration/encryption/commit_test.go | 6 +-- tests/integration/utils2.go | 2 +- 7 files changed, 52 insertions(+), 33 deletions(-) rename internal/encryption/{document.go => encryptor.go} (63%) diff --git a/cli/utils.go b/cli/utils.go index 74e26d83bb..2231afabfd 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -50,8 +50,6 @@ var ( // If a transaction exists, all operations will be executed // in the current transaction context. colContextKey = contextKey("col") - // docEncContextKey is the context key for the document encryption key. - docEncContextKey = contextKey("docEnc") ) // readPassword reads a user input password without echoing it to the terminal. @@ -168,8 +166,7 @@ func setContextDocEncryptionKey(cmd *cobra.Command, docEncryptionKey string) { if docEncryptionKey == "" { return } - encryption.NewContext(cmd.Context()) - ctx := context.WithValue(cmd.Context(), docEncContextKey, docEncryptionKey) + ctx := encryption.ContextWithKey(cmd.Context(), docEncryptionKey) cmd.SetContext(ctx) } diff --git a/internal/core/crdt/lwwreg.go b/internal/core/crdt/lwwreg.go index 7419360089..15baf4060f 100644 --- a/internal/core/crdt/lwwreg.go +++ b/internal/core/crdt/lwwreg.go @@ -114,12 +114,10 @@ func (reg LWWRegister) Merge(ctx context.Context, delta core.Delta) error { data := d.Data - if cipher, ok := encryption.TryGetContextDocEnc(ctx); ok { - var err error - data, err = cipher.Decrypt(string(d.DocID), 0, data) - if err != nil { - return err - } + var err error + data, err = encryption.DecryptDoc(ctx, string(d.DocID), 0, data) + if err != nil { + return err } return reg.setValue(ctx, data, d.GetPriority()) diff --git a/internal/encryption/context.go b/internal/encryption/context.go index f214fae4a0..14d49e8d0c 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -17,17 +17,43 @@ type docEncContextKey struct{} // TryGetContextDocEnc returns a document encryption and a bool indicating if // it was retrieved from the given context. -func TryGetContextDocEnc(ctx context.Context) (*DocCipher, bool) { - d, ok := ctx.Value(docEncContextKey{}).(*DocCipher) - return d, ok +func TryGetContextEncryptor(ctx context.Context) (*DocEncryptor, bool) { + enc, ok := ctx.Value(docEncContextKey{}).(*DocEncryptor) + return enc, ok } -func SetDocEncContext(ctx context.Context, encryptionKey string) context.Context { - cipher, ok := TryGetContextDocEnc(ctx) +func getContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { + enc, ok := TryGetContextEncryptor(ctx) if !ok { - cipher = NewDocCipher() - ctx = context.WithValue(ctx, docEncContextKey{}, cipher) + enc = newDocEncryptor() + ctx = context.WithValue(ctx, docEncContextKey{}, enc) } - cipher.setKey(encryptionKey) + return ctx, enc +} + +func Context(ctx context.Context) context.Context { + ctx, _ = getContextWithDocEnc(ctx) return ctx } + +func ContextWithKey(ctx context.Context, encryptionKey string) context.Context { + _, encryptor := getContextWithDocEnc(ctx) + encryptor.SetKey(encryptionKey) + return context.WithValue(ctx, docEncContextKey{}, encryptor) +} + +func EncryptDoc(ctx context.Context, docID string, fieldID int, plainText []byte) ([]byte, error) { + enc, ok := TryGetContextEncryptor(ctx) + if !ok { + return plainText, nil + } + return enc.Encrypt(docID, fieldID, plainText) +} + +func DecryptDoc(ctx context.Context, docID string, fieldID int, cipherText []byte) ([]byte, error) { + enc, ok := TryGetContextEncryptor(ctx) + if !ok { + return cipherText, nil + } + return enc.Decrypt(docID, fieldID, cipherText) +} diff --git a/internal/encryption/document.go b/internal/encryption/encryptor.go similarity index 63% rename from internal/encryption/document.go rename to internal/encryption/encryptor.go index 979d79dbdb..873d9b0b90 100644 --- a/internal/encryption/document.go +++ b/internal/encryption/encryptor.go @@ -10,22 +10,22 @@ package encryption -type DocCipher struct { +type DocEncryptor struct { encryptionKey string } -func NewDocCipher() *DocCipher { - return &DocCipher{} +func newDocEncryptor() *DocEncryptor { + return &DocEncryptor{} } -func (d *DocCipher) setKey(encryptionKey string) { +func (d *DocEncryptor) SetKey(encryptionKey string) { d.encryptionKey = encryptionKey } -func (d *DocCipher) Encrypt(docID string, fieldID int, plainText []byte) ([]byte, error) { +func (d *DocEncryptor) Encrypt(docID string, fieldID int, plainText []byte) ([]byte, error) { return EncryptAES(plainText, []byte(d.encryptionKey)) } -func (d *DocCipher) Decrypt(docID string, fieldID int, cipherText []byte) ([]byte, error) { +func (d *DocEncryptor) Decrypt(docID string, fieldID int, cipherText []byte) ([]byte, error) { return DecryptAES(cipherText, []byte(d.encryptionKey)) } diff --git a/internal/merkle/crdt/lwwreg.go b/internal/merkle/crdt/lwwreg.go index 1d3a32bcd7..72b46fd845 100644 --- a/internal/merkle/crdt/lwwreg.go +++ b/internal/merkle/crdt/lwwreg.go @@ -57,11 +57,9 @@ func (mlwwreg *MerkleLWWRegister) Save(ctx context.Context, data any) (cidlink.L return cidlink.Link{}, nil, err } - if cipher, ok := encryption.TryGetContextDocEnc(ctx); ok { - bytes, err = cipher.Encrypt(value.DocID, 0, bytes) - if err != nil { - return cidlink.Link{}, nil, err - } + bytes, err = encryption.EncryptDoc(ctx, value.DocID, 0, bytes) + if err != nil { + return cidlink.Link{}, nil, err } // Set() call on underlying LWWRegister CRDT diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 2d86fa447c..96af1d6e38 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -20,7 +20,7 @@ import ( const encKey = "examplekey1234567890examplekey12" -func encryptAES(key string, plaintext []byte) []byte { +func encrypt(key string, plaintext []byte) []byte { val, _ := encryption.EncryptAES(plaintext, []byte(key)) return val } @@ -58,7 +58,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { { "cid": "bafyreicv422zhiuqefs32wp7glrqsbjpy76hgem4ivagm2ttuli43wluci", "collectionID": int64(1), - "delta": encryptAES(encKey, testUtils.CBORValue(21)), + "delta": encrypt(encKey, testUtils.CBORValue(21)), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", "fieldId": "1", "fieldName": "age", @@ -68,7 +68,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { { "cid": "bafyreie6i4dw5jh6bp2anszqkmuwfslsemzatrflipetljhtpjhjn3zbum", "collectionID": int64(1), - "delta": encryptAES(encKey, testUtils.CBORValue("John")), + "delta": encrypt(encKey, testUtils.CBORValue("John")), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", "fieldId": "2", "fieldName": "name", diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 7a1f8be98b..26788a58bd 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1236,7 +1236,7 @@ func createDocViaColSave( func makeContextForDocCreate(ctx context.Context, action *CreateDoc) context.Context { ctx = db.SetContextIdentity(ctx, action.Identity) if action.EncryptionKey.HasValue() { - ctx = encryption.SetDocEncContext(ctx, action.EncryptionKey.Value()) + ctx = encryption.ContextWithKey(ctx, action.EncryptionKey.Value()) } return ctx } From 9c8b79ef4692877d2a3f812ca0d041212b4e7497 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 14 Jun 2024 12:10:59 +0200 Subject: [PATCH 05/42] Add Encstore to Rootstore --- datastore/mocks/txn.go | 47 ++++++++++++++++++++++++++++++ datastore/multi.go | 8 +++++ datastore/store.go | 4 +++ http/client_tx.go | 4 +++ tests/bench/query/planner/utils.go | 1 + tests/clients/cli/wrapper_tx.go | 4 +++ tests/clients/http/wrapper_tx.go | 4 +++ 7 files changed, 72 insertions(+) diff --git a/datastore/mocks/txn.go b/datastore/mocks/txn.go index f29c045dcd..41606260ea 100644 --- a/datastore/mocks/txn.go +++ b/datastore/mocks/txn.go @@ -195,6 +195,53 @@ func (_c *Txn_Discard_Call) RunAndReturn(run func(context.Context)) *Txn_Discard return _c } +// Encstore provides a mock function with given fields: +func (_m *Txn) Encstore() datastore.DSReaderWriter { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Encstore") + } + + var r0 datastore.DSReaderWriter + if rf, ok := ret.Get(0).(func() datastore.DSReaderWriter); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(datastore.DSReaderWriter) + } + } + + return r0 +} + +// Txn_Encstore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Encstore' +type Txn_Encstore_Call struct { + *mock.Call +} + +// Encstore is a helper method to define mock.On call +func (_e *Txn_Expecter) Encstore() *Txn_Encstore_Call { + return &Txn_Encstore_Call{Call: _e.mock.On("Encstore")} +} + +func (_c *Txn_Encstore_Call) Run(run func()) *Txn_Encstore_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *Txn_Encstore_Call) Return(_a0 datastore.DSReaderWriter) *Txn_Encstore_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Txn_Encstore_Call) RunAndReturn(run func() datastore.DSReaderWriter) *Txn_Encstore_Call { + _c.Call.Return(run) + return _c +} + // Headstore provides a mock function with given fields: func (_m *Txn) Headstore() datastore.DSReaderWriter { ret := _m.Called() diff --git a/datastore/multi.go b/datastore/multi.go index a70a24a60d..f863924d5d 100644 --- a/datastore/multi.go +++ b/datastore/multi.go @@ -23,11 +23,13 @@ var ( headStoreKey = rootStoreKey.ChildString("heads") blockStoreKey = rootStoreKey.ChildString("blocks") peerStoreKey = rootStoreKey.ChildString("ps") + encStoreKey = rootStoreKey.ChildString("enc") ) type multistore struct { root DSReaderWriter data DSReaderWriter + enc DSReaderWriter head DSReaderWriter peer DSBatching system DSReaderWriter @@ -43,6 +45,7 @@ func MultiStoreFrom(rootstore ds.Datastore) MultiStore { ms := &multistore{ root: rootRW, data: prefix(rootRW, dataStoreKey), + enc: prefix(rootRW, encStoreKey), head: prefix(rootRW, headStoreKey), peer: namespace.Wrap(rootstore, peerStoreKey), system: prefix(rootRW, systemStoreKey), @@ -57,6 +60,11 @@ func (ms multistore) Datastore() DSReaderWriter { return ms.data } +// Encstore implements MultiStore. +func (ms multistore) Encstore() DSReaderWriter { + return ms.enc +} + // Headstore implements MultiStore. func (ms multistore) Headstore() DSReaderWriter { return ms.head diff --git a/datastore/store.go b/datastore/store.go index 66501270d1..b64d034e94 100644 --- a/datastore/store.go +++ b/datastore/store.go @@ -38,6 +38,10 @@ type MultiStore interface { // under the /data namespace Datastore() DSReaderWriter + // Encstore is a wrapped root DSReaderWriter + // under the /enc namespace + Encstore() DSReaderWriter + // Headstore is a wrapped root DSReaderWriter // under the /head namespace Headstore() DSReaderWriter diff --git a/http/client_tx.go b/http/client_tx.go index a804b934f1..5b99f5aaad 100644 --- a/http/client_tx.go +++ b/http/client_tx.go @@ -91,6 +91,10 @@ func (c *Transaction) Datastore() datastore.DSReaderWriter { panic("client side transaction") } +func (c *Transaction) Encstore() datastore.DSReaderWriter { + panic("client side transaction") +} + func (c *Transaction) Headstore() datastore.DSReaderWriter { panic("client side transaction") } diff --git a/tests/bench/query/planner/utils.go b/tests/bench/query/planner/utils.go index 5a842222f5..b2a6e3c0d6 100644 --- a/tests/bench/query/planner/utils.go +++ b/tests/bench/query/planner/utils.go @@ -134,6 +134,7 @@ type dummyTxn struct{} func (*dummyTxn) Rootstore() datastore.DSReaderWriter { return nil } func (*dummyTxn) Datastore() datastore.DSReaderWriter { return nil } +func (*dummyTxn) Encstore() datastore.DSReaderWriter { return nil } func (*dummyTxn) Headstore() datastore.DSReaderWriter { return nil } func (*dummyTxn) Peerstore() datastore.DSBatching { return nil } func (*dummyTxn) Blockstore() datastore.Blockstore { return nil } diff --git a/tests/clients/cli/wrapper_tx.go b/tests/clients/cli/wrapper_tx.go index 0330b8d47e..46aefd000d 100644 --- a/tests/clients/cli/wrapper_tx.go +++ b/tests/clients/cli/wrapper_tx.go @@ -75,6 +75,10 @@ func (w *Transaction) Datastore() datastore.DSReaderWriter { return w.tx.Datastore() } +func (w *Transaction) Encstore() datastore.DSReaderWriter { + return w.tx.Encstore() +} + func (w *Transaction) Headstore() datastore.DSReaderWriter { return w.tx.Headstore() } diff --git a/tests/clients/http/wrapper_tx.go b/tests/clients/http/wrapper_tx.go index 133d3bc1d3..e4b838a2e9 100644 --- a/tests/clients/http/wrapper_tx.go +++ b/tests/clients/http/wrapper_tx.go @@ -69,6 +69,10 @@ func (w *TxWrapper) Datastore() datastore.DSReaderWriter { return w.server.Datastore() } +func (w *TxWrapper) Encstore() datastore.DSReaderWriter { + return w.server.Encstore() +} + func (w *TxWrapper) Headstore() datastore.DSReaderWriter { return w.server.Headstore() } From 5d64820ce4f4c9cbf45afd0997624e6c2f18b3ab Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 16 Jun 2024 21:11:33 +0200 Subject: [PATCH 06/42] Pass store to encryptor --- cli/collection_create.go | 6 +++--- cli/utils.go | 6 +++++- internal/encryption/context.go | 16 +++++++++++++--- internal/encryption/encryptor.go | 7 +++++++ tests/integration/utils2.go | 17 ++++++++++++----- 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/cli/collection_create.go b/cli/collection_create.go index ace3ff023f..ed4be48224 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -17,6 +17,7 @@ import ( "github.com/spf13/cobra" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/internal/db" ) func MakeCollectionCreateCommand() *cobra.Command { @@ -79,9 +80,8 @@ Example: create from stdin: return cmd.Usage() } - if encryptionKey != "" { - setContextDocEncryptionKey(cmd, encryptionKey) - } + txn, _ := db.TryGetContextTxn(cmd.Context()) + setContextDocEncryptionKey(cmd, encryptionKey, txn) if client.IsJSONArray(docData) { docs, err := client.NewDocsFromJSON(docData, col.Definition()) diff --git a/cli/utils.go b/cli/utils.go index 2231afabfd..d6300a3944 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -25,6 +25,7 @@ import ( acpIdentity "github.com/sourcenetwork/defradb/acp/identity" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/http" "github.com/sourcenetwork/defradb/internal/db" "github.com/sourcenetwork/defradb/internal/encryption" @@ -162,11 +163,14 @@ func setContextIdentity(cmd *cobra.Command, privateKeyHex string) error { } // setContextIdentity sets the identity for the current command context. -func setContextDocEncryptionKey(cmd *cobra.Command, docEncryptionKey string) { +func setContextDocEncryptionKey(cmd *cobra.Command, docEncryptionKey string, txn datastore.Txn) { if docEncryptionKey == "" { return } ctx := encryption.ContextWithKey(cmd.Context(), docEncryptionKey) + if txn != nil { + ctx = encryption.ContextWithStore(ctx, txn) + } cmd.SetContext(ctx) } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index 14d49e8d0c..c2d7616896 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -10,7 +10,11 @@ package encryption -import "context" +import ( + "context" + + "github.com/sourcenetwork/defradb/datastore" +) // docEncContextKey is the key type for document encryption context values. type docEncContextKey struct{} @@ -37,9 +41,15 @@ func Context(ctx context.Context) context.Context { } func ContextWithKey(ctx context.Context, encryptionKey string) context.Context { - _, encryptor := getContextWithDocEnc(ctx) + ctx, encryptor := getContextWithDocEnc(ctx) encryptor.SetKey(encryptionKey) - return context.WithValue(ctx, docEncContextKey{}, encryptor) + return ctx +} + +func ContextWithStore(ctx context.Context, txn datastore.Txn) context.Context { + ctx, encryptor := getContextWithDocEnc(ctx) + encryptor.SetStore(txn.Encstore()) + return ctx } func EncryptDoc(ctx context.Context, docID string, fieldID int, plainText []byte) ([]byte, error) { diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 873d9b0b90..d31788a31e 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -10,8 +10,11 @@ package encryption +import "github.com/sourcenetwork/defradb/datastore" + type DocEncryptor struct { encryptionKey string + store datastore.DSReaderWriter } func newDocEncryptor() *DocEncryptor { @@ -22,6 +25,10 @@ func (d *DocEncryptor) SetKey(encryptionKey string) { d.encryptionKey = encryptionKey } +func (d *DocEncryptor) SetStore(store datastore.DSReaderWriter) { + d.store = store +} + func (d *DocEncryptor) Encrypt(docID string, fieldID int, plainText []byte) ([]byte, error) { return EncryptAES(plainText, []byte(d.encryptionKey)) } diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 26788a58bd..b830727622 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -883,7 +883,11 @@ func refreshDocuments( continue } - ctx := makeContextForDocCreate(s.ctx, &action) + txn := s.txns[0] + if action.NodeID.HasValue() { + txn = s.txns[action.NodeID.Value()] + } + ctx := makeContextForDocCreate(s.ctx, &action, txn) // The document may have been mutated by other actions, so to be sure we have the latest // version without having to worry about the individual update mechanics we fetch it. @@ -1228,16 +1232,19 @@ func createDocViaColSave( txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action, txn) return doc, collections[action.CollectionID].Save(ctx, doc) } -func makeContextForDocCreate(ctx context.Context, action *CreateDoc) context.Context { +func makeContextForDocCreate(ctx context.Context, action *CreateDoc, txn datastore.Txn) context.Context { ctx = db.SetContextIdentity(ctx, action.Identity) if action.EncryptionKey.HasValue() { ctx = encryption.ContextWithKey(ctx, action.EncryptionKey.Value()) } + if txn != nil { + ctx = encryption.ContextWithStore(ctx, txn) + } return ctx } @@ -1260,7 +1267,7 @@ func createDocViaColCreate( txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action, txn) return doc, collections[action.CollectionID].Create(ctx, doc) } @@ -1294,7 +1301,7 @@ func createDocViaGQL( txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action, txn) result := node.ExecRequest( ctx, From 579920b95520ea5c8c8526cd1eda1fc94808b6d6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 17 Jun 2024 09:50:01 +0200 Subject: [PATCH 07/42] Store encKey and read from storage --- cli/utils.go | 2 +- internal/core/key.go | 59 ++++++++--- internal/core/key_test.go | 2 +- internal/db/collection.go | 6 +- internal/db/context.go | 2 + internal/db/fetcher/fetcher.go | 6 +- internal/encryption/context.go | 8 +- internal/encryption/encryptor.go | 64 ++++++++++-- internal/planner/commit.go | 2 +- tests/integration/encryption/commit_test.go | 103 ++++++++++++++++++++ tests/integration/utils2.go | 2 +- 11 files changed, 219 insertions(+), 37 deletions(-) diff --git a/cli/utils.go b/cli/utils.go index d6300a3944..614f9d2ef5 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -167,7 +167,7 @@ func setContextDocEncryptionKey(cmd *cobra.Command, docEncryptionKey string, txn if docEncryptionKey == "" { return } - ctx := encryption.ContextWithKey(cmd.Context(), docEncryptionKey) + ctx := encryption.ContextWithKey(cmd.Context(), []byte(docEncryptionKey)) if txn != nil { ctx = encryption.ContextWithStore(ctx, txn) } diff --git a/internal/core/key.go b/internal/core/key.go index d087c43af8..efc2d73017 100644 --- a/internal/core/key.go +++ b/internal/core/key.go @@ -71,7 +71,7 @@ type DataStoreKey struct { CollectionRootID uint32 InstanceType InstanceType DocID string - FieldId string + FieldID string } var _ Key = (*DataStoreKey)(nil) @@ -238,7 +238,7 @@ func NewDataStoreKey(key string) (DataStoreKey, error) { dataStoreKey.InstanceType = InstanceType(elements[1]) dataStoreKey.DocID = elements[2] if numberOfElements == 4 { - dataStoreKey.FieldId = elements[3] + dataStoreKey.FieldID = elements[3] } return dataStoreKey, nil @@ -429,21 +429,21 @@ func (k DataStoreKey) WithDocID(docID string) DataStoreKey { func (k DataStoreKey) WithInstanceInfo(key DataStoreKey) DataStoreKey { newKey := k newKey.DocID = key.DocID - newKey.FieldId = key.FieldId + newKey.FieldID = key.FieldID newKey.InstanceType = key.InstanceType return newKey } func (k DataStoreKey) WithFieldId(fieldId string) DataStoreKey { newKey := k - newKey.FieldId = fieldId + newKey.FieldID = fieldId return newKey } func (k DataStoreKey) ToHeadStoreKey() HeadStoreKey { return HeadStoreKey{ DocID: k.DocID, - FieldId: k.FieldId, + FieldId: k.FieldID, } } @@ -477,8 +477,8 @@ func (k DataStoreKey) ToString() string { if k.DocID != "" { result = result + "/" + k.DocID } - if k.FieldId != "" { - result = result + "/" + k.FieldId + if k.FieldID != "" { + result = result + "/" + k.FieldID } return result @@ -495,7 +495,7 @@ func (k DataStoreKey) ToDS() ds.Key { func (k DataStoreKey) Equal(other DataStoreKey) bool { return k.CollectionRootID == other.CollectionRootID && k.DocID == other.DocID && - k.FieldId == other.FieldId && + k.FieldID == other.FieldID && k.InstanceType == other.InstanceType } @@ -769,8 +769,8 @@ func (k HeadStoreKey) ToDS() ds.Key { func (k DataStoreKey) PrefixEnd() DataStoreKey { newKey := k - if k.FieldId != "" { - newKey.FieldId = string(bytesPrefixEnd([]byte(k.FieldId))) + if k.FieldID != "" { + newKey.FieldID = string(bytesPrefixEnd([]byte(k.FieldID))) return newKey } if k.DocID != "" { @@ -789,12 +789,12 @@ func (k DataStoreKey) PrefixEnd() DataStoreKey { return newKey } -// FieldID extracts the Field Identifier from the Key. -// In a Primary index, the last key path is the FieldID. +// FieldIDAsUint extracts the Field Identifier from the Key. +// In a Primary index, the last key path is the FieldIDAsUint. // This may be different in Secondary Indexes. // An error is returned if it can't correct convert the field to a uint32. -func (k DataStoreKey) FieldID() (uint32, error) { - fieldID, err := strconv.Atoi(k.FieldId) +func (k DataStoreKey) FieldIDAsUint() (uint32, error) { + fieldID, err := strconv.Atoi(k.FieldID) if err != nil { return 0, NewErrFailedToGetFieldIdOfKey(err) } @@ -814,3 +814,34 @@ func bytesPrefixEnd(b []byte) []byte { // maximal byte string (i.e. already \xff...). return b } + +// EncStoreDocKey is a key for the encryption store. +type EncStoreDocKey struct { + DocID string + FieldID uint32 +} + +var _ Key = (*EncStoreDocKey)(nil) + +// NewEncStoreDocKey creates a new EncStoreDocKey from a docID and fieldID. +func NewEncStoreDocKey(docID string, fieldID uint32) EncStoreDocKey { + return EncStoreDocKey{ + DocID: docID, + FieldID: fieldID, + } +} + +func (k EncStoreDocKey) ToString() string { + if k.FieldID == 0 { + return k.DocID + } + return fmt.Sprintf("%s/%d", k.DocID, k.FieldID) +} + +func (k EncStoreDocKey) Bytes() []byte { + return []byte(k.ToString()) +} + +func (k EncStoreDocKey) ToDS() ds.Key { + return ds.NewKey(k.ToString()) +} diff --git a/internal/core/key_test.go b/internal/core/key_test.go index c5e34073a3..90bd122d6f 100644 --- a/internal/core/key_test.go +++ b/internal/core/key_test.go @@ -54,7 +54,7 @@ func TestNewDataStoreKey_ReturnsCollectionIdAndIndexIdAndDocIDAndFieldIdAndInsta DataStoreKey{ CollectionRootID: collectionRootID, DocID: docID, - FieldId: fieldID, + FieldID: fieldID, InstanceType: InstanceType(instanceType)}, result) assert.Equal(t, fmt.Sprintf("/%v/%s/%s/%s", collectionRootID, instanceType, docID, fieldID), resultString) diff --git a/internal/db/collection.go b/internal/db/collection.go index e2408006cf..19c6943a6e 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -657,7 +657,7 @@ func (c *collection) save( return cid.Undef, err } - link, _, err := merkleCRDT.Save(ctx, &merklecrdt.Field{FieldValue: val}) + link, _, err := merkleCRDT.Save(ctx, &merklecrdt.Field{DocID: primaryKey.DocID, FieldValue: val}) if err != nil { return cid.Undef, err } @@ -905,7 +905,7 @@ func (c *collection) getDataStoreKeyFromDocID(docID client.DocID) core.DataStore } func (c *collection) tryGetFieldKey(primaryKey core.PrimaryDataStoreKey, fieldName string) (core.DataStoreKey, bool) { - fieldId, hasField := c.tryGetFieldID(fieldName) + fieldID, hasField := c.tryGetFieldID(fieldName) if !hasField { return core.DataStoreKey{}, false } @@ -913,7 +913,7 @@ func (c *collection) tryGetFieldKey(primaryKey core.PrimaryDataStoreKey, fieldNa return core.DataStoreKey{ CollectionRootID: c.Description().RootID, DocID: primaryKey.DocID, - FieldId: strconv.FormatUint(uint64(fieldId), 10), + FieldID: strconv.FormatUint(uint64(fieldID), 10), }, true } diff --git a/internal/db/context.go b/internal/db/context.go index 88019af323..7b71758b0c 100644 --- a/internal/db/context.go +++ b/internal/db/context.go @@ -17,6 +17,7 @@ import ( acpIdentity "github.com/sourcenetwork/defradb/acp/identity" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/internal/encryption" ) // txnContextKey is the key type for transaction context values. @@ -62,6 +63,7 @@ func ensureContextTxn(ctx context.Context, db transactionDB, readOnly bool) (con if err != nil { return nil, txn, err } + ctx = encryption.ContextWithStore(ctx, txn) return SetContextTxn(ctx, txn), txn, nil } diff --git a/internal/db/fetcher/fetcher.go b/internal/db/fetcher/fetcher.go index bfaed9d871..06e3255e8c 100644 --- a/internal/db/fetcher/fetcher.go +++ b/internal/db/fetcher/fetcher.go @@ -351,7 +351,7 @@ func (df *DocumentFetcher) nextKey(ctx context.Context, seekNext bool) (spanDone if seekNext { curKey := df.kv.Key - curKey.FieldId = "" // clear field so prefixEnd applies to docID + curKey.FieldID = "" // clear field so prefixEnd applies to docID seekKey := curKey.PrefixEnd().ToString() spanDone, df.kv, err = df.seekKV(seekKey) // handle any internal errors @@ -504,7 +504,7 @@ func (df *DocumentFetcher) processKV(kv *keyValue) error { } } - if kv.Key.FieldId == core.DATASTORE_DOC_VERSION_FIELD_ID { + if kv.Key.FieldID == core.DATASTORE_DOC_VERSION_FIELD_ID { df.doc.schemaVersionID = string(kv.Value) return nil } @@ -515,7 +515,7 @@ func (df *DocumentFetcher) processKV(kv *keyValue) error { } // extract the FieldID and update the encoded doc properties map - fieldID, err := kv.Key.FieldID() + fieldID, err := kv.Key.FieldIDAsUint() if err != nil { return err } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index c2d7616896..c145e463aa 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -29,7 +29,7 @@ func TryGetContextEncryptor(ctx context.Context) (*DocEncryptor, bool) { func getContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { enc, ok := TryGetContextEncryptor(ctx) if !ok { - enc = newDocEncryptor() + enc = newDocEncryptor(ctx) ctx = context.WithValue(ctx, docEncContextKey{}, enc) } return ctx, enc @@ -40,7 +40,7 @@ func Context(ctx context.Context) context.Context { return ctx } -func ContextWithKey(ctx context.Context, encryptionKey string) context.Context { +func ContextWithKey(ctx context.Context, encryptionKey []byte) context.Context { ctx, encryptor := getContextWithDocEnc(ctx) encryptor.SetKey(encryptionKey) return ctx @@ -52,7 +52,7 @@ func ContextWithStore(ctx context.Context, txn datastore.Txn) context.Context { return ctx } -func EncryptDoc(ctx context.Context, docID string, fieldID int, plainText []byte) ([]byte, error) { +func EncryptDoc(ctx context.Context, docID string, fieldID uint32, plainText []byte) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return plainText, nil @@ -60,7 +60,7 @@ func EncryptDoc(ctx context.Context, docID string, fieldID int, plainText []byte return enc.Encrypt(docID, fieldID, plainText) } -func DecryptDoc(ctx context.Context, docID string, fieldID int, cipherText []byte) ([]byte, error) { +func DecryptDoc(ctx context.Context, docID string, fieldID uint32, cipherText []byte) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { return cipherText, nil diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index d31788a31e..97c26bf7de 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -10,18 +10,27 @@ package encryption -import "github.com/sourcenetwork/defradb/datastore" +import ( + "context" + "errors" + + ds "github.com/ipfs/go-datastore" + + "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/defradb/internal/core" +) type DocEncryptor struct { - encryptionKey string + encryptionKey []byte + ctx context.Context store datastore.DSReaderWriter } -func newDocEncryptor() *DocEncryptor { - return &DocEncryptor{} +func newDocEncryptor(ctx context.Context) *DocEncryptor { + return &DocEncryptor{ctx: ctx} } -func (d *DocEncryptor) SetKey(encryptionKey string) { +func (d *DocEncryptor) SetKey(encryptionKey []byte) { d.encryptionKey = encryptionKey } @@ -29,10 +38,47 @@ func (d *DocEncryptor) SetStore(store datastore.DSReaderWriter) { d.store = store } -func (d *DocEncryptor) Encrypt(docID string, fieldID int, plainText []byte) ([]byte, error) { - return EncryptAES(plainText, []byte(d.encryptionKey)) +func (d *DocEncryptor) Encrypt(docID string, fieldID uint32, plainText []byte) ([]byte, error) { + encryptionKey, storeKey, err := d.fetchEncryptionKey(docID, fieldID) + if err != nil { + return nil, err + } + + if len(encryptionKey) == 0 { + if len(d.encryptionKey) == 0 { + return plainText, nil + } + if d.store != nil { + err = d.store.Put(d.ctx, storeKey.ToDS(), d.encryptionKey) + if err != nil { + return nil, err + } + } + encryptionKey = d.encryptionKey + } + return EncryptAES(plainText, encryptionKey) +} + +func (d *DocEncryptor) Decrypt(docID string, fieldID uint32, cipherText []byte) ([]byte, error) { + encKey, _, err := d.fetchEncryptionKey(docID, fieldID) + if err != nil { + return nil, err + } + if len(encKey) == 0 { + return cipherText, nil + } + return DecryptAES(cipherText, encKey) } -func (d *DocEncryptor) Decrypt(docID string, fieldID int, cipherText []byte) ([]byte, error) { - return DecryptAES(cipherText, []byte(d.encryptionKey)) +func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldID uint32) ([]byte, core.EncStoreDocKey, error) { + storeKey := core.NewEncStoreDocKey(docID, fieldID) + if d.store == nil { + return nil, core.EncStoreDocKey{}, nil + } + encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) + isNotFound := errors.Is(err, ds.ErrNotFound) + if err != nil && !isNotFound { + return nil, core.EncStoreDocKey{}, err + } + return encryptionKey, storeKey, nil } diff --git a/internal/planner/commit.go b/internal/planner/commit.go index 3a5bec39f9..bbb5fdc09c 100644 --- a/internal/planner/commit.go +++ b/internal/planner/commit.go @@ -112,7 +112,7 @@ func (n *dagScanNode) Spans(spans core.Spans) { } for i, span := range headSetSpans.Value { - if span.Start().FieldId != fieldId { + if span.Start().FieldID != fieldId { headSetSpans.Value[i] = core.NewSpan(span.Start().WithFieldId(fieldId), core.DataStoreKey{}) } } diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 96af1d6e38..bd48acf615 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -102,3 +102,106 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { testUtils.ExecuteTestCase(t, test) } +func TestDocEncryption_UponUpdate_ShouldEncryptedCommitDelta(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + updateUserCollectionSchema(), + testUtils.CreateDoc{ + Doc: `{ + "name": "John", + "age": 21 + }`, + EncryptionKey: immutable.Some(encKey), + }, + testUtils.UpdateDoc{ + Doc: `{ + "age": 22 + }`, + }, + testUtils.Request{ + Request: ` + query { + commits(fieldId: "1") { + delta + } + } + `, + Results: []map[string]any{ + { + "delta": encrypt(encKey, testUtils.CBORValue(22)), + }, + { + "delta": encrypt(encKey, testUtils.CBORValue(21)), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptedOnlyRelevantDocs(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + updateUserCollectionSchema(), + testUtils.CreateDoc{ + // bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3 + Doc: `{ + "name": "John", + "age": 21 + }`, + EncryptionKey: immutable.Some(encKey), + }, + testUtils.CreateDoc{ + // bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6 + Doc: `{ + "name": "Islam", + "age": 33 + }`, + }, + testUtils.UpdateDoc{ + DocID: 0, + Doc: `{ + "age": 22 + }`, + }, + testUtils.UpdateDoc{ + DocID: 1, + Doc: `{ + "age": 34 + }`, + }, + testUtils.Request{ + Request: ` + query { + commits(fieldId: "1") { + delta + docID + } + } + `, + Results: []map[string]any{ + { + "delta": encrypt(encKey, testUtils.CBORValue(22)), + "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + }, + { + "delta": encrypt(encKey, testUtils.CBORValue(21)), + "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + }, + { + "delta": testUtils.CBORValue(34), + "docID": "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6", + }, + { + "delta": testUtils.CBORValue(33), + "docID": "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6", + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index b830727622..542eb84016 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1240,7 +1240,7 @@ func createDocViaColSave( func makeContextForDocCreate(ctx context.Context, action *CreateDoc, txn datastore.Txn) context.Context { ctx = db.SetContextIdentity(ctx, action.Identity) if action.EncryptionKey.HasValue() { - ctx = encryption.ContextWithKey(ctx, action.EncryptionKey.Value()) + ctx = encryption.ContextWithKey(ctx, []byte(action.EncryptionKey.Value())) } if txn != nil { ctx = encryption.ContextWithStore(ctx, txn) From 5498a64fb1dee2a58cc9ee849d2a8ec0258015cd Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 17 Jun 2024 17:21:35 +0200 Subject: [PATCH 08/42] Add p2p test --- tests/integration/encryption/peer_test.go | 64 ++++++++++++++++++++++ tests/integration/encryption/query_test.go | 2 +- 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 tests/integration/encryption/peer_test.go diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go new file mode 100644 index 0000000000..13a5eed410 --- /dev/null +++ b/tests/integration/encryption/peer_test.go @@ -0,0 +1,64 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" + "github.com/sourcenetwork/immutable" +) + +func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + age: Int + } + `, + }, + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: `{ + "name": "John", + "age": 21 + }`, + EncryptionKey: immutable.Some(encKey), + }, + testUtils.WaitForSync{}, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: `query { + Users { + age + } + }`, + Results: []map[string]any{}, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + diff --git a/tests/integration/encryption/query_test.go b/tests/integration/encryption/query_test.go index 8e1514f427..b52cb9b0d2 100644 --- a/tests/integration/encryption/query_test.go +++ b/tests/integration/encryption/query_test.go @@ -32,7 +32,7 @@ func TestDocEncryption_WithEncryption_ShouldFetchDecrypted(t *testing.T) { "name": "John", "age": 21 }`, - EncryptionKey: immutable.Some("examplekey1234567890examplekey12"), + EncryptionKey: immutable.Some(encKey), }, testUtils.Request{ Request: ` From 5396df7683e624e5a1488750ed66bbf3b151733e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 17 Jun 2024 18:41:16 +0200 Subject: [PATCH 09/42] Lint --- tests/integration/encryption/peer_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 13a5eed410..3496a6b3c4 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -13,8 +13,9 @@ package encryption import ( "testing" - testUtils "github.com/sourcenetwork/defradb/tests/integration" "github.com/sourcenetwork/immutable" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" ) func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { From e36664c85e5dd1760aa84ec959b98967721ec97a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 18 Jun 2024 17:43:28 +0200 Subject: [PATCH 10/42] Make defra generate doc encryption key --- cli/collection_create.go | 14 ++--- cli/utils.go | 6 +-- internal/encryption/context.go | 23 +------- internal/encryption/encryptor.go | 59 ++++++++++++++++++--- internal/encryption/nonce.go | 1 + tests/integration/encryption/commit_test.go | 25 ++++----- tests/integration/encryption/peer_test.go | 2 +- tests/integration/encryption/query_test.go | 3 +- tests/integration/test_case.go | 6 +-- tests/integration/utils2.go | 4 +- 10 files changed, 80 insertions(+), 63 deletions(-) diff --git a/cli/collection_create.go b/cli/collection_create.go index ed4be48224..a25a1cf16d 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -22,9 +22,9 @@ import ( func MakeCollectionCreateCommand() *cobra.Command { var file string - var encryptionKey string + var shouldEncrypt bool var cmd = &cobra.Command{ - Use: "create [-i --identity] [-e --encryption] ", + Use: "create [-i --identity] [-e --encrypt] ", Short: "Create a new document.", Long: `Create a new document. @@ -33,9 +33,9 @@ Options: Marks the document as private and set the identity as the owner. The access to the document and permissions are controlled by ACP (Access Control Policy). - -e, --encryption - Encrypts the document with the encryption key. The encryption key is used to encrypt and decrypt - the document using symmetric AES-GCM encryption algorithm. + -e, --encrypt + Encrypt flag specified if the document needs to be encrypted. If set DefraDB will generate a + symmetric key for encryption using AES-GCM. Example: create from string: defradb client collection create --name User '{ "name": "Bob" }' @@ -81,7 +81,7 @@ Example: create from stdin: } txn, _ := db.TryGetContextTxn(cmd.Context()) - setContextDocEncryptionKey(cmd, encryptionKey, txn) + setContextDocEncryptionKey(cmd, shouldEncrypt, txn) if client.IsJSONArray(docData) { docs, err := client.NewDocsFromJSON(docData, col.Definition()) @@ -98,7 +98,7 @@ Example: create from stdin: return col.Create(cmd.Context(), doc) }, } - cmd.PersistentFlags().StringVarP(&encryptionKey, "encryption", "e", "", + cmd.PersistentFlags().BoolVarP(&shouldEncrypt, "encrypt", "e", false, "Encryption key used to encrypt/decrypt the document") cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)") return cmd diff --git a/cli/utils.go b/cli/utils.go index 614f9d2ef5..4e66e7d2ab 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -163,11 +163,11 @@ func setContextIdentity(cmd *cobra.Command, privateKeyHex string) error { } // setContextIdentity sets the identity for the current command context. -func setContextDocEncryptionKey(cmd *cobra.Command, docEncryptionKey string, txn datastore.Txn) { - if docEncryptionKey == "" { +func setContextDocEncryptionKey(cmd *cobra.Command, shouldEncrypt bool, txn datastore.Txn) { + if !shouldEncrypt { return } - ctx := encryption.ContextWithKey(cmd.Context(), []byte(docEncryptionKey)) + ctx := encryption.Context(cmd.Context()) if txn != nil { ctx = encryption.ContextWithStore(ctx, txn) } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index c145e463aa..d79e62b359 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -36,13 +36,8 @@ func getContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) } func Context(ctx context.Context) context.Context { - ctx, _ = getContextWithDocEnc(ctx) - return ctx -} - -func ContextWithKey(ctx context.Context, encryptionKey []byte) context.Context { ctx, encryptor := getContextWithDocEnc(ctx) - encryptor.SetKey(encryptionKey) + encryptor.EnableKeyGeneration() return ctx } @@ -51,19 +46,3 @@ func ContextWithStore(ctx context.Context, txn datastore.Txn) context.Context { encryptor.SetStore(txn.Encstore()) return ctx } - -func EncryptDoc(ctx context.Context, docID string, fieldID uint32, plainText []byte) ([]byte, error) { - enc, ok := TryGetContextEncryptor(ctx) - if !ok { - return plainText, nil - } - return enc.Encrypt(docID, fieldID, plainText) -} - -func DecryptDoc(ctx context.Context, docID string, fieldID uint32, cipherText []byte) ([]byte, error) { - enc, ok := TryGetContextEncryptor(ctx) - if !ok { - return cipherText, nil - } - return enc.Decrypt(docID, fieldID, cipherText) -} diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 97c26bf7de..97d30a2af4 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -12,7 +12,9 @@ package encryption import ( "context" + "crypto/rand" "errors" + "io" ds "github.com/ipfs/go-datastore" @@ -20,18 +22,36 @@ import ( "github.com/sourcenetwork/defradb/internal/core" ) +var generateEncryptionKeyFunc = generateEncryptionKey + +const keyLength = 32 // 32 bytes for AES-256 + +// generateEncryptionKey generates a random AES key. +func generateEncryptionKey() ([]byte, error) { + key := make([]byte, keyLength) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + return nil, err + } + return key, nil +} + +// generateTestEncryptionKey generates a deterministic encryption key for testing. +func generateTestEncryptionKey() ([]byte, error) { + return []byte("examplekey1234567890examplekey12"), nil +} + type DocEncryptor struct { - encryptionKey []byte - ctx context.Context - store datastore.DSReaderWriter + shouldGenerateKey bool + ctx context.Context + store datastore.DSReaderWriter } func newDocEncryptor(ctx context.Context) *DocEncryptor { return &DocEncryptor{ctx: ctx} } -func (d *DocEncryptor) SetKey(encryptionKey []byte) { - d.encryptionKey = encryptionKey +func (d *DocEncryptor) EnableKeyGeneration() { + d.shouldGenerateKey = true } func (d *DocEncryptor) SetStore(store datastore.DSReaderWriter) { @@ -45,16 +65,21 @@ func (d *DocEncryptor) Encrypt(docID string, fieldID uint32, plainText []byte) ( } if len(encryptionKey) == 0 { - if len(d.encryptionKey) == 0 { + if !d.shouldGenerateKey { return plainText, nil } + + encryptionKey, err = generateEncryptionKeyFunc() + if err != nil { + return nil, err + } + if d.store != nil { - err = d.store.Put(d.ctx, storeKey.ToDS(), d.encryptionKey) + err = d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) if err != nil { return nil, err } } - encryptionKey = d.encryptionKey } return EncryptAES(plainText, encryptionKey) } @@ -70,6 +95,8 @@ func (d *DocEncryptor) Decrypt(docID string, fieldID uint32, cipherText []byte) return DecryptAES(cipherText, encKey) } +// fetchEncryptionKey fetches the encryption key for the given docID and fieldID. +// If the key is not found, it returns an empty key. func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldID uint32) ([]byte, core.EncStoreDocKey, error) { storeKey := core.NewEncStoreDocKey(docID, fieldID) if d.store == nil { @@ -82,3 +109,19 @@ func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldID uint32) ([]byte, } return encryptionKey, storeKey, nil } + +func EncryptDoc(ctx context.Context, docID string, fieldID uint32, plainText []byte) ([]byte, error) { + enc, ok := TryGetContextEncryptor(ctx) + if !ok { + return plainText, nil + } + return enc.Encrypt(docID, fieldID, plainText) +} + +func DecryptDoc(ctx context.Context, docID string, fieldID uint32, cipherText []byte) ([]byte, error) { + enc, ok := TryGetContextEncryptor(ctx) + if !ok { + return cipherText, nil + } + return enc.Decrypt(docID, fieldID, cipherText) +} diff --git a/internal/encryption/nonce.go b/internal/encryption/nonce.go index 8f67c8d958..39cd72bf88 100644 --- a/internal/encryption/nonce.go +++ b/internal/encryption/nonce.go @@ -47,5 +47,6 @@ func init() { // If the binary is a test binary, use a deterministic nonce. if strings.HasSuffix(arg, ".test") || strings.Contains(arg, "/defradb/tests/") { generateNonceFunc = generateTestNonce + generateEncryptionKeyFunc = generateTestEncryptionKey } } diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index bd48acf615..d856d82b82 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -15,13 +15,10 @@ import ( "github.com/sourcenetwork/defradb/internal/encryption" testUtils "github.com/sourcenetwork/defradb/tests/integration" - "github.com/sourcenetwork/immutable" ) -const encKey = "examplekey1234567890examplekey12" - -func encrypt(key string, plaintext []byte) []byte { - val, _ := encryption.EncryptAES(plaintext, []byte(key)) +func encrypt(plaintext []byte) []byte { + val, _ := encryption.EncryptAES(plaintext, []byte("examplekey1234567890examplekey12")) return val } @@ -34,7 +31,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "name": "John", "age": 21 }`, - EncryptionKey: immutable.Some(encKey), + IsEncrypted: true, }, testUtils.Request{ Request: ` @@ -58,7 +55,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { { "cid": "bafyreicv422zhiuqefs32wp7glrqsbjpy76hgem4ivagm2ttuli43wluci", "collectionID": int64(1), - "delta": encrypt(encKey, testUtils.CBORValue(21)), + "delta": encrypt(testUtils.CBORValue(21)), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", "fieldId": "1", "fieldName": "age", @@ -68,7 +65,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { { "cid": "bafyreie6i4dw5jh6bp2anszqkmuwfslsemzatrflipetljhtpjhjn3zbum", "collectionID": int64(1), - "delta": encrypt(encKey, testUtils.CBORValue("John")), + "delta": encrypt(testUtils.CBORValue("John")), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", "fieldId": "2", "fieldName": "name", @@ -111,7 +108,7 @@ func TestDocEncryption_UponUpdate_ShouldEncryptedCommitDelta(t *testing.T) { "name": "John", "age": 21 }`, - EncryptionKey: immutable.Some(encKey), + IsEncrypted: true, }, testUtils.UpdateDoc{ Doc: `{ @@ -128,10 +125,10 @@ func TestDocEncryption_UponUpdate_ShouldEncryptedCommitDelta(t *testing.T) { `, Results: []map[string]any{ { - "delta": encrypt(encKey, testUtils.CBORValue(22)), + "delta": encrypt(testUtils.CBORValue(22)), }, { - "delta": encrypt(encKey, testUtils.CBORValue(21)), + "delta": encrypt(testUtils.CBORValue(21)), }, }, }, @@ -151,7 +148,7 @@ func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptedOnlyRelevantDoc "name": "John", "age": 21 }`, - EncryptionKey: immutable.Some(encKey), + IsEncrypted: true, }, testUtils.CreateDoc{ // bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6 @@ -183,11 +180,11 @@ func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptedOnlyRelevantDoc `, Results: []map[string]any{ { - "delta": encrypt(encKey, testUtils.CBORValue(22)), + "delta": encrypt(testUtils.CBORValue(22)), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", }, { - "delta": encrypt(encKey, testUtils.CBORValue(21)), + "delta": encrypt(testUtils.CBORValue(21)), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", }, { diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 3496a6b3c4..6793f3991f 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -45,7 +45,7 @@ func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { "name": "John", "age": 21 }`, - EncryptionKey: immutable.Some(encKey), + IsEncrypted: true, }, testUtils.WaitForSync{}, testUtils.Request{ diff --git a/tests/integration/encryption/query_test.go b/tests/integration/encryption/query_test.go index b52cb9b0d2..bd3ed279c7 100644 --- a/tests/integration/encryption/query_test.go +++ b/tests/integration/encryption/query_test.go @@ -14,7 +14,6 @@ import ( "testing" testUtils "github.com/sourcenetwork/defradb/tests/integration" - "github.com/sourcenetwork/immutable" ) func TestDocEncryption_WithEncryption_ShouldFetchDecrypted(t *testing.T) { @@ -32,7 +31,7 @@ func TestDocEncryption_WithEncryption_ShouldFetchDecrypted(t *testing.T) { "name": "John", "age": 21 }`, - EncryptionKey: immutable.Some(encKey), + IsEncrypted: true, }, testUtils.Request{ Request: ` diff --git a/tests/integration/test_case.go b/tests/integration/test_case.go index deaef1d04d..948ae1838e 100644 --- a/tests/integration/test_case.go +++ b/tests/integration/test_case.go @@ -230,10 +230,8 @@ type CreateDoc struct { // created document(s) will be owned by this Identity. Identity immutable.Option[acpIdentity.Identity] - // The encryption key to use for the document encryption. Optional. - // - // If an EncryptionKey is not provided the document will not be encrypted. - EncryptionKey immutable.Option[string] + // Specifies whether the document should be encrypted. + IsEncrypted bool // The collection in which this document should be created. CollectionID int diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 542eb84016..db8f36138a 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1239,8 +1239,8 @@ func createDocViaColSave( func makeContextForDocCreate(ctx context.Context, action *CreateDoc, txn datastore.Txn) context.Context { ctx = db.SetContextIdentity(ctx, action.Identity) - if action.EncryptionKey.HasValue() { - ctx = encryption.ContextWithKey(ctx, []byte(action.EncryptionKey.Value())) + if action.IsEncrypted { + ctx = encryption.Context(ctx) } if txn != nil { ctx = encryption.ContextWithStore(ctx, txn) From fc4664d9344d48d64f24e040a8ee344a14039417 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 18 Jun 2024 17:49:49 +0200 Subject: [PATCH 11/42] Remove unused code --- internal/db/config.go | 7 ------- internal/db/db.go | 2 -- 2 files changed, 9 deletions(-) diff --git a/internal/db/config.go b/internal/db/config.go index 83b2017a90..8ce725ebd0 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -28,10 +28,3 @@ func WithMaxRetries(num int) Option { db.maxTxnRetries = immutable.Some(num) } } - -// WithDocEncryption enables document encryption. -func WithEnableDocEncryption(enable bool) Option { - return func(db *db) { - db.isEncrypted = enable - } -} diff --git a/internal/db/db.go b/internal/db/db.go index 314bba8d7d..197ab493d5 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -66,8 +66,6 @@ type db struct { // The maximum number of retries per transaction. maxTxnRetries immutable.Option[int] - - isEncrypted bool // The options used to init the database options []Option From fe7c5c2ab0ed0ad928991ad84caaa748e65459e8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 20 Jun 2024 23:51:47 +0200 Subject: [PATCH 12/42] Store enc flat in a block. On update read the flat from a prev block --- cli/utils.go | 1 + internal/core/block/block.go | 19 +++---- internal/core/crdt/ipld_union.go | 55 ++++++++++++++++++++ internal/core/crdt/lwwreg.go | 7 --- internal/core/crdt/lwwreg_test.go | 6 +-- internal/db/context.go | 3 -- internal/encryption/config.go | 16 ++++++ internal/encryption/context.go | 18 +++++++ internal/encryption/encryptor.go | 6 +-- internal/merkle/clock/clock.go | 57 ++++++++++++++++++++- internal/merkle/crdt/lwwreg.go | 6 --- internal/planner/explain.go | 2 +- internal/planner/multi.go | 2 +- internal/planner/planner.go | 2 +- tests/integration/encryption/commit_test.go | 10 ++-- tests/integration/utils2.go | 1 + 16 files changed, 167 insertions(+), 44 deletions(-) create mode 100644 internal/encryption/config.go diff --git a/cli/utils.go b/cli/utils.go index 4e66e7d2ab..30837b74a5 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -171,6 +171,7 @@ func setContextDocEncryptionKey(cmd *cobra.Command, shouldEncrypt bool, txn data if txn != nil { ctx = encryption.ContextWithStore(ctx, txn) } + ctx = encryption.SetContextConfig(cmd.Context(), encryption.DocEncConfig{IsEncrypted: true}) cmd.SetContext(ctx) } diff --git a/internal/core/block/block.go b/internal/core/block/block.go index 8482a23d91..3faecc1352 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -103,6 +103,8 @@ type Block struct { Delta crdt.CRDT // Links are the links to other blocks in the DAG. Links []DAGLink + // IsEncrypted is a flag that indicates if the block's delta is encrypted. + IsEncrypted *bool } // IPLDSchemaBytes returns the IPLD schema representation for the block. @@ -111,8 +113,9 @@ type Block struct { func (b Block) IPLDSchemaBytes() []byte { return []byte(` type Block struct { - delta CRDT - links [ DAGLink ] + delta CRDT + links [ DAGLink ] + isEncrypted optional Bool }`) } @@ -143,19 +146,9 @@ func New(delta core.Delta, links []DAGLink, heads ...cid.Cid) *Block { blockLinks = append(blockLinks, links...) - var crdtDelta crdt.CRDT - switch delta := delta.(type) { - case *crdt.LWWRegDelta: - crdtDelta = crdt.CRDT{LWWRegDelta: delta} - case *crdt.CompositeDAGDelta: - crdtDelta = crdt.CRDT{CompositeDAGDelta: delta} - case *crdt.CounterDelta: - crdtDelta = crdt.CRDT{CounterDelta: delta} - } - return &Block{ Links: blockLinks, - Delta: crdtDelta, + Delta: crdt.NewCRDT(delta), } } diff --git a/internal/core/crdt/ipld_union.go b/internal/core/crdt/ipld_union.go index 361a41b150..262eafcca6 100644 --- a/internal/core/crdt/ipld_union.go +++ b/internal/core/crdt/ipld_union.go @@ -19,6 +19,19 @@ type CRDT struct { CounterDelta *CounterDelta } +// NewCRDT returns a new CRDT. +func NewCRDT(delta core.Delta) CRDT { + switch d := delta.(type) { + case *LWWRegDelta: + return CRDT{LWWRegDelta: d} + case *CompositeDAGDelta: + return CRDT{CompositeDAGDelta: d} + case *CounterDelta: + return CRDT{CounterDelta: d} + } + return CRDT{} +} + // IPLDSchemaBytes returns the IPLD schema representation for the CRDT. // // This needs to match the [CRDT] struct or [mustSetSchema] will panic on init. @@ -96,6 +109,39 @@ func (c CRDT) GetSchemaVersionID() string { return "" } +// Clone returns a clone of the CRDT. +func (c CRDT) Clone() CRDT { + var cloned CRDT + switch { + case c.LWWRegDelta != nil: + cloned.LWWRegDelta = &LWWRegDelta{ + DocID: c.LWWRegDelta.DocID, + FieldName: c.LWWRegDelta.FieldName, + Priority: c.LWWRegDelta.Priority, + SchemaVersionID: c.LWWRegDelta.SchemaVersionID, + Data: c.LWWRegDelta.Data, + } + case c.CompositeDAGDelta != nil: + cloned.CompositeDAGDelta = &CompositeDAGDelta{ + DocID: c.CompositeDAGDelta.DocID, + FieldName: c.CompositeDAGDelta.FieldName, + Priority: c.CompositeDAGDelta.Priority, + SchemaVersionID: c.CompositeDAGDelta.SchemaVersionID, + Status: c.CompositeDAGDelta.Status, + } + case c.CounterDelta != nil: + cloned.CounterDelta = &CounterDelta{ + DocID: c.CounterDelta.DocID, + FieldName: c.CounterDelta.FieldName, + Priority: c.CounterDelta.Priority, + SchemaVersionID: c.CounterDelta.SchemaVersionID, + Nonce: c.CounterDelta.Nonce, + Data: c.CounterDelta.Data, + } + } + return cloned +} + // GetStatus returns the status of the delta. // // Currently only implemented for CompositeDAGDelta. @@ -116,6 +162,15 @@ func (c CRDT) GetData() []byte { return nil } +// SetData sets the data of the delta. +// +// Currently only implemented for LWWRegDelta. +func (c CRDT) SetData(data []byte) { + if c.LWWRegDelta != nil { + c.LWWRegDelta.Data = data + } +} + // IsComposite returns true if the CRDT is a composite CRDT. func (c CRDT) IsComposite() bool { return c.CompositeDAGDelta != nil diff --git a/internal/core/crdt/lwwreg.go b/internal/core/crdt/lwwreg.go index 15baf4060f..ea0a2f3610 100644 --- a/internal/core/crdt/lwwreg.go +++ b/internal/core/crdt/lwwreg.go @@ -20,7 +20,6 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/db/base" - "github.com/sourcenetwork/defradb/internal/encryption" ) // LWWRegDelta is a single delta operation for an LWWRegister @@ -114,12 +113,6 @@ func (reg LWWRegister) Merge(ctx context.Context, delta core.Delta) error { data := d.Data - var err error - data, err = encryption.DecryptDoc(ctx, string(d.DocID), 0, data) - if err != nil { - return err - } - return reg.setValue(ctx, data, d.GetPriority()) } diff --git a/internal/core/crdt/lwwreg_test.go b/internal/core/crdt/lwwreg_test.go index 2083a5b800..136d5cd09d 100644 --- a/internal/core/crdt/lwwreg_test.go +++ b/internal/core/crdt/lwwreg_test.go @@ -31,7 +31,7 @@ func setupLWWRegister() LWWRegister { return NewLWWRegister(store, core.CollectionSchemaVersionKey{}, key, "") } -func setupLoadedLWWRegster(ctx context.Context) LWWRegister { +func setupLoadedLWWRegister(ctx context.Context) LWWRegister { lww := setupLWWRegister() addDelta := lww.Set([]byte("test")) addDelta.SetPriority(1) @@ -73,7 +73,7 @@ func TestLWWRegisterInitialMerge(t *testing.T) { func TestLWWReisterFollowupMerge(t *testing.T) { ctx := context.Background() - lww := setupLoadedLWWRegster(ctx) + lww := setupLoadedLWWRegister(ctx) addDelta := lww.Set([]byte("test2")) addDelta.SetPriority(2) lww.Merge(ctx, addDelta) @@ -90,7 +90,7 @@ func TestLWWReisterFollowupMerge(t *testing.T) { func TestLWWRegisterOldMerge(t *testing.T) { ctx := context.Background() - lww := setupLoadedLWWRegster(ctx) + lww := setupLoadedLWWRegister(ctx) addDelta := lww.Set([]byte("test-1")) addDelta.SetPriority(0) lww.Merge(ctx, addDelta) diff --git a/internal/db/context.go b/internal/db/context.go index 7b71758b0c..8ad51c86ce 100644 --- a/internal/db/context.go +++ b/internal/db/context.go @@ -89,9 +89,6 @@ func SetContextTxn(ctx context.Context, txn datastore.Txn) context.Context { return context.WithValue(ctx, txnContextKey{}, txn) } -// TryGetContextTxn returns an identity and a bool indicating if the -// identity was retrieved from the given context. - // GetContextIdentity returns the identity from the given context. // // If an identity does not exist `NoIdentity` is returned. diff --git a/internal/encryption/config.go b/internal/encryption/config.go new file mode 100644 index 0000000000..ddb4a3815a --- /dev/null +++ b/internal/encryption/config.go @@ -0,0 +1,16 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +// DocEncConfig is the configuration for document encryption. +type DocEncConfig struct { + IsEncrypted bool +} diff --git a/internal/encryption/context.go b/internal/encryption/context.go index d79e62b359..f3f1fec113 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -14,11 +14,15 @@ import ( "context" "github.com/sourcenetwork/defradb/datastore" + "github.com/sourcenetwork/immutable" ) // docEncContextKey is the key type for document encryption context values. type docEncContextKey struct{} +// configContextKey is the key type for encryption context values. +type configContextKey struct{} + // TryGetContextDocEnc returns a document encryption and a bool indicating if // it was retrieved from the given context. func TryGetContextEncryptor(ctx context.Context) (*DocEncryptor, bool) { @@ -46,3 +50,17 @@ func ContextWithStore(ctx context.Context, txn datastore.Txn) context.Context { encryptor.SetStore(txn.Encstore()) return ctx } + +// GetContextConfig returns the doc encryption config from the given context. +func GetContextConfig(ctx context.Context) immutable.Option[DocEncConfig] { + encConfig, ok := ctx.Value(configContextKey{}).(DocEncConfig) + if ok { + return immutable.Some(encConfig) + } + return immutable.None[DocEncConfig]() +} + +// SetContextConfig returns a new context with the encryption value set. +func SetContextConfig(ctx context.Context, encConfig DocEncConfig) context.Context { + return context.WithValue(ctx, configContextKey{}, encConfig) +} diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index 97d30a2af4..f22cee25ee 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -90,7 +90,7 @@ func (d *DocEncryptor) Decrypt(docID string, fieldID uint32, cipherText []byte) return nil, err } if len(encKey) == 0 { - return cipherText, nil + return nil, nil } return DecryptAES(cipherText, encKey) } @@ -113,7 +113,7 @@ func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldID uint32) ([]byte, func EncryptDoc(ctx context.Context, docID string, fieldID uint32, plainText []byte) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { - return plainText, nil + return nil, nil } return enc.Encrypt(docID, fieldID, plainText) } @@ -121,7 +121,7 @@ func EncryptDoc(ctx context.Context, docID string, fieldID uint32, plainText []b func DecryptDoc(ctx context.Context, docID string, fieldID uint32, cipherText []byte) ([]byte, error) { enc, ok := TryGetContextEncryptor(ctx) if !ok { - return cipherText, nil + return nil, nil } return enc.Decrypt(docID, fieldID, cipherText) } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index d16d1d6a5b..85186e0531 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -16,6 +16,7 @@ package clock import ( "context" + cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" @@ -24,6 +25,7 @@ import ( "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/core" coreblock "github.com/sourcenetwork/defradb/internal/core/block" + "github.com/sourcenetwork/defradb/internal/encryption" ) var ( @@ -86,7 +88,22 @@ func (mc *MerkleClock) AddDelta( block := coreblock.New(delta, links, heads...) // Write the new block to the dag store. - link, err := mc.putBlock(ctx, block) + isEncrypted, err := mc.checkIfBlockEncryptionEnabled(ctx, heads) + if err != nil { + return cidlink.Link{}, nil, err + } + + var dagBlock *coreblock.Block + if isEncrypted { + dagBlock, err = encryptBlock(ctx, block) + if err != nil { + return cidlink.Link{}, nil, err + } + } else { + dagBlock = block + } + + link, err := mc.putBlock(ctx, dagBlock) if err != nil { return cidlink.Link{}, nil, err } @@ -109,6 +126,44 @@ func (mc *MerkleClock) AddDelta( return link, b, err } +func (mc *MerkleClock) checkIfBlockEncryptionEnabled( + ctx context.Context, + heads []cid.Cid, +) (bool, error) { + encConf := encryption.GetContextConfig(ctx) + if encConf.HasValue() && encConf.Value().IsEncrypted { + return true, nil + } + + for _, headCid := range heads { + bytes, err := mc.dagstore.AsIPLDStorage().Get(ctx, headCid.KeyString()) + if err != nil { + return false, NewErrCouldNotFindBlock(headCid, err) + } + prevBlock, err := coreblock.GetFromBytes(bytes) + if err != nil { + return false, err + } + if prevBlock.IsEncrypted != nil && *prevBlock.IsEncrypted { + return true, nil + } + // could do block.EncryptDelta(encKey) + } + + return false, nil +} + +func encryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block, error) { + clonedCRDT := block.Delta.Clone() + bytes, err := encryption.EncryptDoc(ctx, string(clonedCRDT.GetDocID()), 0, clonedCRDT.GetData()) + if err != nil { + return nil, err + } + clonedCRDT.SetData(bytes) + isEncrypted := true + return &coreblock.Block{Delta: clonedCRDT, Links: block.Links, IsEncrypted: &isEncrypted}, nil +} + // ProcessBlock merges the delta CRDT and updates the state accordingly. func (mc *MerkleClock) ProcessBlock( ctx context.Context, diff --git a/internal/merkle/crdt/lwwreg.go b/internal/merkle/crdt/lwwreg.go index 72b46fd845..b76fac4e50 100644 --- a/internal/merkle/crdt/lwwreg.go +++ b/internal/merkle/crdt/lwwreg.go @@ -18,7 +18,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/internal/core" corecrdt "github.com/sourcenetwork/defradb/internal/core/crdt" - "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/merkle/clock" ) @@ -57,11 +56,6 @@ func (mlwwreg *MerkleLWWRegister) Save(ctx context.Context, data any) (cidlink.L return cidlink.Link{}, nil, err } - bytes, err = encryption.EncryptDoc(ctx, value.DocID, 0, bytes) - if err != nil { - return cidlink.Link{}, nil, err - } - // Set() call on underlying LWWRegister CRDT // persist/publish delta delta := mlwwreg.reg.Set(bytes) diff --git a/internal/planner/explain.go b/internal/planner/explain.go index f6d3f57209..34c3b3b644 100644 --- a/internal/planner/explain.go +++ b/internal/planner/explain.go @@ -342,7 +342,7 @@ func collectExecuteExplainInfo(executedPlan planNode) (map[string]any, error) { // Note: This function only fails if the collection of the datapoints goes wrong, otherwise // even if plan execution fails this function would return the collected datapoints. func (p *Planner) executeAndExplainRequest( - ctx context.Context, + _ context.Context, plan planNode, ) ([]map[string]any, error) { executionSuccess := false diff --git a/internal/planner/multi.go b/internal/planner/multi.go index 27d6886d7c..de220c43e5 100644 --- a/internal/planner/multi.go +++ b/internal/planner/multi.go @@ -131,7 +131,7 @@ func (p *parallelNode) Next() (bool, error) { return orNext, nil } -func (p *parallelNode) nextMerge(index int, plan planNode) (bool, error) { +func (p *parallelNode) nextMerge(_ int, plan planNode) (bool, error) { if next, err := plan.Next(); !next { return false, err } diff --git a/internal/planner/planner.go b/internal/planner/planner.go index f7a875af70..0f513a045c 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -528,7 +528,7 @@ func walkAndFindPlanType[T planNode](planNode planNode) (T, bool) { // executeRequest executes the plan graph that represents the request that was made. func (p *Planner) executeRequest( - ctx context.Context, + _ context.Context, planNode planNode, ) ([]map[string]any, error) { if err := planNode.Start(); err != nil { diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index d856d82b82..95568e489f 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -53,7 +53,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { `, Results: []map[string]any{ { - "cid": "bafyreicv422zhiuqefs32wp7glrqsbjpy76hgem4ivagm2ttuli43wluci", + "cid": "bafyreidrbl46bz5nuzuby6s4zqvzliq4gyup3pq6ipy7ljm5o7l5hxtjhm", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21)), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", @@ -63,7 +63,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreie6i4dw5jh6bp2anszqkmuwfslsemzatrflipetljhtpjhjn3zbum", + "cid": "bafyreighzsctnwzhw57nbzici6dbvohozwet5w2baey3p4dxtxp7wxybui", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John")), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", @@ -73,7 +73,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreia747gvxxbowag2mob2up34zwh364olc7ocab3nunj2ikdxq7srom", + "cid": "bafyreidzfgvlx6eaj4furwl3mpvxp3wslbvzs4hvknivhpjw7g275k5v5i", "collectionID": int64(1), "delta": nil, "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", @@ -82,11 +82,11 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreicv422zhiuqefs32wp7glrqsbjpy76hgem4ivagm2ttuli43wluci", + "cid": "bafyreidrbl46bz5nuzuby6s4zqvzliq4gyup3pq6ipy7ljm5o7l5hxtjhm", "name": "age", }, { - "cid": "bafyreie6i4dw5jh6bp2anszqkmuwfslsemzatrflipetljhtpjhjn3zbum", + "cid": "bafyreighzsctnwzhw57nbzici6dbvohozwet5w2baey3p4dxtxp7wxybui", "name": "name", }, }, diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index db8f36138a..d97cdeb808 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1241,6 +1241,7 @@ func makeContextForDocCreate(ctx context.Context, action *CreateDoc, txn datasto ctx = db.SetContextIdentity(ctx, action.Identity) if action.IsEncrypted { ctx = encryption.Context(ctx) + ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) } if txn != nil { ctx = encryption.ContextWithStore(ctx, txn) From 0f2744702f848a40a664927ee3bc1a54878dabec Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 21 Jun 2024 17:08:19 +0200 Subject: [PATCH 13/42] Fix p2p side --- internal/db/merge.go | 3 +++ internal/merkle/clock/clock.go | 9 ++++----- tests/integration/encryption/commit_test.go | 14 +++++++------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index bbfedd98d8..d45cdf6a1d 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -227,6 +227,9 @@ func (mp *mergeProcessor) loadComposites( func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { for e := mp.composites.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) + if block.IsEncrypted != nil && *block.IsEncrypted { + continue + } link, err := block.GenerateLink() if err != nil { return err diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 85186e0531..65bd6496e2 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -36,9 +36,8 @@ var ( type MerkleClock struct { headstore datastore.DSReaderWriter blockstore datastore.Blockstore - // dagSyncer - headset *heads - crdt core.ReplicatedData + headset *heads + crdt core.ReplicatedData } // NewMerkleClock returns a new MerkleClock. @@ -118,7 +117,7 @@ func (mc *MerkleClock) AddDelta( return cidlink.Link{}, nil, err } - b, err := block.Marshal() + b, err := dagBlock.Marshal() if err != nil { return cidlink.Link{}, nil, err } @@ -136,7 +135,7 @@ func (mc *MerkleClock) checkIfBlockEncryptionEnabled( } for _, headCid := range heads { - bytes, err := mc.dagstore.AsIPLDStorage().Get(ctx, headCid.KeyString()) + bytes, err := mc.blockstore.AsIPLDStorage().Get(ctx, headCid.KeyString()) if err != nil { return false, NewErrCouldNotFindBlock(headCid, err) } diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 95568e489f..393303fa1d 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -53,7 +53,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { `, Results: []map[string]any{ { - "cid": "bafyreidrbl46bz5nuzuby6s4zqvzliq4gyup3pq6ipy7ljm5o7l5hxtjhm", + "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21)), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", @@ -63,7 +63,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreighzsctnwzhw57nbzici6dbvohozwet5w2baey3p4dxtxp7wxybui", + "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John")), "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", @@ -73,7 +73,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "links": []map[string]any{}, }, { - "cid": "bafyreidzfgvlx6eaj4furwl3mpvxp3wslbvzs4hvknivhpjw7g275k5v5i", + "cid": "bafyreicvxlfxeqghmc3gy56rp5rzfejnbng4nu77x5e3wjinfydl6wvycq", "collectionID": int64(1), "delta": nil, "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", @@ -82,12 +82,12 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "height": int64(1), "links": []map[string]any{ { - "cid": "bafyreidrbl46bz5nuzuby6s4zqvzliq4gyup3pq6ipy7ljm5o7l5hxtjhm", - "name": "age", + "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", + "name": "name", }, { - "cid": "bafyreighzsctnwzhw57nbzici6dbvohozwet5w2baey3p4dxtxp7wxybui", - "name": "name", + "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", + "name": "age", }, }, }, From e321f84ef9751fcb6ae6f4d169ae276dabf083fa Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 24 Jun 2024 10:44:42 +0200 Subject: [PATCH 14/42] Upon peer sync update only heads --- internal/db/fetcher/versioned.go | 2 +- internal/db/merge.go | 9 ++- internal/merkle/clock/clock.go | 16 +++- internal/merkle/crdt/merklecrdt.go | 5 +- tests/integration/encryption/peer_test.go | 98 +++++++++++++++++++++-- 5 files changed, 114 insertions(+), 16 deletions(-) diff --git a/internal/db/fetcher/versioned.go b/internal/db/fetcher/versioned.go index 892b84e329..0ff58c4eeb 100644 --- a/internal/db/fetcher/versioned.go +++ b/internal/db/fetcher/versioned.go @@ -415,7 +415,7 @@ func (vf *VersionedFetcher) processBlock( vf.mCRDTs[crdtIndex] = mcrdt } - err = mcrdt.Clock().ProcessBlock(vf.ctx, block, blockLink) + err = mcrdt.Clock().ProcessBlock(vf.ctx, block, blockLink, false) return err } diff --git a/internal/db/merge.go b/internal/db/merge.go index d45cdf6a1d..70215da826 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -228,13 +228,13 @@ func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { for e := mp.composites.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) if block.IsEncrypted != nil && *block.IsEncrypted { - continue + onlyHeads = true } link, err := block.GenerateLink() if err != nil { return err } - err = mp.processBlock(ctx, block, link) + err = mp.processBlock(ctx, block, link, onlyHeads) if err != nil { return err } @@ -247,6 +247,7 @@ func (mp *mergeProcessor) processBlock( ctx context.Context, block *coreblock.Block, blockLink cidlink.Link, + onlyHeads bool, ) error { crdt, err := mp.initCRDTForType(block.Delta.GetFieldName()) if err != nil { @@ -259,7 +260,7 @@ func (mp *mergeProcessor) processBlock( return nil } - err = crdt.Clock().ProcessBlock(ctx, block, blockLink) + err = crdt.Clock().ProcessBlock(ctx, block, blockLink, onlyHeads) if err != nil { return err } @@ -279,7 +280,7 @@ func (mp *mergeProcessor) processBlock( return err } - if err := mp.processBlock(ctx, childBlock, link.Link); err != nil { + if err := mp.processBlock(ctx, childBlock, link.Link, onlyHeads); err != nil { return err } } diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 65bd6496e2..c22b648606 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -112,6 +112,7 @@ func (mc *MerkleClock) AddDelta( ctx, block, link, + false, ) if err != nil { return cidlink.Link{}, nil, err @@ -168,13 +169,24 @@ func (mc *MerkleClock) ProcessBlock( ctx context.Context, block *coreblock.Block, blockLink cidlink.Link, + onlyHeads bool, ) error { - priority := block.Delta.GetPriority() - + if !onlyHeads { err := mc.crdt.Merge(ctx, block.Delta.GetDelta()) if err != nil { return NewErrMergingDelta(blockLink.Cid, err) } + } + + return mc.updateHeads(ctx, block, blockLink) +} + +func (mc *MerkleClock) updateHeads( + ctx context.Context, + block *coreblock.Block, + blockLink cidlink.Link, +) error { + priority := block.Delta.GetPriority() // check if we have any HEAD links hasHeads := false diff --git a/internal/merkle/crdt/merklecrdt.go b/internal/merkle/crdt/merklecrdt.go index fc3019b05c..e3a40207db 100644 --- a/internal/merkle/crdt/merklecrdt.go +++ b/internal/merkle/crdt/merklecrdt.go @@ -47,7 +47,10 @@ type MerkleClock interface { delta core.Delta, links ...coreblock.DAGLink, ) (cidlink.Link, []byte, error) - ProcessBlock(context.Context, *coreblock.Block, cidlink.Link) error + // ProcessBlock processes a block and updates the CRDT state. + // The bool argument indicates whether only heads need to be updated. It is needed in case + // merge should be skipped for example if the block is encrypted. + ProcessBlock(context.Context, *coreblock.Block, cidlink.Link, bool) error } // baseMerkleCRDT handles the MerkleCRDT overhead functions that aren't CRDT specific like the mutations and state diff --git a/tests/integration/encryption/peer_test.go b/tests/integration/encryption/peer_test.go index 6793f3991f..6d9c937278 100644 --- a/tests/integration/encryption/peer_test.go +++ b/tests/integration/encryption/peer_test.go @@ -23,14 +23,7 @@ func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { Actions: []any{ testUtils.RandomNetworkingConfig(), testUtils.RandomNetworkingConfig(), - testUtils.SchemaUpdate{ - Schema: ` - type Users { - name: String - age: Int - } - `, - }, + updateUserCollectionSchema(), testUtils.ConnectPeers{ SourceNodeID: 1, TargetNodeID: 0, @@ -63,3 +56,92 @@ func TestDocEncryptionPeer_IfPeerHasNoKey_ShouldNotFetch(t *testing.T) { testUtils.ExecuteTestCase(t, test) } +func TestDocEncryptionPeer_UponSync_ShouldSyncEncryptedDAG(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.RandomNetworkingConfig(), + testUtils.RandomNetworkingConfig(), + updateUserCollectionSchema(), + testUtils.ConnectPeers{ + SourceNodeID: 1, + TargetNodeID: 0, + }, + testUtils.SubscribeToCollection{ + NodeID: 1, + CollectionIDs: []int{0}, + }, + testUtils.CreateDoc{ + NodeID: immutable.Some(0), + Doc: `{ + "name": "John", + "age": 21 + }`, + IsEncrypted: true, + }, + testUtils.WaitForSync{}, + testUtils.Request{ + NodeID: immutable.Some(1), + Request: ` + query { + commits { + cid + collectionID + delta + docID + fieldId + fieldName + height + links { + cid + name + } + } + } + `, + Results: []map[string]any{ + { + "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", + "collectionID": int64(1), + "delta": encrypt(testUtils.CBORValue(21)), + "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "fieldId": "1", + "fieldName": "age", + "height": int64(1), + "links": []map[string]any{}, + }, + { + "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", + "collectionID": int64(1), + "delta": encrypt(testUtils.CBORValue("John")), + "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "fieldId": "2", + "fieldName": "name", + "height": int64(1), + "links": []map[string]any{}, + }, + { + "cid": "bafyreicvxlfxeqghmc3gy56rp5rzfejnbng4nu77x5e3wjinfydl6wvycq", + "collectionID": int64(1), + "delta": nil, + "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "fieldId": "C", + "fieldName": nil, + "height": int64(1), + "links": []map[string]any{ + { + "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", + "name": "name", + }, + { + "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", + "name": "age", + }, + }, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From 2f83d9226996cb671a80288b81a2916b5cdd89b2 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 24 Jun 2024 13:45:51 +0200 Subject: [PATCH 15/42] Follow up --- internal/db/merge.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/db/merge.go b/internal/db/merge.go index 70215da826..4ccee263a3 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -227,6 +227,7 @@ func (mp *mergeProcessor) loadComposites( func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { for e := mp.composites.Front(); e != nil; e = e.Next() { block := e.Value.(*coreblock.Block) + var onlyHeads bool if block.IsEncrypted != nil && *block.IsEncrypted { onlyHeads = true } From 13c9e9c52b3ecc1ae5964645baf4f39837259797 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 24 Jun 2024 13:51:07 +0200 Subject: [PATCH 16/42] Polish --- cli/collection_create.go | 2 +- cli/utils.go | 6 +++--- internal/core/crdt/lwwreg.go | 4 +--- internal/db/collection.go | 2 +- internal/encryption/context.go | 12 ++++++++---- internal/merkle/crdt/counter.go | 2 +- internal/merkle/crdt/field.go | 6 +++++- internal/merkle/crdt/lwwreg.go | 2 +- 8 files changed, 21 insertions(+), 15 deletions(-) diff --git a/cli/collection_create.go b/cli/collection_create.go index a25a1cf16d..9525b821fa 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -81,7 +81,7 @@ Example: create from stdin: } txn, _ := db.TryGetContextTxn(cmd.Context()) - setContextDocEncryptionKey(cmd, shouldEncrypt, txn) + setContextDocEncryption(cmd, shouldEncrypt, txn) if client.IsJSONArray(docData) { docs, err := client.NewDocsFromJSON(docData, col.Definition()) diff --git a/cli/utils.go b/cli/utils.go index 30837b74a5..9289b2c4e0 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -162,8 +162,8 @@ func setContextIdentity(cmd *cobra.Command, privateKeyHex string) error { return nil } -// setContextIdentity sets the identity for the current command context. -func setContextDocEncryptionKey(cmd *cobra.Command, shouldEncrypt bool, txn datastore.Txn) { +// setContextDocEncryption sets doc encryption for the current command context. +func setContextDocEncryption(cmd *cobra.Command, shouldEncrypt bool, txn datastore.Txn) { if !shouldEncrypt { return } @@ -171,7 +171,7 @@ func setContextDocEncryptionKey(cmd *cobra.Command, shouldEncrypt bool, txn data if txn != nil { ctx = encryption.ContextWithStore(ctx, txn) } - ctx = encryption.SetContextConfig(cmd.Context(), encryption.DocEncConfig{IsEncrypted: true}) + ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) cmd.SetContext(ctx) } diff --git a/internal/core/crdt/lwwreg.go b/internal/core/crdt/lwwreg.go index ea0a2f3610..edfff9ca05 100644 --- a/internal/core/crdt/lwwreg.go +++ b/internal/core/crdt/lwwreg.go @@ -111,9 +111,7 @@ func (reg LWWRegister) Merge(ctx context.Context, delta core.Delta) error { return ErrMismatchedMergeType } - data := d.Data - - return reg.setValue(ctx, data, d.GetPriority()) + return reg.setValue(ctx, d.Data, d.GetPriority()) } func (reg LWWRegister) setValue(ctx context.Context, val []byte, priority uint64) error { diff --git a/internal/db/collection.go b/internal/db/collection.go index 19c6943a6e..64f90960cc 100644 --- a/internal/db/collection.go +++ b/internal/db/collection.go @@ -657,7 +657,7 @@ func (c *collection) save( return cid.Undef, err } - link, _, err := merkleCRDT.Save(ctx, &merklecrdt.Field{DocID: primaryKey.DocID, FieldValue: val}) + link, _, err := merkleCRDT.Save(ctx, &merklecrdt.DocField{DocID: primaryKey.DocID, FieldValue: val}) if err != nil { return cid.Undef, err } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index f3f1fec113..cf433ef54d 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -30,7 +30,7 @@ func TryGetContextEncryptor(ctx context.Context) (*DocEncryptor, bool) { return enc, ok } -func getContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { +func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { enc, ok := TryGetContextEncryptor(ctx) if !ok { enc = newDocEncryptor(ctx) @@ -39,14 +39,18 @@ func getContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) return ctx, enc } +// Context enables key generation on the doc encryptor in the context. +// If the doc encryptor is not present, it will be created. func Context(ctx context.Context) context.Context { - ctx, encryptor := getContextWithDocEnc(ctx) + ctx, encryptor := ensureContextWithDocEnc(ctx) encryptor.EnableKeyGeneration() return ctx } +// ContextWithStore sets the store on the doc encryptor in the context. +// If the doc encryptor is not present, it will be created. func ContextWithStore(ctx context.Context, txn datastore.Txn) context.Context { - ctx, encryptor := getContextWithDocEnc(ctx) + ctx, encryptor := ensureContextWithDocEnc(ctx) encryptor.SetStore(txn.Encstore()) return ctx } @@ -60,7 +64,7 @@ func GetContextConfig(ctx context.Context) immutable.Option[DocEncConfig] { return immutable.None[DocEncConfig]() } -// SetContextConfig returns a new context with the encryption value set. +// SetContextConfig returns a new context with the doc encryption config set. func SetContextConfig(ctx context.Context, encConfig DocEncConfig) context.Context { return context.WithValue(ctx, configContextKey{}, encConfig) } diff --git a/internal/merkle/crdt/counter.go b/internal/merkle/crdt/counter.go index 7038097483..1ff6874b08 100644 --- a/internal/merkle/crdt/counter.go +++ b/internal/merkle/crdt/counter.go @@ -49,7 +49,7 @@ func NewMerkleCounter( // Save the value of the Counter to the DAG. func (mc *MerkleCounter) Save(ctx context.Context, data any) (cidlink.Link, []byte, error) { - value, ok := data.(*Field) + value, ok := data.(*DocField) if !ok { return cidlink.Link{}, nil, NewErrUnexpectedValueType(mc.reg.CType(), &client.FieldValue{}, data) } diff --git a/internal/merkle/crdt/field.go b/internal/merkle/crdt/field.go index cd26e70d28..9e3b95f69c 100644 --- a/internal/merkle/crdt/field.go +++ b/internal/merkle/crdt/field.go @@ -2,7 +2,11 @@ package merklecrdt import "github.com/sourcenetwork/defradb/client" -type Field struct { +// DocField is a struct that holds the document ID and the field value. +// This is used to a link between the document and the field value. +// For example, to check if the field value need be encrypted depending on the document-level +// encryption is enabled or not. +type DocField struct { DocID string FieldValue *client.FieldValue } diff --git a/internal/merkle/crdt/lwwreg.go b/internal/merkle/crdt/lwwreg.go index b76fac4e50..11e73089bf 100644 --- a/internal/merkle/crdt/lwwreg.go +++ b/internal/merkle/crdt/lwwreg.go @@ -47,7 +47,7 @@ func NewMerkleLWWRegister( // Save the value of the register to the DAG. func (mlwwreg *MerkleLWWRegister) Save(ctx context.Context, data any) (cidlink.Link, []byte, error) { - value, ok := data.(*Field) + value, ok := data.(*DocField) if !ok { return cidlink.Link{}, nil, NewErrUnexpectedValueType(client.LWW_REGISTER, &client.FieldValue{}, data) } From e326c59c09207027be9c40e87e5dc4d95d18bb9e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 24 Jun 2024 22:17:20 +0200 Subject: [PATCH 17/42] Add tests for encryptor --- internal/encryption/encryptor.go | 14 +-- internal/encryption/encryptor_test.go | 174 ++++++++++++++++++++++++++ internal/encryption/errors.go | 23 ++++ 3 files changed, 204 insertions(+), 7 deletions(-) create mode 100644 internal/encryption/encryptor_test.go create mode 100644 internal/encryption/errors.go diff --git a/internal/encryption/encryptor.go b/internal/encryption/encryptor.go index f22cee25ee..596e9f9903 100644 --- a/internal/encryption/encryptor.go +++ b/internal/encryption/encryptor.go @@ -26,6 +26,8 @@ var generateEncryptionKeyFunc = generateEncryptionKey const keyLength = 32 // 32 bytes for AES-256 +const testEncryptionKey = "examplekey1234567890examplekey12" + // generateEncryptionKey generates a random AES key. func generateEncryptionKey() ([]byte, error) { key := make([]byte, keyLength) @@ -37,7 +39,7 @@ func generateEncryptionKey() ([]byte, error) { // generateTestEncryptionKey generates a deterministic encryption key for testing. func generateTestEncryptionKey() ([]byte, error) { - return []byte("examplekey1234567890examplekey12"), nil + return []byte(testEncryptionKey), nil } type DocEncryptor struct { @@ -74,11 +76,9 @@ func (d *DocEncryptor) Encrypt(docID string, fieldID uint32, plainText []byte) ( return nil, err } - if d.store != nil { - err = d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) - if err != nil { - return nil, err - } + err = d.store.Put(d.ctx, storeKey.ToDS(), encryptionKey) + if err != nil { + return nil, err } } return EncryptAES(plainText, encryptionKey) @@ -100,7 +100,7 @@ func (d *DocEncryptor) Decrypt(docID string, fieldID uint32, cipherText []byte) func (d *DocEncryptor) fetchEncryptionKey(docID string, fieldID uint32) ([]byte, core.EncStoreDocKey, error) { storeKey := core.NewEncStoreDocKey(docID, fieldID) if d.store == nil { - return nil, core.EncStoreDocKey{}, nil + return nil, core.EncStoreDocKey{}, ErrNoStorageProvided } encryptionKey, err := d.store.Get(d.ctx, storeKey.ToDS()) isNotFound := errors.Is(err, ds.ErrNotFound) diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go new file mode 100644 index 0000000000..0428fb814a --- /dev/null +++ b/internal/encryption/encryptor_test.go @@ -0,0 +1,174 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "context" + "errors" + "testing" + + ds "github.com/ipfs/go-datastore" + "github.com/sourcenetwork/defradb/datastore/mocks" + "github.com/sourcenetwork/defradb/internal/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var testErr = errors.New("test error") + +var docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" + +func getPlainText() []byte { + return []byte("test") +} + +func getCipherText(t *testing.T) []byte { + cipherText, err := EncryptAES(getPlainText(), []byte(testEncryptionKey)) + assert.NoError(t, err) + return cipherText +} + +func TestEncryptorEncrypt_IfStorageReturnsError_Error(t *testing.T) { + enc := newDocEncryptor(context.Background()) + st := mocks.NewDSReaderWriter(t) + enc.SetStore(st) + + st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) + + _, err := enc.Encrypt(docID, 0, []byte("test")) + + assert.ErrorIs(t, err, testErr) +} + +func TestEncryptorEncrypt_IfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { + enc := newDocEncryptor(context.Background()) + st := mocks.NewDSReaderWriter(t) + enc.EnableKeyGeneration() + enc.SetStore(st) + + st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) + + storeKey := core.NewEncStoreDocKey(docID, 0) + + st.EXPECT().Put(mock.Anything, storeKey.ToDS(), []byte(testEncryptionKey)).Return(nil) + + cipherText, err := enc.Encrypt(docID, 0, getPlainText()) + + assert.NoError(t, err) + assert.Equal(t, getCipherText(t), cipherText) +} + +func TestEncryptorEncrypt_IfKeyFoundInStorage_ShouldUseItToReturnCipherText(t *testing.T) { + enc := newDocEncryptor(context.Background()) + st := mocks.NewDSReaderWriter(t) + enc.EnableKeyGeneration() + enc.SetStore(st) + + st.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte(testEncryptionKey), nil) + + cipherText, err := enc.Encrypt(docID, 0, getPlainText()) + + assert.NoError(t, err) + assert.Equal(t, getCipherText(t), cipherText) +} + +func TestEncryptorEncrypt_IfStorageFailsToStoreEncryptionKey_ReturnError(t *testing.T) { + enc := newDocEncryptor(context.Background()) + st := mocks.NewDSReaderWriter(t) + enc.EnableKeyGeneration() + enc.SetStore(st) + + st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) + + st.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything).Return(testErr) + + _, err := enc.Encrypt(docID, 0, getPlainText()) + + assert.ErrorIs(t, err, testErr) +} + +func TestEncryptorEncrypt_IfKeyGenerationIsNotEnabled_ShouldReturnPlainText(t *testing.T) { + enc := newDocEncryptor(context.Background()) + st := mocks.NewDSReaderWriter(t) + // we don call enc.EnableKeyGeneration() + enc.SetStore(st) + + st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) + + cipherText, err := enc.Encrypt(docID, 0, getPlainText()) + + assert.NoError(t, err) + assert.Equal(t, getPlainText(), cipherText) +} + +func TestEncryptorEncrypt_IfNoStorageProvided_Error(t *testing.T) { + enc := newDocEncryptor(context.Background()) + enc.EnableKeyGeneration() + // we don call enc.SetStore(st) + + _, err := enc.Encrypt(docID, 0, getPlainText()) + + assert.ErrorIs(t, err, ErrNoStorageProvided) +} + +func TestEncryptorDecrypt_IfNoStorageProvided_Error(t *testing.T) { + enc := newDocEncryptor(context.Background()) + enc.EnableKeyGeneration() + // we don call enc.SetStore(st) + + _, err := enc.Decrypt(docID, 0, getPlainText()) + + assert.ErrorIs(t, err, ErrNoStorageProvided) +} + +func TestEncryptorDecrypt_IfStorageReturnsError_Error(t *testing.T) { + enc := newDocEncryptor(context.Background()) + st := mocks.NewDSReaderWriter(t) + enc.SetStore(st) + + st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, testErr) + + _, err := enc.Decrypt(docID, 0, []byte("test")) + + assert.ErrorIs(t, err, testErr) +} + +func TestEncryptorDecrypt_IfKeyFoundInStorage_ShouldUseItToReturnPlainText(t *testing.T) { + enc := newDocEncryptor(context.Background()) + st := mocks.NewDSReaderWriter(t) + enc.EnableKeyGeneration() + enc.SetStore(st) + + st.EXPECT().Get(mock.Anything, mock.Anything).Return([]byte(testEncryptionKey), nil) + + plainText, err := enc.Decrypt(docID, 0, getCipherText(t)) + + assert.NoError(t, err) + assert.Equal(t, getPlainText(), plainText) +} + +func TestEncryptorDecrypt_IfNoKeyFoundInStorage_ShouldGenerateKeyStoreItAndReturnCipherText(t *testing.T) { + enc := newDocEncryptor(context.Background()) + st := mocks.NewDSReaderWriter(t) + enc.EnableKeyGeneration() + enc.SetStore(st) + + st.EXPECT().Get(mock.Anything, mock.Anything).Return(nil, ds.ErrNotFound) + + storeKey := core.NewEncStoreDocKey(docID, 0) + + st.EXPECT().Put(mock.Anything, storeKey.ToDS(), []byte(testEncryptionKey)).Return(nil) + + cipherText, err := enc.Encrypt(docID, 0, getPlainText()) + + assert.NoError(t, err) + assert.Equal(t, getCipherText(t), cipherText) +} diff --git a/internal/encryption/errors.go b/internal/encryption/errors.go new file mode 100644 index 0000000000..6a443ad834 --- /dev/null +++ b/internal/encryption/errors.go @@ -0,0 +1,23 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package encryption + +import ( + "github.com/sourcenetwork/defradb/errors" +) + +const ( + errNoStorageProvided string = "no storage provided" +) + +var ( + ErrNoStorageProvided = errors.New(errNoStorageProvided) +) From c86dc60c104c07fd6985b165ec4952b1dd8ca17e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 25 Jun 2024 12:59:31 +0200 Subject: [PATCH 18/42] Polish --- internal/merkle/clock/clock.go | 9 ++++----- internal/merkle/crdt/merklecrdt.go | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index c22b648606..f7dd5320ed 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -147,7 +147,6 @@ func (mc *MerkleClock) checkIfBlockEncryptionEnabled( if prevBlock.IsEncrypted != nil && *prevBlock.IsEncrypted { return true, nil } - // could do block.EncryptDelta(encKey) } return false, nil @@ -172,10 +171,10 @@ func (mc *MerkleClock) ProcessBlock( onlyHeads bool, ) error { if !onlyHeads { - err := mc.crdt.Merge(ctx, block.Delta.GetDelta()) - if err != nil { - return NewErrMergingDelta(blockLink.Cid, err) - } + err := mc.crdt.Merge(ctx, block.Delta.GetDelta()) + if err != nil { + return NewErrMergingDelta(blockLink.Cid, err) + } } return mc.updateHeads(ctx, block, blockLink) diff --git a/internal/merkle/crdt/merklecrdt.go b/internal/merkle/crdt/merklecrdt.go index e3a40207db..c7733be778 100644 --- a/internal/merkle/crdt/merklecrdt.go +++ b/internal/merkle/crdt/merklecrdt.go @@ -48,7 +48,7 @@ type MerkleClock interface { links ...coreblock.DAGLink, ) (cidlink.Link, []byte, error) // ProcessBlock processes a block and updates the CRDT state. - // The bool argument indicates whether only heads need to be updated. It is needed in case + // The bool argument indicates whether only heads need to be updated. It is needed in case // merge should be skipped for example if the block is encrypted. ProcessBlock(context.Context, *coreblock.Block, cidlink.Link, bool) error } From dfc104b00e8ca4c593a34a871fc1b45eabd5354a Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 25 Jun 2024 13:57:20 +0200 Subject: [PATCH 19/42] Encrypt counter CRDT fields --- internal/core/crdt/ipld_union.go | 8 +- tests/integration/encryption/commit_test.go | 112 +++++++++++++++++--- tests/integration/encryption/query_test.go | 58 ++++++++++ tests/integration/encryption/utils.go | 2 +- 4 files changed, 163 insertions(+), 17 deletions(-) diff --git a/internal/core/crdt/ipld_union.go b/internal/core/crdt/ipld_union.go index 262eafcca6..95023f28b2 100644 --- a/internal/core/crdt/ipld_union.go +++ b/internal/core/crdt/ipld_union.go @@ -153,21 +153,21 @@ func (c CRDT) GetStatus() uint8 { } // GetData returns the data of the delta. -// -// Currently only implemented for LWWRegDelta. func (c CRDT) GetData() []byte { if c.LWWRegDelta != nil { return c.LWWRegDelta.Data + } else if c.CounterDelta != nil { + return c.CounterDelta.Data } return nil } // SetData sets the data of the delta. -// -// Currently only implemented for LWWRegDelta. func (c CRDT) SetData(data []byte) { if c.LWWRegDelta != nil { c.LWWRegDelta.Data = data + } else if c.CounterDelta != nil { + c.CounterDelta.Data = data } } diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 393303fa1d..f499c79446 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -22,7 +22,9 @@ func encrypt(plaintext []byte) []byte { return val } -func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { +func TestDocEncryption_WithEncryptionOnLWWCRDT_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { + const docID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" + test := testUtils.TestCase{ Actions: []any{ updateUserCollectionSchema(), @@ -56,7 +58,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue(21)), - "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "docID": docID, "fieldId": "1", "fieldName": "age", "height": int64(1), @@ -66,7 +68,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", "collectionID": int64(1), "delta": encrypt(testUtils.CBORValue("John")), - "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "docID": docID, "fieldId": "2", "fieldName": "name", "height": int64(1), @@ -76,7 +78,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { "cid": "bafyreicvxlfxeqghmc3gy56rp5rzfejnbng4nu77x5e3wjinfydl6wvycq", "collectionID": int64(1), "delta": nil, - "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "docID": docID, "fieldId": "C", "fieldName": nil, "height": int64(1), @@ -99,7 +101,7 @@ func TestDocEncryption_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { testUtils.ExecuteTestCase(t, test) } -func TestDocEncryption_UponUpdate_ShouldEncryptedCommitDelta(t *testing.T) { +func TestDocEncryption_UponUpdateOnLWWCRDT_ShouldEncryptCommitDelta(t *testing.T) { test := testUtils.TestCase{ Actions: []any{ updateUserCollectionSchema(), @@ -138,12 +140,14 @@ func TestDocEncryption_UponUpdate_ShouldEncryptedCommitDelta(t *testing.T) { testUtils.ExecuteTestCase(t, test) } -func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptedOnlyRelevantDocs(t *testing.T) { +func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptOnlyRelevantDocs(t *testing.T) { + const johnDocID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" + const islamDocID = "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6" + test := testUtils.TestCase{ Actions: []any{ updateUserCollectionSchema(), testUtils.CreateDoc{ - // bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3 Doc: `{ "name": "John", "age": 21 @@ -151,7 +155,6 @@ func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptedOnlyRelevantDoc IsEncrypted: true, }, testUtils.CreateDoc{ - // bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6 Doc: `{ "name": "Islam", "age": 33 @@ -181,19 +184,104 @@ func TestDocEncryption_WithMultipleDocsUponUpdate_ShouldEncryptedOnlyRelevantDoc Results: []map[string]any{ { "delta": encrypt(testUtils.CBORValue(22)), - "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "docID": johnDocID, }, { "delta": encrypt(testUtils.CBORValue(21)), - "docID": "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3", + "docID": johnDocID, }, { "delta": testUtils.CBORValue(34), - "docID": "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6", + "docID": islamDocID, }, { "delta": testUtils.CBORValue(33), - "docID": "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6", + "docID": islamDocID, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldStoreCommitsDeltaEncrypted(t *testing.T) { + const docID = "bae-d3cc98b4-38d5-5c50-85a3-d3045d44094e" + + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + points: Int @crdt(type: "pcounter") + } + `}, + testUtils.CreateDoc{ + Doc: `{ "points": 5 }`, + IsEncrypted: true, + }, + testUtils.Request{ + Request: ` + query { + commits { + cid + delta + docID + } + } + `, + Results: []map[string]any{ + { + "cid": "bafyreieb6owsoljj4vondkx35ngxmhliauwvphicz4edufcy7biexij7mu", + "delta": encrypt(testUtils.CBORValue(5)), + "docID": docID, + }, + { + "cid": "bafyreif2lejhvdja2rmo237lrwpj45usrm55h6gzr4ewl6gajq3cl4ppsi", + "delta": nil, + "docID": docID, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestDocEncryption_UponUpdateOnCounterCRDT_ShouldEncryptedCommitDelta(t *testing.T) { + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + points: Int @crdt(type: "pcounter") + } + `}, + testUtils.CreateDoc{ + Doc: `{ "points": 5 }`, + IsEncrypted: true, + }, + testUtils.UpdateDoc{ + Doc: `{ + "points": 3 + }`, + }, + testUtils.Request{ + Request: ` + query { + commits(fieldId: "1") { + delta + } + } + `, + Results: []map[string]any{ + { + "delta": encrypt(testUtils.CBORValue(3)), + }, + { + "delta": encrypt(testUtils.CBORValue(5)), }, }, }, diff --git a/tests/integration/encryption/query_test.go b/tests/integration/encryption/query_test.go index bd3ed279c7..eb671d08fa 100644 --- a/tests/integration/encryption/query_test.go +++ b/tests/integration/encryption/query_test.go @@ -55,3 +55,61 @@ func TestDocEncryption_WithEncryption_ShouldFetchDecrypted(t *testing.T) { testUtils.ExecuteTestCase(t, test) } + +func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldFetchDecrypted(t *testing.T) { + const docID = "bae-ab8ae7d9-6473-5101-ba02-66b217948d7a" + + const query = ` + query { + Users { + _docID + name + points + } + }` + + test := testUtils.TestCase{ + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + points: Int @crdt(type: "pcounter") + } + `}, + testUtils.CreateDoc{ + Doc: `{ + "name": "John", + "points": 5 + }`, + IsEncrypted: true, + }, + testUtils.Request{ + Request: query, + Results: []map[string]any{ + { + "_docID": docID, + "name": "John", + "points": 5, + }, + }, + }, + testUtils.UpdateDoc{ + DocID: 0, + Doc: `{ "points": 3 }`, + }, + testUtils.Request{ + Request: query, + Results: []map[string]any{ + { + "_docID": docID, + "name": "John", + "points": 8, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/encryption/utils.go b/tests/integration/encryption/utils.go index 6bcb7acc83..62cae5b19f 100644 --- a/tests/integration/encryption/utils.go +++ b/tests/integration/encryption/utils.go @@ -17,7 +17,7 @@ import ( const userCollectionGQLSchema = (` type Users { name: String - age: Int + age: Int @crdt(type: "lww") verified: Boolean } `) From 35fa1fa624bb987652bf2465751993b630b73c28 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 25 Jun 2024 14:18:48 +0200 Subject: [PATCH 20/42] Fix lint --- internal/encryption/context.go | 3 ++- internal/encryption/encryptor_test.go | 5 +++-- internal/merkle/crdt/errors.go | 2 +- internal/merkle/crdt/field.go | 10 ++++++++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/internal/encryption/context.go b/internal/encryption/context.go index cf433ef54d..f1b3884390 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -13,8 +13,9 @@ package encryption import ( "context" - "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/immutable" + + "github.com/sourcenetwork/defradb/datastore" ) // docEncContextKey is the key type for document encryption context values. diff --git a/internal/encryption/encryptor_test.go b/internal/encryption/encryptor_test.go index 0428fb814a..10abd1f062 100644 --- a/internal/encryption/encryptor_test.go +++ b/internal/encryption/encryptor_test.go @@ -16,10 +16,11 @@ import ( "testing" ds "github.com/ipfs/go-datastore" - "github.com/sourcenetwork/defradb/datastore/mocks" - "github.com/sourcenetwork/defradb/internal/core" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + + "github.com/sourcenetwork/defradb/datastore/mocks" + "github.com/sourcenetwork/defradb/internal/core" ) var testErr = errors.New("test error") diff --git a/internal/merkle/crdt/errors.go b/internal/merkle/crdt/errors.go index 9e828df5dc..58ee8b6bc4 100644 --- a/internal/merkle/crdt/errors.go +++ b/internal/merkle/crdt/errors.go @@ -1,4 +1,4 @@ -// Copyright 2023 Democratized Data Foundation +// Copyright 2024 Democratized Data Foundation // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. diff --git a/internal/merkle/crdt/field.go b/internal/merkle/crdt/field.go index 9e3b95f69c..acc3203e67 100644 --- a/internal/merkle/crdt/field.go +++ b/internal/merkle/crdt/field.go @@ -1,3 +1,13 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + package merklecrdt import "github.com/sourcenetwork/defradb/client" From 3ed643568a590181243e11df8fa3eb42d4704624 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 25 Jun 2024 17:32:43 +0200 Subject: [PATCH 21/42] Update docs --- .../cli/defradb_client_collection_create.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/website/references/cli/defradb_client_collection_create.md b/docs/website/references/cli/defradb_client_collection_create.md index 425be82753..69b6aee8fa 100644 --- a/docs/website/references/cli/defradb_client_collection_create.md +++ b/docs/website/references/cli/defradb_client_collection_create.md @@ -5,6 +5,15 @@ Create a new document. ### Synopsis Create a new document. + +Options: + -i, --identity + Marks the document as private and set the identity as the owner. The access to the document + and permissions are controlled by ACP (Access Control Policy). + + -e, --encrypt + Encrypt flag specified if the document needs to be encrypted. If set DefraDB will generate a + symmetric key for encryption using AES-GCM. Example: create from string: defradb client collection create --name User '{ "name": "Bob" }' @@ -24,12 +33,13 @@ Example: create from stdin: ``` -defradb client collection create [-i --identity] [flags] +defradb client collection create [-i --identity] [-e --encrypt] [flags] ``` ### Options ``` + -e, --encrypt Encryption key used to encrypt/decrypt the document -f, --file string File containing document(s) -h, --help help for create ``` From e3135f25ee85d362b2610e8cb0ac5b916fbb1f21 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 26 Jun 2024 13:01:06 +0200 Subject: [PATCH 22/42] CreateMany tests action --- cli/utils.go | 2 +- http/client_collection.go | 6 ++++++ http/handler_collection.go | 12 ++++++++++-- internal/encryption/context.go | 18 ++++++++++-------- internal/merkle/clock/clock.go | 3 ++- tests/integration/utils2.go | 4 ---- 6 files changed, 29 insertions(+), 16 deletions(-) diff --git a/cli/utils.go b/cli/utils.go index 9289b2c4e0..b2d4c076bc 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -167,7 +167,7 @@ func setContextDocEncryption(cmd *cobra.Command, shouldEncrypt bool, txn datasto if !shouldEncrypt { return } - ctx := encryption.Context(cmd.Context()) + ctx := cmd.Context() if txn != nil { ctx = encryption.ContextWithStore(ctx, txn) } diff --git a/http/client_collection.go b/http/client_collection.go index ee614c1dba..e07c8c8d81 100644 --- a/http/client_collection.go +++ b/http/client_collection.go @@ -24,6 +24,7 @@ import ( sse "github.com/vito/go-sse/sse" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/internal/encryption" ) var _ client.Collection = (*Collection)(nil) @@ -78,6 +79,11 @@ func (c *Collection) Create( return err } + encConf := encryption.GetContextConfig(ctx) + if encConf.HasValue() && encConf.Value().IsEncrypted { + req.Header.Set(DocEncryptionHeader, "1") + } + _, err = c.http.request(req) if err != nil { return err diff --git a/http/handler_collection.go b/http/handler_collection.go index 60c18b3442..2d5111671f 100644 --- a/http/handler_collection.go +++ b/http/handler_collection.go @@ -21,8 +21,11 @@ import ( "github.com/go-chi/chi/v5" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/internal/encryption" ) +const DocEncryptionHeader = "EncryptDoc" + type collectionHandler struct{} type CollectionDeleteRequest struct { @@ -43,6 +46,11 @@ func (s *collectionHandler) Create(rw http.ResponseWriter, req *http.Request) { return } + ctx := req.Context() + if req.Header.Get(DocEncryptionHeader) == "1" { + ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) + } + switch { case client.IsJSONArray(data): docList, err := client.NewDocsFromJSON(data, col.Definition()) @@ -51,7 +59,7 @@ func (s *collectionHandler) Create(rw http.ResponseWriter, req *http.Request) { return } - if err := col.CreateMany(req.Context(), docList); err != nil { + if err := col.CreateMany(ctx, docList); err != nil { responseJSON(rw, http.StatusBadRequest, errorResponse{err}) return } @@ -62,7 +70,7 @@ func (s *collectionHandler) Create(rw http.ResponseWriter, req *http.Request) { responseJSON(rw, http.StatusBadRequest, errorResponse{err}) return } - if err := col.Create(req.Context(), doc); err != nil { + if err := col.Create(ctx, doc); err != nil { responseJSON(rw, http.StatusBadRequest, errorResponse{err}) return } diff --git a/internal/encryption/context.go b/internal/encryption/context.go index f1b3884390..10a03c89c1 100644 --- a/internal/encryption/context.go +++ b/internal/encryption/context.go @@ -28,9 +28,19 @@ type configContextKey struct{} // it was retrieved from the given context. func TryGetContextEncryptor(ctx context.Context) (*DocEncryptor, bool) { enc, ok := ctx.Value(docEncContextKey{}).(*DocEncryptor) + if ok { + checkKeyGenerationFlag(ctx, enc) + } return enc, ok } +func checkKeyGenerationFlag(ctx context.Context, enc *DocEncryptor) { + encConfig := GetContextConfig(ctx) + if encConfig.HasValue() && encConfig.Value().IsEncrypted { + enc.EnableKeyGeneration() + } +} + func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncryptor) { enc, ok := TryGetContextEncryptor(ctx) if !ok { @@ -40,14 +50,6 @@ func ensureContextWithDocEnc(ctx context.Context) (context.Context, *DocEncrypto return ctx, enc } -// Context enables key generation on the doc encryptor in the context. -// If the doc encryptor is not present, it will be created. -func Context(ctx context.Context) context.Context { - ctx, encryptor := ensureContextWithDocEnc(ctx) - encryptor.EnableKeyGeneration() - return ctx -} - // ContextWithStore sets the store on the doc encryptor in the context. // If the doc encryptor is not present, it will be created. func ContextWithStore(ctx context.Context, txn datastore.Txn) context.Context { diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index f7dd5320ed..3cfbb16f92 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -93,13 +93,14 @@ func (mc *MerkleClock) AddDelta( } var dagBlock *coreblock.Block - if isEncrypted { + if isEncrypted && !block.Delta.IsComposite(){ dagBlock, err = encryptBlock(ctx, block) if err != nil { return cidlink.Link{}, nil, err } } else { dagBlock = block + dagBlock.IsEncrypted = &isEncrypted } link, err := mc.putBlock(ctx, dagBlock) diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index d97cdeb808..34f77e8f18 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1240,12 +1240,8 @@ func createDocViaColSave( func makeContextForDocCreate(ctx context.Context, action *CreateDoc, txn datastore.Txn) context.Context { ctx = db.SetContextIdentity(ctx, action.Identity) if action.IsEncrypted { - ctx = encryption.Context(ctx) ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) } - if txn != nil { - ctx = encryption.ContextWithStore(ctx, txn) - } return ctx } From 662e8e1c476e714141a4e604a6122d45618b8a1d Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 26 Jun 2024 16:36:12 +0200 Subject: [PATCH 23/42] Enable CreateMany in integration tests --- client/document.go | 4 +- client/document_test.go | 62 ++++++++++++++++ tests/integration/encryption/commit_test.go | 67 ++++++++++++++++++ tests/integration/gql.go | 37 ++++++++-- tests/integration/utils2.go | 78 ++++++++++++++------- 5 files changed, 216 insertions(+), 32 deletions(-) diff --git a/client/document.go b/client/document.go index ada47cc8f9..a95e4bb91b 100644 --- a/client/document.go +++ b/client/document.go @@ -122,7 +122,9 @@ var jsonArrayPattern = regexp.MustCompile(`^\s*\[.*\]\s*$`) // IsJSONArray returns true if the given byte array is a JSON Array. func IsJSONArray(obj []byte) bool { - return jsonArrayPattern.Match(obj) + var js []interface{} + err := json.Unmarshal(obj, &js) + return err == nil } // NewFromJSON creates a new instance of a Document from a raw JSON object byte array. diff --git a/client/document_test.go b/client/document_test.go index b15c7b019a..450a181af0 100644 --- a/client/document_test.go +++ b/client/document_test.go @@ -206,3 +206,65 @@ func TestNewFromJSON_WithInvalidJSONFieldValueSimpleString_Error(t *testing.T) { _, err := NewDocFromJSON(objWithJSONField, def) require.ErrorContains(t, err, "invalid JSON payload. Payload: blah") } + +func TestIsJSONArray(t *testing.T) { + tests := []struct { + name string + input []byte + expected bool + }{ + { + name: "Valid JSON Array", + input: []byte(`[{"name":"John","age":21},{"name":"Islam","age":33}]`), + expected: true, + }, + { + name: "Valid Empty JSON Array", + input: []byte(`[]`), + expected: true, + }, + { + name: "Valid JSON Object", + input: []byte(`{"name":"John","age":21}`), + expected: false, + }, + { + name: "Invalid JSON String", + input: []byte(`{"name":"John","age":21`), + expected: false, + }, + { + name: "Non-JSON String", + input: []byte(`Hello, World!`), + expected: false, + }, + { + name: "Array of Primitives", + input: []byte(`[1, 2, 3, 4]`), + expected: true, + }, + { + name: "Nested JSON Array", + input: []byte(`[[1, 2], [3, 4]]`), + expected: true, + }, + { + name: "Valid JSON Array with Whitespace", + input: []byte(` + [ + {"name": "John", "age": 21}, + {"name": "Islam", "age": 33} + ]`), + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := IsJSONArray(tt.input) + if actual != tt.expected { + t.Errorf("IsJSONArray(%s) = %v; expected %v", tt.input, actual, tt.expected) + } + }) + } +} diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index f499c79446..9608349f6e 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -290,3 +290,70 @@ func TestDocEncryption_UponUpdateOnCounterCRDT_ShouldEncryptedCommitDelta(t *tes testUtils.ExecuteTestCase(t, test) } + +func TestDocEncryption_UponEncryptionSeveralDocs_ShouldStoreAllCommitsDeltaEncrypted(t *testing.T) { + const johnDocID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" + const islamDocID = "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6" + + test := testUtils.TestCase{ + Actions: []any{ + updateUserCollectionSchema(), + testUtils.CreateDoc{ + Doc: `[{ + "name": "John", + "age": 21 + }, + { + "name": "Islam", + "age": 33 + }]`, + IsEncrypted: true, + }, + testUtils.Request{ + Request: ` + query { + commits { + cid + delta + docID + } + } + `, + Results: []map[string]any{ + { + "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", + "delta": encrypt(testUtils.CBORValue(21)), + "docID": johnDocID, + }, + { + "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", + "delta": encrypt(testUtils.CBORValue("John")), + "docID": johnDocID, + }, + { + "cid": "bafyreicvxlfxeqghmc3gy56rp5rzfejnbng4nu77x5e3wjinfydl6wvycq", + "delta": nil, + "docID": johnDocID, + }, + { + "cid": "bafyreibe24bo67owxewoso3ekinera2bhusguij5qy2ahgyufaq3fbvaxa", + "delta": encrypt(testUtils.CBORValue(33)), + "docID": islamDocID, + }, + { + "cid": "bafyreie2fddpidgc62fhd2fjrsucq3spgh2mgvto2xwolcdmdhb5pdeok4", + "delta": encrypt(testUtils.CBORValue("Islam")), + "docID": islamDocID, + }, + { + "cid": "bafyreifulxdkf4m3wmmdxjg43l4mw7uuxl5il27eabklc22nptilrh64sa", + "delta": nil, + "docID": islamDocID, + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} diff --git a/tests/integration/gql.go b/tests/integration/gql.go index 22a368adf7..1f6cd26d6e 100644 --- a/tests/integration/gql.go +++ b/tests/integration/gql.go @@ -14,15 +14,27 @@ import ( "encoding/json" "fmt" "strings" + + "github.com/sourcenetwork/defradb/client" ) -// jsonToGql transforms a json doc string to a gql string. +// jsonToGQL transforms a json doc string to a gql string. func jsonToGQL(val string) (string, error) { - var doc map[string]any - if err := json.Unmarshal([]byte(val), &doc); err != nil { - return "", err + bytes := []byte(val) + + if client.IsJSONArray(bytes) { + var doc []map[string]any + if err := json.Unmarshal(bytes, &doc); err != nil { + return "", err + } + return arrayToGQL(doc) + } else { + var doc map[string]any + if err := json.Unmarshal(bytes, &doc); err != nil { + return "", err + } + return mapToGQL(doc) } - return mapToGQL(doc) } // valueToGQL transforms a value to a gql string. @@ -41,7 +53,7 @@ func valueToGQL(val any) (string, error) { return string(out), nil } -// mapToGql transforms a map to a gql string. +// mapToGQL transforms a map to a gql string. func mapToGQL(val map[string]any) (string, error) { var entries []string for k, v := range val { @@ -66,3 +78,16 @@ func sliceToGQL(val []any) (string, error) { } return fmt.Sprintf("[%s]", strings.Join(entries, ",")), nil } + +// arrayToGQL transforms an array of maps to a gql string. +func arrayToGQL(val []map[string]any) (string, error) { + var entries []string + for _, v := range val { + out, err := mapToGQL(v) + if err != nil { + return "", err + } + entries = append(entries, out) + } + return fmt.Sprintf("[%s]", strings.Join(entries, ",")), nil +} diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 34f77e8f18..ad7649d678 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1175,7 +1175,7 @@ func createDoc( substituteRelations(s, action) } - var mutation func(*state, CreateDoc, client.DB, []client.Collection) (*client.Document, error) + var mutation func(*state, CreateDoc, client.DB, []client.Collection) ([]*client.Document, error) switch mutationType { case CollectionSaveMutationType: @@ -1189,7 +1189,7 @@ func createDoc( } var expectedErrorRaised bool - var doc *client.Document + var docs []*client.Document actionNodes := getNodes(action.NodeID, s.nodes) for nodeID, collections := range getNodeCollections(action.NodeID, s.collections) { err := withRetry( @@ -1197,7 +1197,7 @@ func createDoc( nodeID, func() error { var err error - doc, err = mutation(s, action, actionNodes[nodeID], collections) + docs, err = mutation(s, action, actionNodes[nodeID], collections) return err }, ) @@ -1210,7 +1210,7 @@ func createDoc( // Expand the slice if required, so that the document can be accessed by collection index s.documents = append(s.documents, make([][]*client.Document, action.CollectionID-len(s.documents)+1)...) } - s.documents[action.CollectionID] = append(s.documents[action.CollectionID], doc) + s.documents[action.CollectionID] = append(s.documents[action.CollectionID], docs...) } func createDocViaColSave( @@ -1218,23 +1218,35 @@ func createDocViaColSave( action CreateDoc, node client.DB, collections []client.Collection, -) (*client.Document, error) { - var err error +) ([]*client.Document, error) { + var docs []*client.Document var doc *client.Document + var err error if action.DocMap != nil { doc, err = client.NewDocFromMap(action.DocMap, collections[action.CollectionID].Definition()) + docs = []*client.Document{doc} } else { - doc, err = client.NewDocFromJSON([]byte(action.Doc), collections[action.CollectionID].Definition()) - } - if err != nil { - return nil, err + bytes := []byte(action.Doc) + if client.IsJSONArray(bytes) { + docs, err = client.NewDocsFromJSON(bytes, collections[action.CollectionID].Definition()) + } else { + doc, err = client.NewDocFromJSON(bytes, collections[action.CollectionID].Definition()) + docs = []*client.Document{doc} + } } + require.NoError(s.t, err) txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action, txn) - return doc, collections[action.CollectionID].Save(ctx, doc) + for _, doc := range docs { + err = collections[action.CollectionID].Save(ctx, doc) + if err != nil { + return nil, err + } + } + return docs, nil } func makeContextForDocCreate(ctx context.Context, action *CreateDoc, txn datastore.Txn) context.Context { @@ -1250,23 +1262,34 @@ func createDocViaColCreate( action CreateDoc, node client.DB, collections []client.Collection, -) (*client.Document, error) { - var err error +) ([]*client.Document, error) { + var docs []*client.Document var doc *client.Document + var err error if action.DocMap != nil { doc, err = client.NewDocFromMap(action.DocMap, collections[action.CollectionID].Definition()) + docs = []*client.Document{doc} } else { - doc, err = client.NewDocFromJSON([]byte(action.Doc), collections[action.CollectionID].Definition()) - } - if err != nil { - return nil, err + if client.IsJSONArray([]byte(action.Doc)) { + docs, err = client.NewDocsFromJSON([]byte(action.Doc), collections[action.CollectionID].Definition()) + } else { + doc, err = client.NewDocFromJSON([]byte(action.Doc), collections[action.CollectionID].Definition()) + docs = []*client.Document{doc} + } } + require.NoError(s.t, err) txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action, txn) - return doc, collections[action.CollectionID].Create(ctx, doc) + if len(docs) > 1 { + err = collections[action.CollectionID].CreateMany(ctx, docs) + } else { + err = collections[action.CollectionID].Create(ctx, doc) + } + + return docs, err } func createDocViaGQL( @@ -1274,7 +1297,7 @@ func createDocViaGQL( action CreateDoc, node client.DB, collections []client.Collection, -) (*client.Document, error) { +) ([]*client.Document, error) { collection := collections[action.CollectionID] var err error var input string @@ -1313,14 +1336,19 @@ func createDocViaGQL( return nil, nil } - docIDString := resultantDocs[0]["_docID"].(string) - docID, err := client.NewDocIDFromString(docIDString) - require.NoError(s.t, err) + var docs []*client.Document + for _, docMap := range resultantDocs { + docIDString := docMap["_docID"].(string) + docID, err := client.NewDocIDFromString(docIDString) + require.NoError(s.t, err) - doc, err := collection.Get(ctx, docID, false) - require.NoError(s.t, err) + doc, err := collection.Get(ctx, docID, false) + require.NoError(s.t, err) + + docs = append(docs, doc) + } - return doc, nil + return docs, nil } // substituteRelations scans the fields defined in [action.DocMap], if any are of type [DocIndex] From bb3c506988e7a37d98bb8638e981fccea7fcbcc7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 26 Jun 2024 16:37:06 +0200 Subject: [PATCH 24/42] Adjust CLI client --- tests/clients/cli/wrapper_collection.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/clients/cli/wrapper_collection.go b/tests/clients/cli/wrapper_collection.go index 62458dae99..c22b87786a 100644 --- a/tests/clients/cli/wrapper_collection.go +++ b/tests/clients/cli/wrapper_collection.go @@ -21,6 +21,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/http" + "github.com/sourcenetwork/defradb/internal/encryption" ) var _ client.Collection = (*Collection)(nil) @@ -65,6 +66,11 @@ func (c *Collection) Create( args := []string{"client", "collection", "create"} args = append(args, "--name", c.Description().Name.Value()) + encConf := encryption.GetContextConfig(ctx) + if encConf.HasValue() && encConf.Value().IsEncrypted { + args = append(args, "--encrypt") + } + document, err := doc.String() if err != nil { return err From 8cdc986490c6196a767a2c593f1d6264453c7f09 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 26 Jun 2024 22:21:43 +0200 Subject: [PATCH 25/42] Roll back some prev change --- tests/integration/utils2.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index ad7649d678..8eaa51405e 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1234,7 +1234,9 @@ func createDocViaColSave( docs = []*client.Document{doc} } } - require.NoError(s.t, err) + if err != nil { + return nil, err + } txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) @@ -1277,7 +1279,9 @@ func createDocViaColCreate( docs = []*client.Document{doc} } } - require.NoError(s.t, err) + if err != nil { + return nil, err + } txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) From bb9ce22e2faa613e781200f9a33e3187ca879a04 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 26 Jun 2024 23:19:13 +0200 Subject: [PATCH 26/42] Set Block.IsEncrypted only if true --- internal/merkle/clock/clock.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index 3cfbb16f92..fde0e8b4e0 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -92,15 +92,16 @@ func (mc *MerkleClock) AddDelta( return cidlink.Link{}, nil, err } - var dagBlock *coreblock.Block - if isEncrypted && !block.Delta.IsComposite(){ - dagBlock, err = encryptBlock(ctx, block) - if err != nil { - return cidlink.Link{}, nil, err + dagBlock := block + if isEncrypted { + if !block.Delta.IsComposite() { + dagBlock, err = encryptBlock(ctx, block) + if err != nil { + return cidlink.Link{}, nil, err + } + } else { + dagBlock.IsEncrypted = &isEncrypted } - } else { - dagBlock = block - dagBlock.IsEncrypted = &isEncrypted } link, err := mc.putBlock(ctx, dagBlock) From 1fc60f70f65d9c41ae60e5d50f74db56d5a307a8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 27 Jun 2024 07:51:55 +0200 Subject: [PATCH 27/42] Polish --- client/document.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/document.go b/client/document.go index a95e4bb91b..3105e5a2fb 100644 --- a/client/document.go +++ b/client/document.go @@ -13,7 +13,6 @@ package client import ( "encoding/json" "errors" - "regexp" "strings" "sync" "time" @@ -118,11 +117,9 @@ func NewDocFromMap(data map[string]any, collectionDefinition CollectionDefinitio return doc, nil } -var jsonArrayPattern = regexp.MustCompile(`^\s*\[.*\]\s*$`) - // IsJSONArray returns true if the given byte array is a JSON Array. func IsJSONArray(obj []byte) bool { - var js []interface{} + var js []any err := json.Unmarshal(obj, &js) return err == nil } From 6d90c1f6fafdbe80d0774e0a403de2e7da7c40c8 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 27 Jun 2024 09:34:29 +0200 Subject: [PATCH 28/42] Fix CreateMany for QGL mutation type --- .../events/simple/with_update_test.go | 9 ++- tests/integration/utils2.go | 77 +++++++++++-------- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/tests/integration/events/simple/with_update_test.go b/tests/integration/events/simple/with_update_test.go index 0b49486aa4..2d5fbfc5fe 100644 --- a/tests/integration/events/simple/with_update_test.go +++ b/tests/integration/events/simple/with_update_test.go @@ -49,17 +49,18 @@ func TestEventsSimpleWithUpdate(t *testing.T) { "Users": []func(c client.Collection){ func(c client.Collection) { err = c.Save(context.Background(), doc1) - assert.Nil(t, err) + assert.NoError(t, err) }, func(c client.Collection) { err = c.Save(context.Background(), doc2) - assert.Nil(t, err) + assert.NoError(t, err) }, func(c client.Collection) { // Update John - doc1.Set("name", "Johnnnnn") + err = doc1.Set("name", "Johnnnnn") + assert.NoError(t, err) err = c.Save(context.Background(), doc1) - assert.Nil(t, err) + assert.NoError(t, err) }, }, }, diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 8eaa51405e..39e43d067e 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -12,6 +12,7 @@ package tests import ( "context" + "encoding/json" "fmt" "os" "reflect" @@ -883,11 +884,7 @@ func refreshDocuments( continue } - txn := s.txns[0] - if action.NodeID.HasValue() { - txn = s.txns[action.NodeID.Value()] - } - ctx := makeContextForDocCreate(s.ctx, &action, txn) + ctx := makeContextForDocCreate(s.ctx, &action) // The document may have been mutated by other actions, so to be sure we have the latest // version without having to worry about the individual update mechanics we fetch it. @@ -1240,7 +1237,7 @@ func createDocViaColSave( txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action, txn) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) for _, doc := range docs { err = collections[action.CollectionID].Save(ctx, doc) @@ -1251,7 +1248,7 @@ func createDocViaColSave( return docs, nil } -func makeContextForDocCreate(ctx context.Context, action *CreateDoc, txn datastore.Txn) context.Context { +func makeContextForDocCreate(ctx context.Context, action *CreateDoc) context.Context { ctx = db.SetContextIdentity(ctx, action.Identity) if action.IsEncrypted { ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) @@ -1285,7 +1282,7 @@ func createDocViaColCreate( txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action, txn) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) if len(docs) > 1 { err = collections[action.CollectionID].CreateMany(ctx, docs) @@ -1303,46 +1300,58 @@ func createDocViaGQL( collections []client.Collection, ) ([]*client.Document, error) { collection := collections[action.CollectionID] - var err error - var input string + var inputs []string if action.DocMap != nil { - input, err = valueToGQL(action.DocMap) + input, err := valueToGQL(action.DocMap) + require.NoError(s.t, err) + inputs = append(inputs, input) + } else if client.IsJSONArray([]byte(action.Doc)) { + var docMaps []map[string]any + err := json.Unmarshal([]byte(action.Doc), &docMaps) + require.NoError(s.t, err) + for _, docMap := range docMaps { + input, err := valueToGQL(docMap) + require.NoError(s.t, err) + inputs = append(inputs, input) + } } else { - input, err = jsonToGQL(action.Doc) + input, err := jsonToGQL(action.Doc) + require.NoError(s.t, err) + inputs = append(inputs, input) } - require.NoError(s.t, err) - request := fmt.Sprintf( - `mutation { + var docs []*client.Document + + for _, input := range inputs { + request := fmt.Sprintf( + `mutation { create_%s(input: %s) { _docID } }`, - collection.Name().Value(), - input, - ) + collection.Name().Value(), + input, + ) - txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) + txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action, txn) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) - result := node.ExecRequest( - ctx, - request, - ) - if len(result.GQL.Errors) > 0 { - return nil, result.GQL.Errors[0] - } + result := node.ExecRequest( + ctx, + request, + ) + if len(result.GQL.Errors) > 0 { + return nil, result.GQL.Errors[0] + } - resultantDocs, ok := result.GQL.Data.([]map[string]any) - if !ok || len(resultantDocs) == 0 { - return nil, nil - } + resultantDocs, ok := result.GQL.Data.([]map[string]any) + if !ok || len(resultantDocs) == 0 { + return nil, nil + } - var docs []*client.Document - for _, docMap := range resultantDocs { - docIDString := docMap["_docID"].(string) + docIDString := resultantDocs[0]["_docID"].(string) docID, err := client.NewDocIDFromString(docIDString) require.NoError(s.t, err) From 0ba0a96300801a4bb2fffd8af301126c8a1dfeb0 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 27 Jun 2024 15:04:14 +0200 Subject: [PATCH 29/42] Fix GQL mutation --- cli/collection_create.go | 4 ++-- cli/request.go | 13 +++++++++++++ http/client.go | 7 +++++++ http/handler_store.go | 8 +++++++- tests/clients/cli/wrapper.go | 6 ++++++ 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/cli/collection_create.go b/cli/collection_create.go index 9525b821fa..f4c36fbd53 100644 --- a/cli/collection_create.go +++ b/cli/collection_create.go @@ -34,7 +34,7 @@ Options: and permissions are controlled by ACP (Access Control Policy). -e, --encrypt - Encrypt flag specified if the document needs to be encrypted. If set DefraDB will generate a + Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a symmetric key for encryption using AES-GCM. Example: create from string: @@ -99,7 +99,7 @@ Example: create from stdin: }, } cmd.PersistentFlags().BoolVarP(&shouldEncrypt, "encrypt", "e", false, - "Encryption key used to encrypt/decrypt the document") + "Flag to enable encryption of the document") cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)") return cmd } diff --git a/cli/request.go b/cli/request.go index b6ec8e05ce..795046ece9 100644 --- a/cli/request.go +++ b/cli/request.go @@ -26,11 +26,21 @@ const ( func MakeRequestCommand() *cobra.Command { var filePath string + var shouldEncrypt bool var cmd = &cobra.Command{ Use: "query [-i --identity] [request]", Short: "Send a DefraDB GraphQL query request", Long: `Send a DefraDB GraphQL query request to the database. +Options: + -i, --identity + Marks the document as private and set the identity as the owner. The access to the document + and permissions are controlled by ACP (Access Control Policy). + + -e, --encrypt + Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a + symmetric key for encryption using AES-GCM. + A query request can be sent as a single argument. Example command: defradb client query 'query { ... }' @@ -71,6 +81,7 @@ To learn more about the DefraDB GraphQL Query Language, refer to https://docs.so } store := mustGetContextStore(cmd) + setContextDocEncryption(cmd, shouldEncrypt, nil) result := store.ExecRequest(cmd.Context(), request) var errors []string @@ -89,6 +100,8 @@ To learn more about the DefraDB GraphQL Query Language, refer to https://docs.so }, } + cmd.PersistentFlags().BoolVarP(&shouldEncrypt, "encrypt", "e", false, + "Flag to enable encryption of the document") cmd.Flags().StringVarP(&filePath, "file", "f", "", "File containing the query request") return cmd } diff --git a/http/client.go b/http/client.go index 2082604599..8105ccb6e9 100644 --- a/http/client.go +++ b/http/client.go @@ -29,6 +29,7 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/event" + "github.com/sourcenetwork/defradb/internal/encryption" ) var _ client.DB = (*Client)(nil) @@ -355,6 +356,12 @@ func (c *Client) ExecRequest( return result } err = c.http.setDefaultHeaders(req) + + encConf := encryption.GetContextConfig(ctx) + if encConf.HasValue() && encConf.Value().IsEncrypted { + req.Header.Set(DocEncryptionHeader, "1") + } + if err != nil { result.GQL.Errors = []error{err} return result diff --git a/http/handler_store.go b/http/handler_store.go index de534a8c1d..df2136db87 100644 --- a/http/handler_store.go +++ b/http/handler_store.go @@ -22,6 +22,7 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/internal/encryption" ) type storeHandler struct{} @@ -312,7 +313,12 @@ func (s *storeHandler) ExecRequest(rw http.ResponseWriter, req *http.Request) { return } - result := store.ExecRequest(req.Context(), request.Query) + ctx := req.Context() + if req.Header.Get(DocEncryptionHeader) == "1" { + ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) + } + + result := store.ExecRequest(ctx, request.Query) if result.Subscription == nil { responseJSON(rw, http.StatusOK, GraphQLResponse{result.GQL.Data, result.GQL.Errors}) diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index a6f3feef4c..9e5f734454 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -31,6 +31,7 @@ import ( "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/http" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/node" ) @@ -399,6 +400,11 @@ func (w *Wrapper) ExecRequest( result := &client.RequestResult{} + encCond := encryption.GetContextConfig(ctx) + if encCond.HasValue() && encCond.Value().IsEncrypted { + args = append(args, "--encrypt") + } + stdOut, stdErr, err := w.cmd.executeStream(ctx, args) if err != nil { result.GQL.Errors = []error{err} From bfffa9894ea271cb361e96bd2b27fd46e09575f1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 27 Jun 2024 15:08:10 +0200 Subject: [PATCH 30/42] Update docs --- .../references/cli/defradb_client_collection_create.md | 4 ++-- docs/website/references/cli/defradb_client_query.md | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/website/references/cli/defradb_client_collection_create.md b/docs/website/references/cli/defradb_client_collection_create.md index 69b6aee8fa..5425e3f860 100644 --- a/docs/website/references/cli/defradb_client_collection_create.md +++ b/docs/website/references/cli/defradb_client_collection_create.md @@ -12,7 +12,7 @@ Options: and permissions are controlled by ACP (Access Control Policy). -e, --encrypt - Encrypt flag specified if the document needs to be encrypted. If set DefraDB will generate a + Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a symmetric key for encryption using AES-GCM. Example: create from string: @@ -39,7 +39,7 @@ defradb client collection create [-i --identity] [-e --encrypt] [flag ### Options ``` - -e, --encrypt Encryption key used to encrypt/decrypt the document + -e, --encrypt Flag to enable encryption of the document -f, --file string File containing document(s) -h, --help help for create ``` diff --git a/docs/website/references/cli/defradb_client_query.md b/docs/website/references/cli/defradb_client_query.md index ec868456b1..7c9b8e72fc 100644 --- a/docs/website/references/cli/defradb_client_query.md +++ b/docs/website/references/cli/defradb_client_query.md @@ -6,6 +6,15 @@ Send a DefraDB GraphQL query request Send a DefraDB GraphQL query request to the database. +Options: + -i, --identity + Marks the document as private and set the identity as the owner. The access to the document + and permissions are controlled by ACP (Access Control Policy). + + -e, --encrypt + Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a + symmetric key for encryption using AES-GCM. + A query request can be sent as a single argument. Example command: defradb client query 'query { ... }' @@ -30,6 +39,7 @@ defradb client query [-i --identity] [request] [flags] ### Options ``` + -e, --encrypt Flag to enable encryption of the document -f, --file string File containing the query request -h, --help help for query ``` From ca1bda7f54d03578a5b5acb4be4172486fa5e564 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 27 Jun 2024 18:08:06 +0200 Subject: [PATCH 31/42] Add encConf context upon http CreateMany --- http/client_collection.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/http/client_collection.go b/http/client_collection.go index e07c8c8d81..4190d28963 100644 --- a/http/client_collection.go +++ b/http/client_collection.go @@ -120,6 +120,11 @@ func (c *Collection) CreateMany( return err } + encConf := encryption.GetContextConfig(ctx) + if encConf.HasValue() && encConf.Value().IsEncrypted { + req.Header.Set(DocEncryptionHeader, "1") + } + _, err = c.http.request(req) if err != nil { return err From 485f5fccbbb126033a3f05b3a1726f2924e4c2a5 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Thu, 27 Jun 2024 18:29:48 +0200 Subject: [PATCH 32/42] Add separate regular CreateMany test --- .../create/simple_create_many_test.go | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/integration/mutation/create/simple_create_many_test.go diff --git a/tests/integration/mutation/create/simple_create_many_test.go b/tests/integration/mutation/create/simple_create_many_test.go new file mode 100644 index 0000000000..aa4dda3143 --- /dev/null +++ b/tests/integration/mutation/create/simple_create_many_test.go @@ -0,0 +1,70 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package create + +import ( + "testing" + + testUtils "github.com/sourcenetwork/defradb/tests/integration" +) + +func TestMutationCreateMany(t *testing.T) { + test := testUtils.TestCase{ + Description: "Simple create mutation", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type Users { + name: String + age: Int + } + `, + }, + testUtils.CreateDoc{ + Doc: `[ + { + "name": "John", + "age": 27 + }, + { + "name": "Islam", + "age": 33 + } + ]`, + }, + testUtils.Request{ + Request: ` + query { + Users { + _docID + name + age + } + } + `, + Results: []map[string]any{ + { + "_docID": "bae-48339725-ed14-55b1-8e63-3fda5f590725", + "name": "Islam", + "age": int64(33), + }, + { + "_docID": "bae-8c89a573-c287-5d8c-8ba6-c47c814c594d", + "name": "John", + "age": int64(27), + }, + }, + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From b812dc8f044a31071fd2eeb55c111e0f90df54cf Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 28 Jun 2024 12:26:05 +0200 Subject: [PATCH 33/42] Make create mutation accept array of docs --- client/request/consts.go | 1 + client/request/mutation.go | 6 + internal/planner/create.go | 105 ++++++++---------- internal/planner/mapper/mapper.go | 1 + internal/planner/mapper/mutation.go | 13 +-- internal/planner/planner.go | 2 +- internal/request/graphql/parser/mutation.go | 12 ++ internal/request/graphql/schema/generate.go | 44 +++++--- .../explain/default/create_test.go | 8 +- .../explain/execute/create_test.go | 4 +- .../create/simple_create_many_test.go | 2 +- tests/integration/utils2.go | 66 ++++++----- 12 files changed, 141 insertions(+), 123 deletions(-) diff --git a/client/request/consts.go b/client/request/consts.go index 1a1d653a25..86bd887c92 100644 --- a/client/request/consts.go +++ b/client/request/consts.go @@ -21,6 +21,7 @@ const ( Cid = "cid" Input = "input" + Inputs = "inputs" FieldName = "field" FieldIDName = "fieldId" ShowDeleted = "showDeleted" diff --git a/client/request/mutation.go b/client/request/mutation.go index 81fcc823c9..f78ed87ea0 100644 --- a/client/request/mutation.go +++ b/client/request/mutation.go @@ -41,6 +41,12 @@ type ObjectMutation struct { // // This is ignored for [DeleteObjects] mutations. Input map[string]any + + // Inputs is the array of json representations of the fieldName-value pairs of document + // properties to mutate. + // + // This is ignored for [DeleteObjects] mutations. + Inputs []map[string]any } // ToSelect returns a basic Select object, with the same Name, Alias, and Fields as diff --git a/internal/planner/create.go b/internal/planner/create.go index 21a36fcc24..8688b9fc6b 100644 --- a/internal/planner/create.go +++ b/internal/planner/create.go @@ -36,13 +36,12 @@ type createNode struct { collection client.Collection // input map of fields and values - input map[string]any - doc *client.Document + input []map[string]any + docs []*client.Document err error - returned bool - results planNode + results planNode execInfo createExecInfo } @@ -56,76 +55,65 @@ func (n *createNode) Kind() string { return "createNode" } func (n *createNode) Init() error { return nil } -func (n *createNode) Start() error { - doc, err := client.NewDocFromMap(n.input, n.collection.Definition()) - if err != nil { - n.err = err - return err +func docIDsToSpans(ids []string, desc client.CollectionDescription) core.Spans { + spans := make([]core.Span, len(ids)) + for i, id := range ids { + docID := base.MakeDataStoreKeyWithCollectionAndDocID(desc, id) + spans[i] = core.NewSpan(docID, docID.PrefixEnd()) } - n.doc = doc - return nil + return core.NewSpans(spans...) } -// Next only returns once. -func (n *createNode) Next() (bool, error) { - n.execInfo.iterations++ - - if n.err != nil { - return false, n.err - } - - if n.returned { - return false, nil - } - - if err := n.collection.Create( - n.p.ctx, - n.doc, - ); err != nil { - return false, err +func documentsToDocIDs(docs []*client.Document) []string { + docIDs := make([]string, len(docs)) + for i, doc := range docs { + docIDs[i] = doc.ID().String() } + return docIDs +} - currentValue := n.documentMapping.NewDoc() +func (n *createNode) Start() (err error) { + n.docs = make([]*client.Document, len(n.input)) + defer func() { n.err = err }() - currentValue.SetID(n.doc.ID().String()) - for i, value := range n.doc.Values() { - if len(n.documentMapping.IndexesByName[i.Name()]) > 0 { - n.documentMapping.SetFirstOfName(¤tValue, i.Name(), value.Value()) - } else if aliasName := i.Name() + request.RelatedObjectID; len(n.documentMapping.IndexesByName[aliasName]) > 0 { - n.documentMapping.SetFirstOfName(¤tValue, aliasName, value.Value()) - } else { - return false, client.NewErrFieldNotExist(i.Name()) + for i, input := range n.input { + doc, err := client.NewDocFromMap(input, n.collection.Definition()) + if err != nil { + return err } + n.docs[i] = doc } - n.returned = true - n.currentValue = currentValue - - desc := n.collection.Description() - docID := base.MakeDataStoreKeyWithCollectionAndDocID(desc, currentValue.GetID()) - n.results.Spans(core.NewSpans(core.NewSpan(docID, docID.PrefixEnd()))) - - err := n.results.Init() - if err != nil { - return false, err + if len(n.docs) == 1 { + err = n.collection.Create(n.p.ctx, n.docs[0]) + } else { + err = n.collection.CreateMany(n.p.ctx, n.docs) } - err = n.results.Start() if err != nil { - return false, err + return err } - // get the next result based on our point lookup - next, err := n.results.Next() + n.results.Spans(docIDsToSpans(documentsToDocIDs(n.docs), n.collection.Description())) + + err = n.results.Init() if err != nil { - return false, err + return err } - if !next { - return false, nil + + return n.results.Start() +} + +func (n *createNode) Next() (bool, error) { + n.execInfo.iterations++ + + if n.err != nil { + return false, n.err } + next, err := n.results.Next() n.currentValue = n.results.Value() - return true, nil + return next, err } func (n *createNode) Spans(spans core.Spans) { /* no-op */ } @@ -155,7 +143,7 @@ func (n *createNode) Explain(explainType request.ExplainType) (map[string]any, e } } -func (p *Planner) CreateDoc(parsed *mapper.Mutation) (planNode, error) { +func (p *Planner) CreateDocs(parsed *mapper.Mutation) (planNode, error) { results, err := p.Select(&parsed.Select) if err != nil { return nil, err @@ -164,10 +152,13 @@ func (p *Planner) CreateDoc(parsed *mapper.Mutation) (planNode, error) { // create a mutation createNode. create := &createNode{ p: p, - input: parsed.Input, + input: parsed.Inputs, results: results, docMapper: docMapper{parsed.DocumentMapping}, } + if parsed.Input != nil { + create.input = []map[string]any{parsed.Input} + } // get collection col, err := p.db.GetCollectionByName(p.ctx, parsed.Name) diff --git a/internal/planner/mapper/mapper.go b/internal/planner/mapper/mapper.go index be52066b54..f659f346a9 100644 --- a/internal/planner/mapper/mapper.go +++ b/internal/planner/mapper/mapper.go @@ -1168,6 +1168,7 @@ func ToMutation(ctx context.Context, store client.Store, mutationRequest *reques Select: *underlyingSelect, Type: MutationType(mutationRequest.Type), Input: mutationRequest.Input, + Inputs: mutationRequest.Inputs, }, nil } diff --git a/internal/planner/mapper/mutation.go b/internal/planner/mapper/mutation.go index a38444e01c..faf2f5209b 100644 --- a/internal/planner/mapper/mutation.go +++ b/internal/planner/mapper/mutation.go @@ -29,16 +29,7 @@ type Mutation struct { // Input is the map of fields and values used for the mutation. Input map[string]any -} - -func (m *Mutation) CloneTo(index int) Requestable { - return m.cloneTo(index) -} -func (m *Mutation) cloneTo(index int) *Mutation { - return &Mutation{ - Select: *m.Select.cloneTo(index), - Type: m.Type, - Input: m.Input, - } + // Inputs is the array of maps of fields and values used for the mutation. + Inputs []map[string]any } diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 0f513a045c..db7b0510ab 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -163,7 +163,7 @@ func (p *Planner) newPlan(stmt any) (planNode, error) { func (p *Planner) newObjectMutationPlan(stmt *mapper.Mutation) (planNode, error) { switch stmt.Type { case mapper.CreateObjects: - return p.CreateDoc(stmt) + return p.CreateDocs(stmt) case mapper.UpdateObjects: return p.UpdateDocs(stmt) diff --git a/internal/request/graphql/parser/mutation.go b/internal/request/graphql/parser/mutation.go index 92071b6e93..c1a3b31062 100644 --- a/internal/request/graphql/parser/mutation.go +++ b/internal/request/graphql/parser/mutation.go @@ -102,6 +102,18 @@ func parseMutation(schema gql.Schema, parent *gql.Object, field *ast.Field) (*re if prop == request.Input { // parse input raw := argument.Value.(*ast.ObjectValue) mut.Input = parseMutationInputObject(raw) + } else if prop == request.Inputs { + raw := argument.Value.(*ast.ListValue) + + mut.Inputs = make([]map[string]any, len(raw.Values)) + + for i, val := range raw.Values { + doc, ok := val.(*ast.ObjectValue) + if !ok { + return nil, client.NewErrUnexpectedType[*ast.ObjectValue]("doc array element", val) + } + mut.Inputs[i] = parseMutationInputObject(doc) + } } else if prop == request.FilterClause { // parse filter obj := argument.Value.(*ast.ObjectValue) filterType, ok := getArgumentType(fieldDef, request.FilterClause) diff --git a/internal/request/graphql/schema/generate.go b/internal/request/graphql/schema/generate.go index 5fd6b5ecf6..b3c88ddd24 100644 --- a/internal/request/graphql/schema/generate.go +++ b/internal/request/graphql/schema/generate.go @@ -27,6 +27,12 @@ import ( // create a fully DefraDB complaint GraphQL schema using a "code-first" dynamic // approach +const ( + filterInputNameSuffix = "FilterArg" + mutationInputNameSuffix = "MutationInputArg" + mutationInputsNameSuffix = "MutationInputsArg" +) + // Generator creates all the necessary typed schema definitions from an AST Document // and adds them to the Schema via the SchemaManager type Generator struct { @@ -171,7 +177,7 @@ func (g *Generator) generate(ctx context.Context, collections []client.Collectio for name, aggregateTarget := range def.Args { expandedField := &gql.InputObjectFieldConfig{ Description: aggregateFilterArgDescription, - Type: g.manager.schema.TypeMap()[name+"FilterArg"], + Type: g.manager.schema.TypeMap()[name+filterInputNameSuffix], } aggregateTarget.Type.(*gql.InputObject).AddFieldConfig(request.FilterClause, expandedField) } @@ -308,7 +314,7 @@ func (g *Generator) createExpandedFieldAggregate( target := aggregateTarget.Name() var filterTypeName string if target == request.GroupFieldName { - filterTypeName = obj.Name() + "FilterArg" + filterTypeName = obj.Name() + filterInputNameSuffix } else { if targeted := obj.Fields()[target]; targeted != nil { if list, isList := targeted.Type.(*gql.List); isList && gql.IsLeafType(list.OfType) { @@ -319,10 +325,10 @@ func (g *Generator) createExpandedFieldAggregate( // underlying name like this if it is a nullable type. filterTypeName = fmt.Sprintf("NotNull%sFilterArg", notNull.OfType.Name()) } else { - filterTypeName = genTypeName(list.OfType, "FilterArg") + filterTypeName = genTypeName(list.OfType, filterInputNameSuffix) } } else { - filterTypeName = targeted.Type.Name() + "FilterArg" + filterTypeName = targeted.Type.Name() + filterInputNameSuffix } } else { return NewErrAggregateTargetNotFound(obj.Name(), target) @@ -353,7 +359,7 @@ func (g *Generator) createExpandedFieldSingle( Type: t, Args: gql.FieldConfigArgument{ "filter": schemaTypes.NewArgConfig( - g.manager.schema.TypeMap()[typeName+"FilterArg"], + g.manager.schema.TypeMap()[typeName+filterInputNameSuffix], singleFieldFilterArgDescription, ), }, @@ -375,7 +381,7 @@ func (g *Generator) createExpandedFieldList( request.DocIDArgName: schemaTypes.NewArgConfig(gql.String, docIDArgDescription), request.DocIDsArgName: schemaTypes.NewArgConfig(gql.NewList(gql.NewNonNull(gql.String)), docIDsArgDescription), "filter": schemaTypes.NewArgConfig( - g.manager.schema.TypeMap()[typeName+"FilterArg"], + g.manager.schema.TypeMap()[typeName+filterInputNameSuffix], listFieldFilterArgDescription, ), "groupBy": schemaTypes.NewArgConfig( @@ -540,7 +546,7 @@ func (g *Generator) buildMutationInputTypes(collections []client.CollectionDefin // will be reassigned before the thunk is run // TODO remove when Go 1.22 collection := c - mutationInputName := collection.Description.Name.Value() + "MutationInputArg" + mutationInputName := collection.Description.Name.Value() + mutationInputNameSuffix // check if mutation input type exists if _, ok := g.manager.schema.TypeMap()[mutationInputName]; ok { @@ -587,6 +593,10 @@ func (g *Generator) buildMutationInputTypes(collections []client.CollectionDefin mutationObj := gql.NewInputObject(mutationObjConf) g.manager.schema.TypeMap()[mutationObj.Name()] = mutationObj + + mutationList := gql.NewList(mutationObj) + mutationListName := collection.Description.Name.Value() + mutationInputsNameSuffix + g.manager.schema.TypeMap()[mutationListName] = mutationList } return nil @@ -1027,24 +1037,32 @@ func (g *Generator) GenerateMutationInputForGQLType(obj *gql.Object) ([]*gql.Fie return nil, obj.Error() } - filterInputName := genTypeName(obj, "FilterArg") - mutationInputName := genTypeName(obj, "MutationInputArg") + filterInputName := genTypeName(obj, filterInputNameSuffix) + mutationInputName := genTypeName(obj, mutationInputNameSuffix) + mutationInputsName := genTypeName(obj, mutationInputsNameSuffix) filterInput, ok := g.manager.schema.TypeMap()[filterInputName].(*gql.InputObject) if !ok { return nil, NewErrTypeNotFound(filterInputName) } + mutationInput, ok := g.manager.schema.TypeMap()[mutationInputName] if !ok { return nil, NewErrTypeNotFound(mutationInputName) } + mutationInputs, ok := g.manager.schema.TypeMap()[mutationInputsName] + if !ok { + return nil, NewErrTypeNotFound(mutationInputsName) + } + create := &gql.Field{ Name: "create_" + obj.Name(), Description: createDocumentDescription, Type: obj, Args: gql.FieldConfigArgument{ - "input": schemaTypes.NewArgConfig(mutationInput, "Create field values"), + "input": schemaTypes.NewArgConfig(mutationInput, "Create field values"), + "inputs": schemaTypes.NewArgConfig(mutationInputs, "Create field values"), }, } @@ -1092,7 +1110,7 @@ func (g *Generator) genTypeFilterArgInput(obj *gql.Object) *gql.InputObject { var selfRefType *gql.InputObject inputCfg := gql.InputObjectConfig{ - Name: genTypeName(obj, "FilterArg"), + Name: genTypeName(obj, filterInputNameSuffix), } fieldThunk := (gql.InputObjectConfigFieldMapThunk)( func() (gql.InputObjectConfigFieldMap, error) { @@ -1136,7 +1154,7 @@ func (g *Generator) genTypeFilterArgInput(obj *gql.Object) *gql.InputObject { // We want the FilterArg for the object, not the list of objects. fieldType = l.OfType } - filterType, isFilterable := g.manager.schema.TypeMap()[genTypeName(fieldType, "FilterArg")] + filterType, isFilterable := g.manager.schema.TypeMap()[genTypeName(fieldType, filterInputNameSuffix)] if !isFilterable { filterType = &gql.InputObjectField{} } @@ -1169,7 +1187,7 @@ func (g *Generator) genLeafFilterArgInput(obj gql.Type) *gql.InputObject { } inputCfg := gql.InputObjectConfig{ - Name: fmt.Sprintf("%s%s", filterTypeName, "FilterArg"), + Name: fmt.Sprintf("%s%s", filterTypeName, filterInputNameSuffix), } var fieldThunk gql.InputObjectConfigFieldMapThunk = func() (gql.InputObjectConfigFieldMap, error) { diff --git a/tests/integration/explain/default/create_test.go b/tests/integration/explain/default/create_test.go index dc57671bdd..3fdccc8b44 100644 --- a/tests/integration/explain/default/create_test.go +++ b/tests/integration/explain/default/create_test.go @@ -52,11 +52,11 @@ func TestDefaultExplainMutationRequestWithCreate(t *testing.T) { TargetNodeName: "createNode", IncludeChildNodes: false, ExpectedAttributes: dataMap{ - "input": dataMap{ + "input": []dataMap{{ "age": int32(27), "name": "Shahzad Lone", "verified": true, - }, + }}, }, }, }, @@ -90,10 +90,10 @@ func TestDefaultExplainMutationRequestDoesNotCreateDocGivenDuplicate(t *testing. TargetNodeName: "createNode", IncludeChildNodes: false, ExpectedAttributes: dataMap{ - "input": dataMap{ + "input": []dataMap{{ "age": int32(27), "name": "Shahzad Lone", - }, + }}, }, }, }, diff --git a/tests/integration/explain/execute/create_test.go b/tests/integration/explain/execute/create_test.go index 58736edb90..54876b57f5 100644 --- a/tests/integration/explain/execute/create_test.go +++ b/tests/integration/explain/execute/create_test.go @@ -42,10 +42,10 @@ func TestExecuteExplainMutationRequestWithCreate(t *testing.T) { "iterations": uint64(2), "selectTopNode": dataMap{ "selectNode": dataMap{ - "iterations": uint64(1), + "iterations": uint64(2), "filterMatches": uint64(1), "scanNode": dataMap{ - "iterations": uint64(1), + "iterations": uint64(2), "docFetches": uint64(1), "fieldFetches": uint64(1), "indexFetches": uint64(0), diff --git a/tests/integration/mutation/create/simple_create_many_test.go b/tests/integration/mutation/create/simple_create_many_test.go index aa4dda3143..5f1e425549 100644 --- a/tests/integration/mutation/create/simple_create_many_test.go +++ b/tests/integration/mutation/create/simple_create_many_test.go @@ -18,7 +18,7 @@ import ( func TestMutationCreateMany(t *testing.T) { test := testUtils.TestCase{ - Description: "Simple create mutation", + Description: "Simple create many mutation", Actions: []any{ testUtils.SchemaUpdate{ Schema: ` diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 39e43d067e..7b5834e170 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1300,58 +1300,56 @@ func createDocViaGQL( collections []client.Collection, ) ([]*client.Document, error) { collection := collections[action.CollectionID] - var inputs []string + var input string + paramName := "input" + + var err error if action.DocMap != nil { - input, err := valueToGQL(action.DocMap) - require.NoError(s.t, err) - inputs = append(inputs, input) + input, err = valueToGQL(action.DocMap) } else if client.IsJSONArray([]byte(action.Doc)) { var docMaps []map[string]any - err := json.Unmarshal([]byte(action.Doc), &docMaps) + err = json.Unmarshal([]byte(action.Doc), &docMaps) require.NoError(s.t, err) - for _, docMap := range docMaps { - input, err := valueToGQL(docMap) - require.NoError(s.t, err) - inputs = append(inputs, input) - } + paramName = "inputs" + input, err = arrayToGQL(docMaps) } else { - input, err := jsonToGQL(action.Doc) - require.NoError(s.t, err) - inputs = append(inputs, input) + input, err = jsonToGQL(action.Doc) } + require.NoError(s.t, err) var docs []*client.Document - for _, input := range inputs { - request := fmt.Sprintf( - `mutation { - create_%s(input: %s) { + request := fmt.Sprintf( + `mutation { + create_%s(%s: %s) { _docID } }`, - collection.Name().Value(), - input, - ) + collection.Name().Value(), + paramName, + input, + ) - txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) + txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) - ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) + ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) - result := node.ExecRequest( - ctx, - request, - ) - if len(result.GQL.Errors) > 0 { - return nil, result.GQL.Errors[0] - } + result := node.ExecRequest( + ctx, + request, + ) + if len(result.GQL.Errors) > 0 { + return nil, result.GQL.Errors[0] + } - resultantDocs, ok := result.GQL.Data.([]map[string]any) - if !ok || len(resultantDocs) == 0 { - return nil, nil - } + resultantDocs, ok := result.GQL.Data.([]map[string]any) + if !ok || len(resultantDocs) == 0 { + return nil, nil + } - docIDString := resultantDocs[0]["_docID"].(string) + for _, docMap := range resultantDocs { + docIDString := docMap["_docID"].(string) docID, err := client.NewDocIDFromString(docIDString) require.NoError(s.t, err) From e90a959fd06f1e0305679d82571f51191d4144e4 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 28 Jun 2024 16:25:14 +0200 Subject: [PATCH 34/42] Add encrypt param to create gql mutation --- cli/request.go | 13 ------------- client/request/consts.go | 2 ++ client/request/mutation.go | 5 ++++- .../references/cli/defradb_client_query.md | 10 ---------- http/client_collection.go | 17 +++++++++-------- http/handler_store.go | 8 +------- internal/planner/create.go | 5 +++++ internal/planner/mapper/mapper.go | 9 +++++---- internal/planner/mapper/mutation.go | 3 +++ internal/request/graphql/parser/mutation.go | 2 ++ internal/request/graphql/schema/generate.go | 5 +++-- tests/integration/utils2.go | 11 ++++++++--- 12 files changed, 42 insertions(+), 48 deletions(-) diff --git a/cli/request.go b/cli/request.go index 795046ece9..b6ec8e05ce 100644 --- a/cli/request.go +++ b/cli/request.go @@ -26,21 +26,11 @@ const ( func MakeRequestCommand() *cobra.Command { var filePath string - var shouldEncrypt bool var cmd = &cobra.Command{ Use: "query [-i --identity] [request]", Short: "Send a DefraDB GraphQL query request", Long: `Send a DefraDB GraphQL query request to the database. -Options: - -i, --identity - Marks the document as private and set the identity as the owner. The access to the document - and permissions are controlled by ACP (Access Control Policy). - - -e, --encrypt - Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a - symmetric key for encryption using AES-GCM. - A query request can be sent as a single argument. Example command: defradb client query 'query { ... }' @@ -81,7 +71,6 @@ To learn more about the DefraDB GraphQL Query Language, refer to https://docs.so } store := mustGetContextStore(cmd) - setContextDocEncryption(cmd, shouldEncrypt, nil) result := store.ExecRequest(cmd.Context(), request) var errors []string @@ -100,8 +89,6 @@ To learn more about the DefraDB GraphQL Query Language, refer to https://docs.so }, } - cmd.PersistentFlags().BoolVarP(&shouldEncrypt, "encrypt", "e", false, - "Flag to enable encryption of the document") cmd.Flags().StringVarP(&filePath, "file", "f", "", "File containing the query request") return cmd } diff --git a/client/request/consts.go b/client/request/consts.go index 86bd887c92..cba609b788 100644 --- a/client/request/consts.go +++ b/client/request/consts.go @@ -26,6 +26,8 @@ const ( FieldIDName = "fieldId" ShowDeleted = "showDeleted" + EncryptArgName = "encrypt" + FilterClause = "filter" GroupByClause = "groupBy" LimitClause = "limit" diff --git a/client/request/mutation.go b/client/request/mutation.go index f78ed87ea0..70d0bed1d9 100644 --- a/client/request/mutation.go +++ b/client/request/mutation.go @@ -42,11 +42,14 @@ type ObjectMutation struct { // This is ignored for [DeleteObjects] mutations. Input map[string]any - // Inputs is the array of json representations of the fieldName-value pairs of document + // Inputs is the array of json representations of the fieldName-value pairs of document // properties to mutate. // // This is ignored for [DeleteObjects] mutations. Inputs []map[string]any + + // Encrypt is a boolean flag that indicates whether the input data should be encrypted. + Encrypt bool } // ToSelect returns a basic Select object, with the same Name, Alias, and Fields as diff --git a/docs/website/references/cli/defradb_client_query.md b/docs/website/references/cli/defradb_client_query.md index 7c9b8e72fc..ec868456b1 100644 --- a/docs/website/references/cli/defradb_client_query.md +++ b/docs/website/references/cli/defradb_client_query.md @@ -6,15 +6,6 @@ Send a DefraDB GraphQL query request Send a DefraDB GraphQL query request to the database. -Options: - -i, --identity - Marks the document as private and set the identity as the owner. The access to the document - and permissions are controlled by ACP (Access Control Policy). - - -e, --encrypt - Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a - symmetric key for encryption using AES-GCM. - A query request can be sent as a single argument. Example command: defradb client query 'query { ... }' @@ -39,7 +30,6 @@ defradb client query [-i --identity] [request] [flags] ### Options ``` - -e, --encrypt Flag to enable encryption of the document -f, --file string File containing the query request -h, --help help for query ``` diff --git a/http/client_collection.go b/http/client_collection.go index 4190d28963..b1761366a7 100644 --- a/http/client_collection.go +++ b/http/client_collection.go @@ -79,10 +79,7 @@ func (c *Collection) Create( return err } - encConf := encryption.GetContextConfig(ctx) - if encConf.HasValue() && encConf.Value().IsEncrypted { - req.Header.Set(DocEncryptionHeader, "1") - } + setDocEncryptionHeaderIfNeeded(ctx, req) _, err = c.http.request(req) if err != nil { @@ -120,10 +117,7 @@ func (c *Collection) CreateMany( return err } - encConf := encryption.GetContextConfig(ctx) - if encConf.HasValue() && encConf.Value().IsEncrypted { - req.Header.Set(DocEncryptionHeader, "1") - } + setDocEncryptionHeaderIfNeeded(ctx, req) _, err = c.http.request(req) if err != nil { @@ -136,6 +130,13 @@ func (c *Collection) CreateMany( return nil } +func setDocEncryptionHeaderIfNeeded(ctx context.Context, req *http.Request) { + encConf := encryption.GetContextConfig(ctx) + if encConf.HasValue() && encConf.Value().IsEncrypted { + req.Header.Set(DocEncryptionHeader, "1") + } +} + func (c *Collection) Update( ctx context.Context, doc *client.Document, diff --git a/http/handler_store.go b/http/handler_store.go index df2136db87..de534a8c1d 100644 --- a/http/handler_store.go +++ b/http/handler_store.go @@ -22,7 +22,6 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" - "github.com/sourcenetwork/defradb/internal/encryption" ) type storeHandler struct{} @@ -313,12 +312,7 @@ func (s *storeHandler) ExecRequest(rw http.ResponseWriter, req *http.Request) { return } - ctx := req.Context() - if req.Header.Get(DocEncryptionHeader) == "1" { - ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) - } - - result := store.ExecRequest(ctx, request.Query) + result := store.ExecRequest(req.Context(), request.Query) if result.Subscription == nil { responseJSON(rw, http.StatusOK, GraphQLResponse{result.GQL.Data, result.GQL.Errors}) diff --git a/internal/planner/create.go b/internal/planner/create.go index 8688b9fc6b..2b0a47332d 100644 --- a/internal/planner/create.go +++ b/internal/planner/create.go @@ -15,6 +15,7 @@ import ( "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/db/base" + "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/internal/planner/mapper" ) @@ -160,6 +161,10 @@ func (p *Planner) CreateDocs(parsed *mapper.Mutation) (planNode, error) { create.input = []map[string]any{parsed.Input} } + if parsed.Encrypt { + p.ctx = encryption.SetContextConfig(p.ctx, encryption.DocEncConfig{IsEncrypted: true}) + } + // get collection col, err := p.db.GetCollectionByName(p.ctx, parsed.Name) if err != nil { diff --git a/internal/planner/mapper/mapper.go b/internal/planner/mapper/mapper.go index f659f346a9..858ef0e0ae 100644 --- a/internal/planner/mapper/mapper.go +++ b/internal/planner/mapper/mapper.go @@ -1165,10 +1165,11 @@ func ToMutation(ctx context.Context, store client.Store, mutationRequest *reques } return &Mutation{ - Select: *underlyingSelect, - Type: MutationType(mutationRequest.Type), - Input: mutationRequest.Input, - Inputs: mutationRequest.Inputs, + Select: *underlyingSelect, + Type: MutationType(mutationRequest.Type), + Input: mutationRequest.Input, + Inputs: mutationRequest.Inputs, + Encrypt: mutationRequest.Encrypt, }, nil } diff --git a/internal/planner/mapper/mutation.go b/internal/planner/mapper/mutation.go index faf2f5209b..251d01298f 100644 --- a/internal/planner/mapper/mutation.go +++ b/internal/planner/mapper/mutation.go @@ -32,4 +32,7 @@ type Mutation struct { // Inputs is the array of maps of fields and values used for the mutation. Inputs []map[string]any + + // Encrypt is a flag to indicate if the input data should be encrypted. + Encrypt bool } diff --git a/internal/request/graphql/parser/mutation.go b/internal/request/graphql/parser/mutation.go index c1a3b31062..2ed4ebf539 100644 --- a/internal/request/graphql/parser/mutation.go +++ b/internal/request/graphql/parser/mutation.go @@ -140,6 +140,8 @@ func parseMutation(schema gql.Schema, parent *gql.Object, field *ast.Field) (*re ids[i] = id.Value } mut.DocIDs = immutable.Some(ids) + } else if prop == request.EncryptArgName { + mut.Encrypt = argument.Value.(*ast.BooleanValue).Value } } diff --git a/internal/request/graphql/schema/generate.go b/internal/request/graphql/schema/generate.go index b3c88ddd24..12fe1860dd 100644 --- a/internal/request/graphql/schema/generate.go +++ b/internal/request/graphql/schema/generate.go @@ -1061,8 +1061,9 @@ func (g *Generator) GenerateMutationInputForGQLType(obj *gql.Object) ([]*gql.Fie Description: createDocumentDescription, Type: obj, Args: gql.FieldConfigArgument{ - "input": schemaTypes.NewArgConfig(mutationInput, "Create field values"), - "inputs": schemaTypes.NewArgConfig(mutationInputs, "Create field values"), + "input": schemaTypes.NewArgConfig(mutationInput, "Create field values"), + "inputs": schemaTypes.NewArgConfig(mutationInputs, "Create field values"), + "encrypt": schemaTypes.NewArgConfig(gql.Boolean, "Encrypt input document(s)"), }, } diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index 7b5834e170..efaa8724ba 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1320,15 +1320,20 @@ func createDocViaGQL( var docs []*client.Document + params := paramName + ": " + input + + if action.IsEncrypted { + params = params + ", encrypt: true" + } + request := fmt.Sprintf( `mutation { - create_%s(%s: %s) { + create_%s(%s) { _docID } }`, collection.Name().Value(), - paramName, - input, + params, ) txn := getTransaction(s, node, immutable.None[int](), action.ExpectedError) From 487a48c78b6e217cb4f1d5641ae12632feef99b7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Fri, 28 Jun 2024 22:32:35 +0200 Subject: [PATCH 35/42] Polish --- internal/request/graphql/schema/descriptions.go | 5 +++++ internal/request/graphql/schema/generate.go | 10 ++-------- tests/integration/utils2.go | 8 ++++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/internal/request/graphql/schema/descriptions.go b/internal/request/graphql/schema/descriptions.go index 6f89932c0a..281002366a 100644 --- a/internal/request/graphql/schema/descriptions.go +++ b/internal/request/graphql/schema/descriptions.go @@ -155,5 +155,10 @@ Indicates as to whether or not this document has been deleted. ` versionFieldDescription string = ` Returns the head commit for this document. +` + + encryptArgDescription string = ` +Encrypt flag specified if the input document(s) needs to be encrypted. If set, DefraDB will generate a +symmetric key for encryption using AES-GCM. ` ) diff --git a/internal/request/graphql/schema/generate.go b/internal/request/graphql/schema/generate.go index 12fe1860dd..27c9500b16 100644 --- a/internal/request/graphql/schema/generate.go +++ b/internal/request/graphql/schema/generate.go @@ -1039,7 +1039,6 @@ func (g *Generator) GenerateMutationInputForGQLType(obj *gql.Object) ([]*gql.Fie filterInputName := genTypeName(obj, filterInputNameSuffix) mutationInputName := genTypeName(obj, mutationInputNameSuffix) - mutationInputsName := genTypeName(obj, mutationInputsNameSuffix) filterInput, ok := g.manager.schema.TypeMap()[filterInputName].(*gql.InputObject) if !ok { @@ -1051,19 +1050,14 @@ func (g *Generator) GenerateMutationInputForGQLType(obj *gql.Object) ([]*gql.Fie return nil, NewErrTypeNotFound(mutationInputName) } - mutationInputs, ok := g.manager.schema.TypeMap()[mutationInputsName] - if !ok { - return nil, NewErrTypeNotFound(mutationInputsName) - } - create := &gql.Field{ Name: "create_" + obj.Name(), Description: createDocumentDescription, Type: obj, Args: gql.FieldConfigArgument{ "input": schemaTypes.NewArgConfig(mutationInput, "Create field values"), - "inputs": schemaTypes.NewArgConfig(mutationInputs, "Create field values"), - "encrypt": schemaTypes.NewArgConfig(gql.Boolean, "Encrypt input document(s)"), + "inputs": schemaTypes.NewArgConfig(gql.NewList(mutationInput), "Create field values"), + "encrypt": schemaTypes.NewArgConfig(gql.Boolean, encryptArgDescription), }, } diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index efaa8724ba..f73075754d 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -1318,8 +1318,6 @@ func createDocViaGQL( } require.NoError(s.t, err) - var docs []*client.Document - params := paramName + ": " + input if action.IsEncrypted { @@ -1353,7 +1351,9 @@ func createDocViaGQL( return nil, nil } - for _, docMap := range resultantDocs { + docs := make([]*client.Document, len(resultantDocs)) + + for i, docMap := range resultantDocs { docIDString := docMap["_docID"].(string) docID, err := client.NewDocIDFromString(docIDString) require.NoError(s.t, err) @@ -1361,7 +1361,7 @@ func createDocViaGQL( doc, err := collection.Get(ctx, docID, false) require.NoError(s.t, err) - docs = append(docs, doc) + docs[i] = doc } return docs, nil From f252cdd68bbea2dd162ed8ee749281cf4fa4bca1 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 30 Jun 2024 23:47:10 +0200 Subject: [PATCH 36/42] Remove superfluous qql schema type records --- internal/request/graphql/schema/generate.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/request/graphql/schema/generate.go b/internal/request/graphql/schema/generate.go index 27c9500b16..e744f12de5 100644 --- a/internal/request/graphql/schema/generate.go +++ b/internal/request/graphql/schema/generate.go @@ -593,10 +593,6 @@ func (g *Generator) buildMutationInputTypes(collections []client.CollectionDefin mutationObj := gql.NewInputObject(mutationObjConf) g.manager.schema.TypeMap()[mutationObj.Name()] = mutationObj - - mutationList := gql.NewList(mutationObj) - mutationListName := collection.Description.Name.Value() + mutationInputsNameSuffix - g.manager.schema.TypeMap()[mutationListName] = mutationList } return nil From 1cbab95b727b086bdccace10b05f54991f108364 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 1 Jul 2024 09:24:58 +0200 Subject: [PATCH 37/42] Polish docs --- internal/db/merge.go | 1 + internal/encryption/nonce.go | 1 + internal/merkle/clock/clock.go | 1 + internal/merkle/crdt/field.go | 4 ++-- internal/request/graphql/schema/generate.go | 4 ++-- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/db/merge.go b/internal/db/merge.go index 4ccee263a3..e588cb60a4 100644 --- a/internal/db/merge.go +++ b/internal/db/merge.go @@ -244,6 +244,7 @@ func (mp *mergeProcessor) mergeComposites(ctx context.Context) error { } // processBlock merges the block and its children to the datastore and sets the head accordingly. +// If onlyHeads is true, it will skip merging and update only the heads. func (mp *mergeProcessor) processBlock( ctx context.Context, block *coreblock.Block, diff --git a/internal/encryption/nonce.go b/internal/encryption/nonce.go index 39cd72bf88..67a5467a4e 100644 --- a/internal/encryption/nonce.go +++ b/internal/encryption/nonce.go @@ -45,6 +45,7 @@ func generateTestNonce() ([]byte, error) { func init() { arg := os.Args[0] // If the binary is a test binary, use a deterministic nonce. + // TODO: We should try to find a better way to detect this https://github.com/sourcenetwork/defradb/issues/2801 if strings.HasSuffix(arg, ".test") || strings.Contains(arg, "/defradb/tests/") { generateNonceFunc = generateTestNonce generateEncryptionKeyFunc = generateTestEncryptionKey diff --git a/internal/merkle/clock/clock.go b/internal/merkle/clock/clock.go index fde0e8b4e0..1cb79ed756 100644 --- a/internal/merkle/clock/clock.go +++ b/internal/merkle/clock/clock.go @@ -166,6 +166,7 @@ func encryptBlock(ctx context.Context, block *coreblock.Block) (*coreblock.Block } // ProcessBlock merges the delta CRDT and updates the state accordingly. +// If onlyHeads is true, it will skip merging and update only the heads. func (mc *MerkleClock) ProcessBlock( ctx context.Context, block *coreblock.Block, diff --git a/internal/merkle/crdt/field.go b/internal/merkle/crdt/field.go index acc3203e67..6426165f49 100644 --- a/internal/merkle/crdt/field.go +++ b/internal/merkle/crdt/field.go @@ -13,8 +13,8 @@ package merklecrdt import "github.com/sourcenetwork/defradb/client" // DocField is a struct that holds the document ID and the field value. -// This is used to a link between the document and the field value. -// For example, to check if the field value need be encrypted depending on the document-level +// This is used to have a link between the document and the field value. +// For example, to check if the field value needs to be encrypted depending on the document-level // encryption is enabled or not. type DocField struct { DocID string diff --git a/internal/request/graphql/schema/generate.go b/internal/request/graphql/schema/generate.go index e744f12de5..82ff15d057 100644 --- a/internal/request/graphql/schema/generate.go +++ b/internal/request/graphql/schema/generate.go @@ -1051,8 +1051,8 @@ func (g *Generator) GenerateMutationInputForGQLType(obj *gql.Object) ([]*gql.Fie Description: createDocumentDescription, Type: obj, Args: gql.FieldConfigArgument{ - "input": schemaTypes.NewArgConfig(mutationInput, "Create field values"), - "inputs": schemaTypes.NewArgConfig(gql.NewList(mutationInput), "Create field values"), + "input": schemaTypes.NewArgConfig(mutationInput, "Create a "+obj.Name()+" document"), + "inputs": schemaTypes.NewArgConfig(gql.NewList(mutationInput), "Create "+obj.Name()+" documents"), "encrypt": schemaTypes.NewArgConfig(gql.Boolean, encryptArgDescription), }, } From 225eedf1aebf21c42059ca3cefc889e678374028 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 1 Jul 2024 09:28:24 +0200 Subject: [PATCH 38/42] Remove unnecessary cli flag --- tests/clients/cli/wrapper.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/clients/cli/wrapper.go b/tests/clients/cli/wrapper.go index 9e5f734454..a6f3feef4c 100644 --- a/tests/clients/cli/wrapper.go +++ b/tests/clients/cli/wrapper.go @@ -31,7 +31,6 @@ import ( "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/event" "github.com/sourcenetwork/defradb/http" - "github.com/sourcenetwork/defradb/internal/encryption" "github.com/sourcenetwork/defradb/node" ) @@ -400,11 +399,6 @@ func (w *Wrapper) ExecRequest( result := &client.RequestResult{} - encCond := encryption.GetContextConfig(ctx) - if encCond.HasValue() && encCond.Value().IsEncrypted { - args = append(args, "--encrypt") - } - stdOut, stdErr, err := w.cmd.executeStream(ctx, args) if err != nil { result.GQL.Errors = []error{err} From 592dfdb2b5400b6f00395b1450c850b6b11f3544 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 1 Jul 2024 11:42:05 +0200 Subject: [PATCH 39/42] Fix cli handling of CreateMany --- tests/clients/cli/wrapper_collection.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/clients/cli/wrapper_collection.go b/tests/clients/cli/wrapper_collection.go index c22b87786a..f26142c8e9 100644 --- a/tests/clients/cli/wrapper_collection.go +++ b/tests/clients/cli/wrapper_collection.go @@ -96,21 +96,22 @@ func (c *Collection) CreateMany( args := []string{"client", "collection", "create"} args = append(args, "--name", c.Description().Name.Value()) - docMapList := make([]map[string]any, len(docs)) + encConf := encryption.GetContextConfig(ctx) + if encConf.HasValue() && encConf.Value().IsEncrypted { + args = append(args, "--encrypt") + } + + docStrings := make([]string, len(docs)) for i, doc := range docs { - docMap, err := doc.ToMap() + docStr, err := doc.String() if err != nil { return err } - docMapList[i] = docMap + docStrings[i] = docStr } - documents, err := json.Marshal(docMapList) - if err != nil { - return err - } - args = append(args, string(documents)) + args = append(args, "["+strings.Join(docStrings, ",")+"]") - _, err = c.cmd.execute(ctx, args) + _, err := c.cmd.execute(ctx, args) if err != nil { return err } From 87574dc6f7b41c2fc8dde9a0af38b52c4d001cd9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 2 Jul 2024 08:57:10 +0200 Subject: [PATCH 40/42] Set query param instead of header --- http/client.go | 7 ++----- http/client_collection.go | 10 ++++++---- http/handler_collection.go | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/http/client.go b/http/client.go index 8105ccb6e9..6e5cc21276 100644 --- a/http/client.go +++ b/http/client.go @@ -29,7 +29,6 @@ import ( "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/event" - "github.com/sourcenetwork/defradb/internal/encryption" ) var _ client.DB = (*Client)(nil) @@ -350,6 +349,7 @@ func (c *Client) ExecRequest( result.GQL.Errors = []error{err} return result } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, methodURL.String(), bytes.NewBuffer(body)) if err != nil { result.GQL.Errors = []error{err} @@ -357,10 +357,7 @@ func (c *Client) ExecRequest( } err = c.http.setDefaultHeaders(req) - encConf := encryption.GetContextConfig(ctx) - if encConf.HasValue() && encConf.Value().IsEncrypted { - req.Header.Set(DocEncryptionHeader, "1") - } + setDocEncryptionFlagIfNeeded(ctx, req) if err != nil { result.GQL.Errors = []error{err} diff --git a/http/client_collection.go b/http/client_collection.go index b1761366a7..8df094f5fc 100644 --- a/http/client_collection.go +++ b/http/client_collection.go @@ -79,7 +79,7 @@ func (c *Collection) Create( return err } - setDocEncryptionHeaderIfNeeded(ctx, req) + setDocEncryptionFlagIfNeeded(ctx, req) _, err = c.http.request(req) if err != nil { @@ -117,7 +117,7 @@ func (c *Collection) CreateMany( return err } - setDocEncryptionHeaderIfNeeded(ctx, req) + setDocEncryptionFlagIfNeeded(ctx, req) _, err = c.http.request(req) if err != nil { @@ -130,10 +130,12 @@ func (c *Collection) CreateMany( return nil } -func setDocEncryptionHeaderIfNeeded(ctx context.Context, req *http.Request) { +func setDocEncryptionFlagIfNeeded(ctx context.Context, req *http.Request) { encConf := encryption.GetContextConfig(ctx) if encConf.HasValue() && encConf.Value().IsEncrypted { - req.Header.Set(DocEncryptionHeader, "1") + q := req.URL.Query() + q.Set(docEncryptParam, "true") + req.URL.RawQuery = q.Encode() } } diff --git a/http/handler_collection.go b/http/handler_collection.go index 2d5111671f..412f486602 100644 --- a/http/handler_collection.go +++ b/http/handler_collection.go @@ -24,7 +24,7 @@ import ( "github.com/sourcenetwork/defradb/internal/encryption" ) -const DocEncryptionHeader = "EncryptDoc" +const docEncryptParam = "encrypt" type collectionHandler struct{} @@ -47,7 +47,7 @@ func (s *collectionHandler) Create(rw http.ResponseWriter, req *http.Request) { } ctx := req.Context() - if req.Header.Get(DocEncryptionHeader) == "1" { + if req.URL.Query().Get(docEncryptParam) == "true" { ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true}) } From 9a99d3fd8a17ba83af5f2289003d0b83d8a46694 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 Jul 2024 08:40:12 +0200 Subject: [PATCH 41/42] Use regexp to determine if json bytes is array --- client/document.go | 7 ++++--- client/document_test.go | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/client/document.go b/client/document.go index 3105e5a2fb..f8d427c349 100644 --- a/client/document.go +++ b/client/document.go @@ -13,6 +13,7 @@ package client import ( "encoding/json" "errors" + "regexp" "strings" "sync" "time" @@ -117,11 +118,11 @@ func NewDocFromMap(data map[string]any, collectionDefinition CollectionDefinitio return doc, nil } +var jsonArrayPattern = regexp.MustCompile(`(?s)^\s*\[.*\]\s*$`) + // IsJSONArray returns true if the given byte array is a JSON Array. func IsJSONArray(obj []byte) bool { - var js []any - err := json.Unmarshal(obj, &js) - return err == nil + return jsonArrayPattern.Match(obj) } // NewFromJSON creates a new instance of a Document from a raw JSON object byte array. diff --git a/client/document_test.go b/client/document_test.go index 450a181af0..a11a6a67c8 100644 --- a/client/document_test.go +++ b/client/document_test.go @@ -252,9 +252,10 @@ func TestIsJSONArray(t *testing.T) { name: "Valid JSON Array with Whitespace", input: []byte(` [ - {"name": "John", "age": 21}, - {"name": "Islam", "age": 33} - ]`), + { "name": "John", "age": 21 }, + { "name": "Islam", "age": 33 } + ] + `), expected: true, }, } From 8bc56c7d7830b80f4eb581921807eb173bd10a1e Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 3 Jul 2024 23:55:38 +0200 Subject: [PATCH 42/42] Code review fixups --- datastore/store.go | 20 ++++------ internal/core/block/block.go | 1 + internal/planner/create.go | 44 ++++++++++----------- tests/integration/encryption/commit_test.go | 15 +++---- tests/integration/encryption/query_test.go | 7 +--- tests/integration/encryption/utils.go | 2 + tests/integration/utils2.go | 17 ++++---- 7 files changed, 47 insertions(+), 59 deletions(-) diff --git a/datastore/store.go b/datastore/store.go index b64d034e94..516bfe0b65 100644 --- a/datastore/store.go +++ b/datastore/store.go @@ -34,30 +34,26 @@ type Rootstore interface { type MultiStore interface { Rootstore() DSReaderWriter - // Datastore is a wrapped root DSReaderWriter - // under the /data namespace + // Datastore is a wrapped root DSReaderWriter under the /data namespace Datastore() DSReaderWriter - // Encstore is a wrapped root DSReaderWriter - // under the /enc namespace + // Encstore is a wrapped root DSReaderWriter under the /enc namespace + // This store is used for storing symmetric encryption keys for doc encryption. + // The store keys are comprised of docID + field name. Encstore() DSReaderWriter - // Headstore is a wrapped root DSReaderWriter - // under the /head namespace + // Headstore is a wrapped root DSReaderWriter under the /head namespace Headstore() DSReaderWriter - // Peerstore is a wrapped root DSReaderWriter - // as a ds.Batching, embedded into a DSBatching + // Peerstore is a wrapped root DSReaderWriter as a ds.Batching, embedded into a DSBatching // under the /peers namespace Peerstore() DSBatching - // Blockstore is a wrapped root DSReaderWriter - // as a Blockstore, embedded into a Blockstore + // Blockstore is a wrapped root DSReaderWriter as a Blockstore, embedded into a Blockstore // under the /blocks namespace Blockstore() Blockstore - // Headstore is a wrapped root DSReaderWriter - // under the /system namespace + // Headstore is a wrapped root DSReaderWriter under the /system namespace Systemstore() DSReaderWriter } diff --git a/internal/core/block/block.go b/internal/core/block/block.go index 3faecc1352..d2caa610f7 100644 --- a/internal/core/block/block.go +++ b/internal/core/block/block.go @@ -104,6 +104,7 @@ type Block struct { // Links are the links to other blocks in the DAG. Links []DAGLink // IsEncrypted is a flag that indicates if the block's delta is encrypted. + // It needs to be a pointer so that it can be translated from and to `optional Bool` in the IPLD schema. IsEncrypted *bool } diff --git a/internal/planner/create.go b/internal/planner/create.go index 2b0a47332d..b03f2c1765 100644 --- a/internal/planner/create.go +++ b/internal/planner/create.go @@ -40,7 +40,7 @@ type createNode struct { input []map[string]any docs []*client.Document - err error + didCreate bool results planNode @@ -73,9 +73,8 @@ func documentsToDocIDs(docs []*client.Document) []string { return docIDs } -func (n *createNode) Start() (err error) { +func (n *createNode) Start() error { n.docs = make([]*client.Document, len(n.input)) - defer func() { n.err = err }() for i, input := range n.input { doc, err := client.NewDocFromMap(input, n.collection.Definition()) @@ -85,31 +84,30 @@ func (n *createNode) Start() (err error) { n.docs[i] = doc } - if len(n.docs) == 1 { - err = n.collection.Create(n.p.ctx, n.docs[0]) - } else { - err = n.collection.CreateMany(n.p.ctx, n.docs) - } - - if err != nil { - return err - } - - n.results.Spans(docIDsToSpans(documentsToDocIDs(n.docs), n.collection.Description())) - - err = n.results.Init() - if err != nil { - return err - } - - return n.results.Start() + return nil } func (n *createNode) Next() (bool, error) { n.execInfo.iterations++ - if n.err != nil { - return false, n.err + if !n.didCreate { + err := n.collection.CreateMany(n.p.ctx, n.docs) + if err != nil { + return false, err + } + + n.results.Spans(docIDsToSpans(documentsToDocIDs(n.docs), n.collection.Description())) + + err = n.results.Init() + if err != nil { + return false, err + } + + err = n.results.Start() + if err != nil { + return false, err + } + n.didCreate = true } next, err := n.results.Next() diff --git a/tests/integration/encryption/commit_test.go b/tests/integration/encryption/commit_test.go index 9608349f6e..6a94621b3a 100644 --- a/tests/integration/encryption/commit_test.go +++ b/tests/integration/encryption/commit_test.go @@ -292,9 +292,6 @@ func TestDocEncryption_UponUpdateOnCounterCRDT_ShouldEncryptedCommitDelta(t *tes } func TestDocEncryption_UponEncryptionSeveralDocs_ShouldStoreAllCommitsDeltaEncrypted(t *testing.T) { - const johnDocID = "bae-c9fb0fa4-1195-589c-aa54-e68333fb90b3" - const islamDocID = "bae-d55bd956-1cc4-5d26-aa71-b98807ad49d6" - test := testUtils.TestCase{ Actions: []any{ updateUserCollectionSchema(), @@ -323,32 +320,32 @@ func TestDocEncryption_UponEncryptionSeveralDocs_ShouldStoreAllCommitsDeltaEncry { "cid": "bafyreih7ry7ef26xn3lm2rhxusf2rbgyvl535tltrt6ehpwtvdnhlmptiu", "delta": encrypt(testUtils.CBORValue(21)), - "docID": johnDocID, + "docID": testUtils.NewDocIndex(0, 0), }, { "cid": "bafyreifusejlwidaqswasct37eorazlfix6vyyn5af42pmjvktilzj5cty", "delta": encrypt(testUtils.CBORValue("John")), - "docID": johnDocID, + "docID": testUtils.NewDocIndex(0, 0), }, { "cid": "bafyreicvxlfxeqghmc3gy56rp5rzfejnbng4nu77x5e3wjinfydl6wvycq", "delta": nil, - "docID": johnDocID, + "docID": testUtils.NewDocIndex(0, 0), }, { "cid": "bafyreibe24bo67owxewoso3ekinera2bhusguij5qy2ahgyufaq3fbvaxa", "delta": encrypt(testUtils.CBORValue(33)), - "docID": islamDocID, + "docID": testUtils.NewDocIndex(0, 1), }, { "cid": "bafyreie2fddpidgc62fhd2fjrsucq3spgh2mgvto2xwolcdmdhb5pdeok4", "delta": encrypt(testUtils.CBORValue("Islam")), - "docID": islamDocID, + "docID": testUtils.NewDocIndex(0, 1), }, { "cid": "bafyreifulxdkf4m3wmmdxjg43l4mw7uuxl5il27eabklc22nptilrh64sa", "delta": nil, - "docID": islamDocID, + "docID": testUtils.NewDocIndex(0, 1), }, }, }, diff --git a/tests/integration/encryption/query_test.go b/tests/integration/encryption/query_test.go index eb671d08fa..32d9bd2c94 100644 --- a/tests/integration/encryption/query_test.go +++ b/tests/integration/encryption/query_test.go @@ -44,7 +44,7 @@ func TestDocEncryption_WithEncryption_ShouldFetchDecrypted(t *testing.T) { }`, Results: []map[string]any{ { - "_docID": "bae-0b2f15e5-bfe7-5cb7-8045-471318d7dbc3", + "_docID": testUtils.NewDocIndex(0, 0), "name": "John", "age": int64(21), }, @@ -57,12 +57,9 @@ func TestDocEncryption_WithEncryption_ShouldFetchDecrypted(t *testing.T) { } func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldFetchDecrypted(t *testing.T) { - const docID = "bae-ab8ae7d9-6473-5101-ba02-66b217948d7a" - const query = ` query { Users { - _docID name points } @@ -88,7 +85,6 @@ func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldFetchDecrypted(t *testi Request: query, Results: []map[string]any{ { - "_docID": docID, "name": "John", "points": 5, }, @@ -102,7 +98,6 @@ func TestDocEncryption_WithEncryptionOnCounterCRDT_ShouldFetchDecrypted(t *testi Request: query, Results: []map[string]any{ { - "_docID": docID, "name": "John", "points": 8, }, diff --git a/tests/integration/encryption/utils.go b/tests/integration/encryption/utils.go index 62cae5b19f..400a0d34c3 100644 --- a/tests/integration/encryption/utils.go +++ b/tests/integration/encryption/utils.go @@ -14,6 +14,8 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) +// we explicitly set LWW CRDT type because we want to test encryption with this specific CRDT type +// and we don't wat to rely on the default behavior const userCollectionGQLSchema = (` type Users { name: String diff --git a/tests/integration/utils2.go b/tests/integration/utils2.go index f73075754d..5012642f22 100644 --- a/tests/integration/utils2.go +++ b/tests/integration/utils2.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/require" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/crypto" "github.com/sourcenetwork/defradb/datastore" badgerds "github.com/sourcenetwork/defradb/datastore/badger/v4" @@ -107,6 +108,7 @@ func init() { if value, ok := os.LookupEnv(skipNetworkTestsEnvName); ok { skipNetworkTests, _ = strconv.ParseBool(value) } + mutationType = GQLRequestMutationType } // AssertPanic asserts that the code inside the specified PanicTestFunc panics. @@ -1302,7 +1304,7 @@ func createDocViaGQL( collection := collections[action.CollectionID] var input string - paramName := "input" + paramName := request.Input var err error if action.DocMap != nil { @@ -1311,7 +1313,7 @@ func createDocViaGQL( var docMaps []map[string]any err = json.Unmarshal([]byte(action.Doc), &docMaps) require.NoError(s.t, err) - paramName = "inputs" + paramName = request.Inputs input, err = arrayToGQL(docMaps) } else { input, err = jsonToGQL(action.Doc) @@ -1321,10 +1323,10 @@ func createDocViaGQL( params := paramName + ": " + input if action.IsEncrypted { - params = params + ", encrypt: true" + params = params + ", " + request.EncryptArgName + ": true" } - request := fmt.Sprintf( + req := fmt.Sprintf( `mutation { create_%s(%s) { _docID @@ -1338,10 +1340,7 @@ func createDocViaGQL( ctx := makeContextForDocCreate(db.SetContextTxn(s.ctx, txn), &action) - result := node.ExecRequest( - ctx, - request, - ) + result := node.ExecRequest(ctx, req) if len(result.GQL.Errors) > 0 { return nil, result.GQL.Errors[0] } @@ -1354,7 +1353,7 @@ func createDocViaGQL( docs := make([]*client.Document, len(resultantDocs)) for i, docMap := range resultantDocs { - docIDString := docMap["_docID"].(string) + docIDString := docMap[request.DocIDFieldName].(string) docID, err := client.NewDocIDFromString(docIDString) require.NoError(s.t, err)