feat: Doc encryption with symmetric key (#2731)

## Relevant issue(s)

Resolves #2711 #2808

## Description

This change introduces doc encryption. Upon creation of a document the
user can pass the encryption flag which will signal to db to create a
new symmetric key using AES-GCM and will store it locally.
With the encryption flag all doc fields (deltas) being stored in the DAG
encrypted. The datastore will still store data as plain text, as for
it's encryption we can use encryption-at-rest.
Decryption is not used at the moment, as it is relevant only p2p
environment and we don't have yet key exchange mechanism. All peers sync
encrypted data.

This PR also adds 2 new parameters to GraphQL `create_` mutation:
1. `inputs`: a list of documents to be created
2. `encrypt`: flag that indicates if document(s) need to be encrypted.

cli/collection_create.go

@@ -17,14 +17,25 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/internal/db"
 )
 
 func MakeCollectionCreateCommand() *cobra.Command {
 	var file string
+	var shouldEncrypt bool
 	var cmd = &cobra.Command{
-		Use:   "create [-i --identity] <document>",
+		Use:   "create [-i --identity] [-e --encrypt] <document>",
 		Short: "Create a new document.",
 		Long: `Create a new document.
+		
+Options:
+    -i, --identity 
+        Marks the document as private and set the identity as the owner. The access to the document
+		and permissions are controlled by ACP (Access Control Policy).
+
+	-e, --encrypt
+		Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a
+		symmetric key for encryption using AES-GCM.
 
 Example: create from string:
   defradb client collection create --name User '{ "name": "Bob" }'
@@ -69,6 +80,9 @@ Example: create from stdin:
 				return cmd.Usage()
 			}
 
+			txn, _ := db.TryGetContextTxn(cmd.Context())
+			setContextDocEncryption(cmd, shouldEncrypt, txn)
+
 			if client.IsJSONArray(docData) {
 				docs, err := client.NewDocsFromJSON(docData, col.Definition())
 				if err != nil {
@@ -84,6 +98,8 @@ Example: create from stdin:
 			return col.Create(cmd.Context(), doc)
 		},
 	}
+	cmd.PersistentFlags().BoolVarP(&shouldEncrypt, "encrypt", "e", false,
+		"Flag to enable encryption of the document")
 	cmd.Flags().StringVarP(&file, "file", "f", "", "File containing document(s)")
 	return cmd
 }

cli/utils.go

@@ -25,8 +25,10 @@ import (
 
 	acpIdentity "github.com/sourcenetwork/defradb/acp/identity"
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/datastore"
 	"github.com/sourcenetwork/defradb/http"
 	"github.com/sourcenetwork/defradb/internal/db"
+	"github.com/sourcenetwork/defradb/internal/encryption"
 	"github.com/sourcenetwork/defradb/keyring"
 )
 
@@ -160,6 +162,19 @@ func setContextIdentity(cmd *cobra.Command, privateKeyHex string) error {
 	return nil
 }
 
+// setContextDocEncryption sets doc encryption for the current command context.
+func setContextDocEncryption(cmd *cobra.Command, shouldEncrypt bool, txn datastore.Txn) {
+	if !shouldEncrypt {
+		return
+	}
+	ctx := cmd.Context()
+	if txn != nil {
+		ctx = encryption.ContextWithStore(ctx, txn)
+	}
+	ctx = encryption.SetContextConfig(ctx, encryption.DocEncConfig{IsEncrypted: true})
+	cmd.SetContext(ctx)
+}
+
 // setContextRootDir sets the rootdir for the current command context.
 func setContextRootDir(cmd *cobra.Command) error {
 	rootdir, err := cmd.Root().PersistentFlags().GetString("rootdir")

client/document.go

@@ -118,7 +118,7 @@ func NewDocFromMap(data map[string]any, collectionDefinition CollectionDefinitio
 	return doc, nil
 }
 
-var jsonArrayPattern = regexp.MustCompile(`^\s*\[.*\]\s*$`)
+var jsonArrayPattern = regexp.MustCompile(`(?s)^\s*\[.*\]\s*$`)
 
 // IsJSONArray returns true if the given byte array is a JSON Array.
 func IsJSONArray(obj []byte) bool {

client/document_test.go

@@ -206,3 +206,66 @@ func TestNewFromJSON_WithInvalidJSONFieldValueSimpleString_Error(t *testing.T) {
 	_, err := NewDocFromJSON(objWithJSONField, def)
 	require.ErrorContains(t, err, "invalid JSON payload. Payload: blah")
 }
+
+func TestIsJSONArray(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []byte
+		expected bool
+	}{
+		{
+			name:     "Valid JSON Array",
+			input:    []byte(`[{"name":"John","age":21},{"name":"Islam","age":33}]`),
+			expected: true,
+		},
+		{
+			name:     "Valid Empty JSON Array",
+			input:    []byte(`[]`),
+			expected: true,
+		},
+		{
+			name:     "Valid JSON Object",
+			input:    []byte(`{"name":"John","age":21}`),
+			expected: false,
+		},
+		{
+			name:     "Invalid JSON String",
+			input:    []byte(`{"name":"John","age":21`),
+			expected: false,
+		},
+		{
+			name:     "Non-JSON String",
+			input:    []byte(`Hello, World!`),
+			expected: false,
+		},
+		{
+			name:     "Array of Primitives",
+			input:    []byte(`[1, 2, 3, 4]`),
+			expected: true,
+		},
+		{
+			name:     "Nested JSON Array",
+			input:    []byte(`[[1, 2], [3, 4]]`),
+			expected: true,
+		},
+		{
+			name: "Valid JSON Array with Whitespace",
+			input: []byte(`
+				[
+					{	"name": "John", "age": 21	},
+					{	"name": "Islam", "age": 33	}
+				]
+			`),
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := IsJSONArray(tt.input)
+			if actual != tt.expected {
+				t.Errorf("IsJSONArray(%s) = %v; expected %v", tt.input, actual, tt.expected)
+			}
+		})
+	}
+}

client/request/consts.go

@@ -21,10 +21,13 @@ const (
 
 	Cid         = "cid"
 	Input       = "input"
+	Inputs      = "inputs"
 	FieldName   = "field"
 	FieldIDName = "fieldId"
 	ShowDeleted = "showDeleted"
 
+	EncryptArgName = "encrypt"
+
 	FilterClause  = "filter"
 	GroupByClause = "groupBy"
 	LimitClause   = "limit"

client/request/mutation.go

@@ -41,6 +41,15 @@ type ObjectMutation struct {
 	//
 	// This is ignored for [DeleteObjects] mutations.
 	Input map[string]any
+
+	// Inputs is the array of json representations of the fieldName-value pairs of document
+	// properties to mutate.
+	//
+	// This is ignored for [DeleteObjects] mutations.
+	Inputs []map[string]any
+
+	// Encrypt is a boolean flag that indicates whether the input data should be encrypted.
+	Encrypt bool
 }
 
 // ToSelect returns a basic Select object, with the same Name, Alias, and Fields as

datastore/multi.go

@@ -23,11 +23,13 @@ var (
 	headStoreKey   = rootStoreKey.ChildString("heads")
 	blockStoreKey  = rootStoreKey.ChildString("blocks")
 	peerStoreKey   = rootStoreKey.ChildString("ps")
+	encStoreKey    = rootStoreKey.ChildString("enc")
 )
 
 type multistore struct {
 	root   DSReaderWriter
 	data   DSReaderWriter
+	enc    DSReaderWriter
 	head   DSReaderWriter
 	peer   DSBatching
 	system DSReaderWriter
@@ -43,6 +45,7 @@ func MultiStoreFrom(rootstore ds.Datastore) MultiStore {
 	ms := &multistore{
 		root:   rootRW,
 		data:   prefix(rootRW, dataStoreKey),
+		enc:    prefix(rootRW, encStoreKey),
 		head:   prefix(rootRW, headStoreKey),
 		peer:   namespace.Wrap(rootstore, peerStoreKey),
 		system: prefix(rootRW, systemStoreKey),
@@ -57,6 +60,11 @@ func (ms multistore) Datastore() DSReaderWriter {
 	return ms.data
 }
 
+// Encstore implements MultiStore.
+func (ms multistore) Encstore() DSReaderWriter {
+	return ms.enc
+}
+
 // Headstore implements MultiStore.
 func (ms multistore) Headstore() DSReaderWriter {
 	return ms.head

datastore/store.go

@@ -34,26 +34,26 @@ type Rootstore interface {
 type MultiStore interface {
 	Rootstore() DSReaderWriter
 
-	// Datastore is a wrapped root DSReaderWriter
-	// under the /data namespace
+	// Datastore is a wrapped root DSReaderWriter under the /data namespace
 	Datastore() DSReaderWriter
 
-	// Headstore is a wrapped root DSReaderWriter
-	// under the /head namespace
+	// Encstore is a wrapped root DSReaderWriter under the /enc namespace
+	// This store is used for storing symmetric encryption keys for doc encryption.
+	// The store keys are comprised of docID + field name.
+	Encstore() DSReaderWriter
+
+	// Headstore is a wrapped root DSReaderWriter under the /head namespace
 	Headstore() DSReaderWriter
 
-	// Peerstore is a wrapped root DSReaderWriter
-	// as a ds.Batching, embedded into a DSBatching
+	// Peerstore is a wrapped root DSReaderWriter as a ds.Batching, embedded into a DSBatching
 	// under the /peers namespace
 	Peerstore() DSBatching
 
-	// Blockstore is a wrapped root DSReaderWriter
-	// as a Blockstore, embedded into a Blockstore
+	// Blockstore is a wrapped root DSReaderWriter as a Blockstore, embedded into a Blockstore
 	// under the /blocks namespace
 	Blockstore() Blockstore
 
-	// Headstore is a wrapped root DSReaderWriter
-	// under the /system namespace
+	// Headstore is a wrapped root DSReaderWriter under the /system namespace
 	Systemstore() DSReaderWriter
 }
 

docs/website/references/cli/defradb_client_collection_create.md

@@ -5,6 +5,15 @@ Create a new document.
 ### Synopsis
 
 Create a new document.
+
+Options:
+    -i, --identity 
+        Marks the document as private and set the identity as the owner. The access to the document
+		and permissions are controlled by ACP (Access Control Policy).
+
+	-e, --encrypt
+		Encrypt flag specified if the document needs to be encrypted. If set, DefraDB will generate a
+		symmetric key for encryption using AES-GCM.
 
 Example: create from string:
   defradb client collection create --name User '{ "name": "Bob" }'
@@ -24,12 +33,13 @@ Example: create from stdin:
 
 
 ```
-defradb client collection create [-i --identity] <document> [flags]
+defradb client collection create [-i --identity] [-e --encrypt] <document> [flags]
 ```
 
 ### Options
 
 ```
+  -e, --encrypt       Flag to enable encryption of the document
   -f, --file string   File containing document(s)
   -h, --help          help for create
 ```

http/client.go

@@ -349,12 +349,16 @@ func (c *Client) ExecRequest(
 		result.GQL.Errors = []error{err}
 		return result
 	}
+
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, methodURL.String(), bytes.NewBuffer(body))
 	if err != nil {
 		result.GQL.Errors = []error{err}
 		return result
 	}
 	err = c.http.setDefaultHeaders(req)
+
+	setDocEncryptionFlagIfNeeded(ctx, req)
+
 	if err != nil {
 		result.GQL.Errors = []error{err}
 		return result

http/client_collection.go

@@ -24,6 +24,7 @@ import (
 	sse "github.com/vito/go-sse/sse"
 
 	"github.com/sourcenetwork/defradb/client"
+	"github.com/sourcenetwork/defradb/internal/encryption"
 )
 
 var _ client.Collection = (*Collection)(nil)
@@ -78,6 +79,8 @@ func (c *Collection) Create(
 		return err
 	}
 
+	setDocEncryptionFlagIfNeeded(ctx, req)
+
 	_, err = c.http.request(req)
 	if err != nil {
 		return err
@@ -114,6 +117,8 @@ func (c *Collection) CreateMany(
 		return err
 	}
 
+	setDocEncryptionFlagIfNeeded(ctx, req)
+
 	_, err = c.http.request(req)
 	if err != nil {
 		return err
@@ -125,6 +130,15 @@ func (c *Collection) CreateMany(
 	return nil
 }
 
+func setDocEncryptionFlagIfNeeded(ctx context.Context, req *http.Request) {
+	encConf := encryption.GetContextConfig(ctx)
+	if encConf.HasValue() && encConf.Value().IsEncrypted {
+		q := req.URL.Query()
+		q.Set(docEncryptParam, "true")
+		req.URL.RawQuery = q.Encode()
+	}
+}
+
 func (c *Collection) Update(
 	ctx context.Context,
 	doc *client.Document,