From a7276dca04b3b107c7b14a68b3c2cf08e8e7cf2d Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Wed, 14 Aug 2024 15:46:52 +1000 Subject: [PATCH] refactor: typesafe encrypted DB columns --- .go-arch-lint.yml | 6 +- backend/controller/cronjobs/sql/models.go | 7 ++- backend/controller/dal/async_calls.go | 9 +-- backend/controller/dal/dal.go | 18 +++--- backend/controller/dal/encryption.go | 28 ++++----- backend/controller/dal/events.go | 9 ++- backend/controller/dal/fsm.go | 9 +-- backend/controller/dal/pubsub.go | 3 +- backend/controller/sql/models.go | 7 ++- backend/controller/sql/querier.go | 3 +- backend/controller/sql/queries.sql.go | 17 +++--- ...40815003340_typesafe_encrypted_columns.sql | 15 +++++ internal/configuration/sql/models.go | 7 ++- internal/encryption/database.go | 58 +++++++++++++++++++ internal/encryption/encryption.go | 48 ++++++++------- internal/encryption/encryption_test.go | 15 +++-- internal/encryption/integration_test.go | 8 ++- sqlc.yaml | 10 ++++ 18 files changed, 193 insertions(+), 84 deletions(-) create mode 100644 backend/controller/sql/schema/20240815003340_typesafe_encrypted_columns.sql create mode 100644 internal/encryption/database.go diff --git a/.go-arch-lint.yml b/.go-arch-lint.yml index 1016fc3b0e..16b0fc4452 100644 --- a/.go-arch-lint.yml +++ b/.go-arch-lint.yml @@ -100,4 +100,8 @@ deps: - common - internal - sql - - leases \ No newline at end of file + - leases + encoding: + mayDependOn: + - common + - internal \ No newline at end of file diff --git a/backend/controller/cronjobs/sql/models.go b/backend/controller/cronjobs/sql/models.go index 0c59dbef5c..ab3a9693ca 100644 --- a/backend/controller/cronjobs/sql/models.go +++ b/backend/controller/cronjobs/sql/models.go @@ -13,6 +13,7 @@ import ( "github.com/TBD54566975/ftl/backend/controller/leases" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" "github.com/alecthomas/types/optional" "github.com/google/uuid" @@ -378,8 +379,8 @@ type AsyncCall struct { State AsyncCallState Origin string ScheduledAt time.Time - Request []byte - Response []byte + Request encryption.EncryptedAsyncColumn + Response encryption.OptionalEncryptedAsyncColumn Error optional.Option[string] RemainingAttempts int32 Backoff sqltypes.Duration @@ -525,7 +526,7 @@ type Timeline struct { CustomKey2 optional.Option[string] CustomKey3 optional.Option[string] CustomKey4 optional.Option[string] - Payload []byte + Payload encryption.EncryptedTimelineColumn ParentRequestID optional.Option[string] } diff --git a/backend/controller/dal/async_calls.go b/backend/controller/dal/async_calls.go index e4ab2f4688..6358c90283 100644 --- a/backend/controller/dal/async_calls.go +++ b/backend/controller/dal/async_calls.go @@ -115,7 +115,7 @@ func (d *DAL) AcquireAsyncCall(ctx context.Context) (call *AsyncCall, err error) return nil, fmt.Errorf("failed to parse origin key %q: %w", row.Origin, err) } - decryptedRequest, err := d.decrypt(encryption.AsyncSubKey, row.Request) + decryptedRequest, err := d.decrypt(&row.Request) if err != nil { return nil, fmt.Errorf("failed to decrypt async call request: %w", err) } @@ -158,11 +158,12 @@ func (d *DAL) CompleteAsyncCall(ctx context.Context, didScheduleAnotherCall = false switch result := result.(type) { case either.Left[[]byte, string]: // Successful response. - encryptedResult, err := d.encrypt(encryption.AsyncSubKey, result.Get()) + var encryptedResult encryption.EncryptedAsyncColumn + err := d.encrypt(result.Get(), &encryptedResult) if err != nil { return false, fmt.Errorf("failed to encrypt async call result: %w", err) } - _, err = tx.db.SucceedAsyncCall(ctx, encryptedResult, call.ID) + _, err = tx.db.SucceedAsyncCall(ctx, optional.Some(encryptedResult), call.ID) if err != nil { return false, dalerrs.TranslatePGError(err) //nolint:wrapcheck } @@ -227,7 +228,7 @@ func (d *DAL) LoadAsyncCall(ctx context.Context, id int64) (*AsyncCall, error) { if err != nil { return nil, fmt.Errorf("failed to parse origin key %q: %w", row.Origin, err) } - request, err := d.decrypt(encryption.AsyncSubKey, row.Request) + request, err := d.decrypt(&row.Request) if err != nil { return nil, fmt.Errorf("failed to decrypt async call request: %w", err) } diff --git a/backend/controller/dal/dal.go b/backend/controller/dal/dal.go index 7172fe71dc..0b47b57c4a 100644 --- a/backend/controller/dal/dal.go +++ b/backend/controller/dal/dal.go @@ -709,10 +709,11 @@ func (d *DAL) SetDeploymentReplicas(ctx context.Context, key model.DeploymentKey return dalerrs.TranslatePGError(err) } } - payload, err := d.encryptJSON(encryption.TimelineSubKey, map[string]interface{}{ + var payload encryption.EncryptedTimelineColumn + err = d.encryptJSON(map[string]interface{}{ "prev_min_replicas": deployment.MinReplicas, "min_replicas": minReplicas, - }) + }, &payload) if err != nil { return fmt.Errorf("failed to encrypt payload for InsertDeploymentUpdatedEvent: %w", err) } @@ -782,10 +783,11 @@ func (d *DAL) ReplaceDeployment(ctx context.Context, newDeploymentKey model.Depl } } - payload, err := d.encryptJSON(encryption.TimelineSubKey, map[string]any{ + var payload encryption.EncryptedTimelineColumn + err = d.encryptJSON(map[string]any{ "min_replicas": int32(minReplicas), "replaced": replacedDeploymentKey, - }) + }, &payload) if err != nil { return fmt.Errorf("replace deployment failed to encrypt payload: %w", err) } @@ -1057,7 +1059,8 @@ func (d *DAL) InsertLogEvent(ctx context.Context, log *LogEvent) error { "error": log.Error, "stack": log.Stack, } - encryptedPayload, err := d.encryptJSON(encryption.TimelineSubKey, payload) + var encryptedPayload encryption.EncryptedTimelineColumn + err := d.encryptJSON(payload, &encryptedPayload) if err != nil { return fmt.Errorf("failed to encrypt log payload: %w", err) } @@ -1137,13 +1140,14 @@ func (d *DAL) InsertCallEvent(ctx context.Context, call *CallEvent) error { if pr, ok := call.ParentRequestKey.Get(); ok { parentRequestKey = optional.Some(pr.String()) } - payload, err := d.encryptJSON(encryption.TimelineSubKey, map[string]any{ + var payload encryption.EncryptedTimelineColumn + err := d.encryptJSON(map[string]any{ "duration_ms": call.Duration.Milliseconds(), "request": call.Request, "response": call.Response, "error": call.Error, "stack": call.Stack, - }) + }, &payload) if err != nil { return fmt.Errorf("failed to encrypt call payload: %w", err) } diff --git a/backend/controller/dal/encryption.go b/backend/controller/dal/encryption.go index 3bedc9476a..af0fd59f89 100644 --- a/backend/controller/dal/encryption.go +++ b/backend/controller/dal/encryption.go @@ -10,45 +10,45 @@ import ( "github.com/TBD54566975/ftl/internal/log" ) -func (d *DAL) encrypt(subKey encryption.SubKey, cleartext []byte) ([]byte, error) { +func (d *DAL) encrypt(cleartext []byte, dest encryption.Encrypted) error { if d.encryptor == nil { - return nil, fmt.Errorf("encryptor not set") + return fmt.Errorf("encryptor not set") } - v, err := d.encryptor.Encrypt(subKey, cleartext) + err := d.encryptor.Encrypt(cleartext, dest) if err != nil { - return nil, fmt.Errorf("failed to encrypt binary with subkey %s: %w", subKey, err) + return fmt.Errorf("failed to encrypt binary with subkey %s: %w", dest.SubKey(), err) } - return v, nil + return nil } -func (d *DAL) decrypt(subKey encryption.SubKey, encrypted []byte) ([]byte, error) { +func (d *DAL) decrypt(encrypted encryption.Encrypted) ([]byte, error) { if d.encryptor == nil { return nil, fmt.Errorf("encryptor not set") } - v, err := d.encryptor.Decrypt(subKey, encrypted) + v, err := d.encryptor.Decrypt(encrypted) if err != nil { - return nil, fmt.Errorf("failed to decrypt binary with subkey %s: %w", subKey, err) + return nil, fmt.Errorf("failed to decrypt binary with subkey %s: %w", encrypted.SubKey(), err) } return v, nil } -func (d *DAL) encryptJSON(subKey encryption.SubKey, v any) ([]byte, error) { +func (d *DAL) encryptJSON(v any, dest encryption.Encrypted) error { serialized, err := json.Marshal(v) if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %w", err) + return fmt.Errorf("failed to marshal JSON: %w", err) } - return d.encrypt(subKey, serialized) + return d.encrypt(serialized, dest) } -func (d *DAL) decryptJSON(subKey encryption.SubKey, encrypted []byte, v any) error { //nolint:unparam - decrypted, err := d.decrypt(subKey, encrypted) +func (d *DAL) decryptJSON(encrypted encryption.Encrypted, v any) error { //nolint:unparam + decrypted, err := d.decrypt(encrypted) if err != nil { - return fmt.Errorf("failed to decrypt json with subkey %s: %w", subKey, err) + return fmt.Errorf("failed to decrypt json with subkey %s: %w", encrypted.SubKey(), err) } if err = json.Unmarshal(decrypted, v); err != nil { diff --git a/backend/controller/dal/events.go b/backend/controller/dal/events.go index 1ae4282b11..85d0c0bb8c 100644 --- a/backend/controller/dal/events.go +++ b/backend/controller/dal/events.go @@ -13,7 +13,6 @@ import ( "github.com/TBD54566975/ftl/backend/controller/sql" dalerrs "github.com/TBD54566975/ftl/backend/dal" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" ) @@ -349,7 +348,7 @@ func (d *DAL) transformRowsToTimelineEvents(deploymentKeys map[int64]model.Deplo switch row.Type { case sql.EventTypeLog: var jsonPayload eventLogJSON - if err := d.decryptJSON(encryption.TimelineSubKey, row.Payload, &jsonPayload); err != nil { + if err := d.decryptJSON(&row.Payload, &jsonPayload); err != nil { return nil, fmt.Errorf("failed to decrypt log event: %w", err) } @@ -371,7 +370,7 @@ func (d *DAL) transformRowsToTimelineEvents(deploymentKeys map[int64]model.Deplo case sql.EventTypeCall: var jsonPayload eventCallJSON - if err := d.decryptJSON(encryption.TimelineSubKey, row.Payload, &jsonPayload); err != nil { + if err := d.decryptJSON(&row.Payload, &jsonPayload); err != nil { return nil, fmt.Errorf("failed to decrypt call event: %w", err) } var sourceVerb optional.Option[schema.Ref] @@ -396,7 +395,7 @@ func (d *DAL) transformRowsToTimelineEvents(deploymentKeys map[int64]model.Deplo case sql.EventTypeDeploymentCreated: var jsonPayload eventDeploymentCreatedJSON - if err := d.decryptJSON(encryption.TimelineSubKey, row.Payload, &jsonPayload); err != nil { + if err := d.decryptJSON(&row.Payload, &jsonPayload); err != nil { return nil, fmt.Errorf("failed to decrypt call event: %w", err) } out = append(out, &DeploymentCreatedEvent{ @@ -411,7 +410,7 @@ func (d *DAL) transformRowsToTimelineEvents(deploymentKeys map[int64]model.Deplo case sql.EventTypeDeploymentUpdated: var jsonPayload eventDeploymentUpdatedJSON - if err := d.decryptJSON(encryption.TimelineSubKey, row.Payload, &jsonPayload); err != nil { + if err := d.decryptJSON(&row.Payload, &jsonPayload); err != nil { return nil, fmt.Errorf("failed to decrypt call event: %w", err) } out = append(out, &DeploymentUpdatedEvent{ diff --git a/backend/controller/dal/fsm.go b/backend/controller/dal/fsm.go index b9b871b26c..af8e9bf008 100644 --- a/backend/controller/dal/fsm.go +++ b/backend/controller/dal/fsm.go @@ -32,11 +32,11 @@ import ( // // Note: no validation of the FSM is performed. func (d *DAL) StartFSMTransition(ctx context.Context, fsm schema.RefKey, instanceKey string, destinationState schema.RefKey, request []byte, encrypted bool, retryParams schema.RetryParams) (err error) { - var encryptedRequest []byte + var encryptedRequest encryption.EncryptedAsyncColumn if encrypted { - encryptedRequest = request + encryptedRequest = encryption.EncryptedAsyncColumn(request) } else { - encryptedRequest, err = d.encrypt(encryption.AsyncSubKey, request) + err = d.encrypt(request, &encryptedRequest) if err != nil { return fmt.Errorf("failed to encrypt FSM request: %w", err) } @@ -146,7 +146,8 @@ func (d *DAL) PopNextFSMEvent(ctx context.Context, fsm schema.RefKey, instanceKe } func (d *DAL) SetNextFSMEvent(ctx context.Context, fsm schema.RefKey, instanceKey string, nextState schema.RefKey, request json.RawMessage, requestType schema.Type) error { - encryptedRequest, err := d.encryptJSON(encryption.AsyncSubKey, request) + var encryptedRequest encryption.EncryptedAsyncColumn + err := d.encryptJSON(request, &encryptedRequest) if err != nil { return fmt.Errorf("failed to encrypt FSM request: %w", err) } diff --git a/backend/controller/dal/pubsub.go b/backend/controller/dal/pubsub.go index aed6077479..795201353b 100644 --- a/backend/controller/dal/pubsub.go +++ b/backend/controller/dal/pubsub.go @@ -21,7 +21,8 @@ import ( ) func (d *DAL) PublishEventForTopic(ctx context.Context, module, topic, caller string, payload []byte) error { - encryptedPayload, err := d.encrypt(encryption.AsyncSubKey, payload) + var encryptedPayload encryption.EncryptedAsyncColumn + err := d.encrypt(payload, &encryptedPayload) if err != nil { return fmt.Errorf("failed to encrypt payload: %w", err) } diff --git a/backend/controller/sql/models.go b/backend/controller/sql/models.go index 0c59dbef5c..ab3a9693ca 100644 --- a/backend/controller/sql/models.go +++ b/backend/controller/sql/models.go @@ -13,6 +13,7 @@ import ( "github.com/TBD54566975/ftl/backend/controller/leases" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" "github.com/alecthomas/types/optional" "github.com/google/uuid" @@ -378,8 +379,8 @@ type AsyncCall struct { State AsyncCallState Origin string ScheduledAt time.Time - Request []byte - Response []byte + Request encryption.EncryptedAsyncColumn + Response encryption.OptionalEncryptedAsyncColumn Error optional.Option[string] RemainingAttempts int32 Backoff sqltypes.Duration @@ -525,7 +526,7 @@ type Timeline struct { CustomKey2 optional.Option[string] CustomKey3 optional.Option[string] CustomKey4 optional.Option[string] - Payload []byte + Payload encryption.EncryptedTimelineColumn ParentRequestID optional.Option[string] } diff --git a/backend/controller/sql/querier.go b/backend/controller/sql/querier.go index a48ae4034b..74b4582179 100644 --- a/backend/controller/sql/querier.go +++ b/backend/controller/sql/querier.go @@ -12,6 +12,7 @@ import ( "github.com/TBD54566975/ftl/backend/controller/leases" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" "github.com/alecthomas/types/optional" "github.com/google/uuid" @@ -115,7 +116,7 @@ type Querier interface { // // "key" is the unique identifier for the FSM execution. StartFSMTransition(ctx context.Context, arg StartFSMTransitionParams) (FsmInstance, error) - SucceedAsyncCall(ctx context.Context, response []byte, iD int64) (bool, error) + SucceedAsyncCall(ctx context.Context, response encryption.OptionalEncryptedAsyncColumn, iD int64) (bool, error) SucceedFSMInstance(ctx context.Context, fsm schema.RefKey, key string) (bool, error) UpsertController(ctx context.Context, key model.ControllerKey, endpoint string) (int64, error) UpsertModule(ctx context.Context, language string, name string) (int64, error) diff --git a/backend/controller/sql/queries.sql.go b/backend/controller/sql/queries.sql.go index a7525ab4d1..d85ce0a2f8 100644 --- a/backend/controller/sql/queries.sql.go +++ b/backend/controller/sql/queries.sql.go @@ -13,6 +13,7 @@ import ( "github.com/TBD54566975/ftl/backend/controller/leases" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" "github.com/alecthomas/types/optional" "github.com/google/uuid" @@ -67,7 +68,7 @@ type AcquireAsyncCallRow struct { Origin string Verb schema.RefKey CatchVerb optional.Option[schema.RefKey] - Request []byte + Request encryption.EncryptedAsyncColumn ScheduledAt time.Time RemainingAttempts int32 Error optional.Option[string] @@ -218,7 +219,7 @@ RETURNING id type CreateAsyncCallParams struct { Verb schema.RefKey Origin string - Request []byte + Request encryption.EncryptedAsyncColumn RemainingAttempts int32 Backoff sqltypes.Duration MaxBackoff sqltypes.Duration @@ -2045,7 +2046,7 @@ type InsertTimelineCallEventParams struct { SourceVerb optional.Option[string] DestModule string DestVerb string - Payload []byte + Payload encryption.EncryptedTimelineColumn } func (q *Queries) InsertTimelineCallEvent(ctx context.Context, arg InsertTimelineCallEventParams) error { @@ -2088,7 +2089,7 @@ type InsertTimelineDeploymentCreatedEventParams struct { DeploymentKey model.DeploymentKey Language string ModuleName string - Payload []byte + Payload encryption.EncryptedTimelineColumn } func (q *Queries) InsertTimelineDeploymentCreatedEvent(ctx context.Context, arg InsertTimelineDeploymentCreatedEventParams) error { @@ -2126,7 +2127,7 @@ type InsertTimelineDeploymentUpdatedEventParams struct { DeploymentKey model.DeploymentKey Language string ModuleName string - Payload []byte + Payload encryption.EncryptedTimelineColumn } func (q *Queries) InsertTimelineDeploymentUpdatedEvent(ctx context.Context, arg InsertTimelineDeploymentUpdatedEventParams) error { @@ -2156,7 +2157,7 @@ type InsertTimelineEventParams struct { CustomKey2 optional.Option[string] CustomKey3 optional.Option[string] CustomKey4 optional.Option[string] - Payload []byte + Payload encryption.EncryptedTimelineColumn } func (q *Queries) InsertTimelineEvent(ctx context.Context, arg InsertTimelineEventParams) error { @@ -2203,7 +2204,7 @@ type InsertTimelineLogEventParams struct { RequestKey optional.Option[string] TimeStamp time.Time Level int32 - Payload []byte + Payload encryption.EncryptedTimelineColumn } func (q *Queries) InsertTimelineLogEvent(ctx context.Context, arg InsertTimelineLogEventParams) error { @@ -2637,7 +2638,7 @@ WHERE id = $2 RETURNING true ` -func (q *Queries) SucceedAsyncCall(ctx context.Context, response []byte, iD int64) (bool, error) { +func (q *Queries) SucceedAsyncCall(ctx context.Context, response encryption.OptionalEncryptedAsyncColumn, iD int64) (bool, error) { row := q.db.QueryRowContext(ctx, succeedAsyncCall, response, iD) var column_1 bool err := row.Scan(&column_1) diff --git a/backend/controller/sql/schema/20240815003340_typesafe_encrypted_columns.sql b/backend/controller/sql/schema/20240815003340_typesafe_encrypted_columns.sql new file mode 100644 index 0000000000..05e9f5d724 --- /dev/null +++ b/backend/controller/sql/schema/20240815003340_typesafe_encrypted_columns.sql @@ -0,0 +1,15 @@ +-- migrate:up + +CREATE DOMAIN encrypted_timeline AS BYTEA; + +ALTER TABLE timeline + ALTER COLUMN payload TYPE encrypted_timeline; + +CREATE DOMAIN encrypted_async AS BYTEA; + +ALTER TABLE async_calls + ALTER COLUMN request TYPE encrypted_async, + ALTER COLUMN response TYPE encrypted_async; + +-- migrate:down + diff --git a/internal/configuration/sql/models.go b/internal/configuration/sql/models.go index 0c59dbef5c..ab3a9693ca 100644 --- a/internal/configuration/sql/models.go +++ b/internal/configuration/sql/models.go @@ -13,6 +13,7 @@ import ( "github.com/TBD54566975/ftl/backend/controller/leases" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/internal/encryption" "github.com/TBD54566975/ftl/internal/model" "github.com/alecthomas/types/optional" "github.com/google/uuid" @@ -378,8 +379,8 @@ type AsyncCall struct { State AsyncCallState Origin string ScheduledAt time.Time - Request []byte - Response []byte + Request encryption.EncryptedAsyncColumn + Response encryption.OptionalEncryptedAsyncColumn Error optional.Option[string] RemainingAttempts int32 Backoff sqltypes.Duration @@ -525,7 +526,7 @@ type Timeline struct { CustomKey2 optional.Option[string] CustomKey3 optional.Option[string] CustomKey4 optional.Option[string] - Payload []byte + Payload encryption.EncryptedTimelineColumn ParentRequestID optional.Option[string] } diff --git a/internal/encryption/database.go b/internal/encryption/database.go new file mode 100644 index 0000000000..a1269f3d25 --- /dev/null +++ b/internal/encryption/database.go @@ -0,0 +1,58 @@ +package encryption + +import ( + "database/sql" + "database/sql/driver" + "fmt" + + "github.com/alecthomas/types/optional" +) + +var _ Encrypted = &EncryptedColumn[TimelineSubKey]{} + +// EncryptedColumn is a type that represents an encrypted column. +// +// It can be used by sqlc to map to/from a bytea column in the database. +type EncryptedColumn[SK SubKey] []byte + +var _ driver.Valuer = &EncryptedColumn[TimelineSubKey]{} +var _ sql.Scanner = &EncryptedColumn[TimelineSubKey]{} + +func (e *EncryptedColumn[SK]) SubKey() string { var sk SK; return sk.SubKey() } +func (e *EncryptedColumn[SK]) Bytes() []byte { return *e } +func (e *EncryptedColumn[SK]) Set(b []byte) { *e = b } +func (e *EncryptedColumn[SK]) Value() (driver.Value, error) { + return []byte(*e), nil +} + +func (e *EncryptedColumn[SK]) Scan(src interface{}) error { + if src == nil { + *e = nil + return nil + } + b, ok := src.([]byte) + if !ok { + return fmt.Errorf("expected []byte, got %T", src) + } + *e = b + return nil +} + +type EncryptedTimelineColumn = EncryptedColumn[TimelineSubKey] +type EncryptedAsyncColumn = EncryptedColumn[AsyncSubKey] + +type OptionalEncryptedTimelineColumn = optional.Option[EncryptedTimelineColumn] +type OptionalEncryptedAsyncColumn = optional.Option[EncryptedAsyncColumn] + +// SubKey is an interface for types that specify their own encryption subkey. +type SubKey interface{ SubKey() string } + +// TimelineSubKey is a type that represents the subkey for logs. +type TimelineSubKey struct{} + +func (TimelineSubKey) SubKey() string { return "logs" } + +// AsyncSubKey is a type that represents the subkey for async. +type AsyncSubKey struct{} + +func (AsyncSubKey) SubKey() string { return "async" } diff --git a/internal/encryption/encryption.go b/internal/encryption/encryption.go index 9815b846bc..a9b17c224f 100644 --- a/internal/encryption/encryption.go +++ b/internal/encryption/encryption.go @@ -16,16 +16,16 @@ import ( "github.com/tink-crypto/tink-go/v2/tink" ) -type SubKey string - -const ( - TimelineSubKey SubKey = "timeline" - AsyncSubKey SubKey = "async" -) +// Encrypted is an interface for values that contain encrypted data. +type Encrypted interface { + SubKey() string + Bytes() []byte + Set(data []byte) +} type DataEncryptor interface { - Encrypt(subKey SubKey, cleartext []byte) ([]byte, error) - Decrypt(subKey SubKey, encrypted []byte) ([]byte, error) + Encrypt(cleartext []byte, dest Encrypted) error + Decrypt(encrypted Encrypted) ([]byte, error) } // NoOpEncryptorNext does not encrypt and just passes the input as is. @@ -35,12 +35,15 @@ func NewNoOpEncryptor() NoOpEncryptorNext { return NoOpEncryptorNext{} } -func (n NoOpEncryptorNext) Encrypt(_ SubKey, cleartext []byte) ([]byte, error) { - return cleartext, nil +var _ DataEncryptor = NoOpEncryptorNext{} + +func (n NoOpEncryptorNext) Encrypt(cleartext []byte, dest Encrypted) error { + dest.Set(cleartext) + return nil } -func (n NoOpEncryptorNext) Decrypt(_ SubKey, encrypted []byte) ([]byte, error) { - return encrypted, nil +func (n NoOpEncryptorNext) Decrypt(encrypted Encrypted) ([]byte, error) { + return encrypted.Bytes(), nil } // KMSEncryptor encrypts and decrypts using a KMS key via tink. @@ -51,6 +54,8 @@ type KMSEncryptor struct { cachedDerived map[SubKey]tink.AEAD } +var _ DataEncryptor = &KMSEncryptor{} + func newClientWithAEAD(uri string, kms *awsv1kms.KMS) (tink.AEAD, error) { var client registry.KMSClient var err error @@ -164,7 +169,7 @@ func (k *KMSEncryptor) getDerivedPrimitive(subKey SubKey) (tink.AEAD, error) { return primitive, nil } - derived, err := deriveKeyset(k.root, []byte(subKey)) + derived, err := deriveKeyset(k.root, []byte(subKey.SubKey())) if err != nil { return nil, fmt.Errorf("failed to derive keyset: %w", err) } @@ -178,27 +183,28 @@ func (k *KMSEncryptor) getDerivedPrimitive(subKey SubKey) (tink.AEAD, error) { return primitive, nil } -func (k *KMSEncryptor) Encrypt(subKey SubKey, cleartext []byte) ([]byte, error) { - primitive, err := k.getDerivedPrimitive(subKey) +func (k *KMSEncryptor) Encrypt(cleartext []byte, dest Encrypted) error { + primitive, err := k.getDerivedPrimitive(dest) if err != nil { - return nil, fmt.Errorf("failed to get derived primitive: %w", err) + return fmt.Errorf("failed to get derived primitive: %w", err) } encrypted, err := primitive.Encrypt(cleartext, nil) if err != nil { - return nil, fmt.Errorf("failed to encrypt: %w", err) + return fmt.Errorf("failed to encrypt: %w", err) } - return encrypted, nil + dest.Set(encrypted) + return nil } -func (k *KMSEncryptor) Decrypt(subKey SubKey, encrypted []byte) ([]byte, error) { - primitive, err := k.getDerivedPrimitive(subKey) +func (k *KMSEncryptor) Decrypt(encrypted Encrypted) ([]byte, error) { + primitive, err := k.getDerivedPrimitive(encrypted) if err != nil { return nil, fmt.Errorf("failed to get derived primitive: %w", err) } - decrypted, err := primitive.Decrypt(encrypted, nil) + decrypted, err := primitive.Decrypt(encrypted.Bytes(), nil) if err != nil { return nil, fmt.Errorf("failed to decrypt: %w", err) } diff --git a/internal/encryption/encryption_test.go b/internal/encryption/encryption_test.go index b30719ff17..d8b1fd1b8f 100644 --- a/internal/encryption/encryption_test.go +++ b/internal/encryption/encryption_test.go @@ -9,13 +9,14 @@ import ( func TestNoOpEncryptor(t *testing.T) { encryptor := NoOpEncryptorNext{} - encrypted, err := encryptor.Encrypt(TimelineSubKey, []byte("hunter2")) + var encrypted EncryptedTimelineColumn + err := encryptor.Encrypt([]byte("hunter2"), &encrypted) assert.NoError(t, err) - decrypted, err := encryptor.Decrypt(TimelineSubKey, encrypted) + decryptedLogs, err := encryptor.Decrypt(&encrypted) assert.NoError(t, err) - assert.Equal(t, "hunter2", string(decrypted)) + assert.Equal(t, "hunter2", string(decryptedLogs)) } // echo -n "fake-kms://" && tinkey create-keyset --key-template AES128_GCM --out-format binary | base64 | tr '+/' '-_' | tr -d '=' @@ -25,14 +26,16 @@ func TestKMSEncryptorFakeKMS(t *testing.T) { encryptor, err := NewKMSEncryptorGenerateKey(uri, nil) assert.NoError(t, err) - encrypted, err := encryptor.Encrypt(TimelineSubKey, []byte("hunter2")) + var encrypted EncryptedTimelineColumn + err = encryptor.Encrypt([]byte("hunter2"), &encrypted) assert.NoError(t, err) - decrypted, err := encryptor.Decrypt(TimelineSubKey, encrypted) + decrypted, err := encryptor.Decrypt(&encrypted) assert.NoError(t, err) assert.Equal(t, "hunter2", string(decrypted)) + wrongSubKey := EncryptedAsyncColumn(encrypted) // Should fail to decrypt with the wrong subkey - _, err = encryptor.Decrypt(AsyncSubKey, encrypted) + _, err = encryptor.Decrypt(&wrongSubKey) assert.Error(t, err) } diff --git a/internal/encryption/integration_test.go b/internal/encryption/integration_test.go index 804f243dae..29e7c037c7 100644 --- a/internal/encryption/integration_test.go +++ b/internal/encryption/integration_test.go @@ -149,14 +149,16 @@ func TestKMSEncryptorLocalstack(t *testing.T) { encryptor, err := NewKMSEncryptorGenerateKey(uri, v1client) assert.NoError(t, err) - encrypted, err := encryptor.Encrypt(TimelineSubKey, []byte("hunter2")) + var encrypted EncryptedTimelineColumn + err = encryptor.Encrypt([]byte("hunter2"), &encrypted) assert.NoError(t, err) - decrypted, err := encryptor.Decrypt(TimelineSubKey, encrypted) + decrypted, err := encryptor.Decrypt(&encrypted) assert.NoError(t, err) assert.Equal(t, "hunter2", string(decrypted)) // Should fail to decrypt with the wrong subkey - _, err = encryptor.Decrypt(AsyncSubKey, encrypted) + wrongSubKey := EncryptedAsyncColumn(encrypted) + _, err = encryptor.Decrypt(&wrongSubKey) assert.Error(t, err) } diff --git a/sqlc.yaml b/sqlc.yaml index 65826ebafd..185560d9c1 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -64,6 +64,16 @@ sql: nullable: true go_type: type: "optional.Option[model.CronJobKey]" + - db_type: "encrypted_async" + go_type: "github.com/TBD54566975/ftl/internal/encryption.EncryptedAsyncColumn" + - db_type: "encrypted_async" + nullable: true + go_type: "github.com/TBD54566975/ftl/internal/encryption.OptionalEncryptedAsyncColumn" + - db_type: "encrypted_timeline" + go_type: "github.com/TBD54566975/ftl/internal/encryption.EncryptedTimelineColumn" + - db_type: "encrypted_timeline" + nullable: true + go_type: "github.com/TBD54566975/ftl/internal/encryption.OptionalEncryptedTimelineColumn" - db_type: "lease_key" go_type: "github.com/TBD54566975/ftl/backend/controller/leases.Key" - db_type: "lease_key"