From 9d6947b88ab5a258137211da6e506a26a9b0b30f Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 13 Sep 2023 16:37:09 +0800 Subject: [PATCH 01/36] feat: initial migration schema --- migrations/20230913081932_add_hooks_table.up.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 migrations/20230913081932_add_hooks_table.up.sql diff --git a/migrations/20230913081932_add_hooks_table.up.sql b/migrations/20230913081932_add_hooks_table.up.sql new file mode 100644 index 0000000000..820da89a62 --- /dev/null +++ b/migrations/20230913081932_add_hooks_table.up.sql @@ -0,0 +1,12 @@ +-- auth.hooks definition + +create table if not exists {{ index .Options "Namespace" }}.hooks( + name text null, + hook_uri text not null, + secret text not null, + extensibility_point text not null, + metadata json null, + constraint extensibility_point_pkey primary key (extensibility_point) +); + +comment on table {{ index .Options "Namespace" }}.hooks is 'Auth: Store of hook configuration - can be used to customize hooks for given extensibility points.'; From 7bc3cfb7740b3ef939a80b47e3f781bdec6fdab6 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 13 Sep 2023 17:17:41 +0800 Subject: [PATCH 02/36] chore: rename hooks table add trigger stubs --- internal/api/hooks.go | 6 ++++ internal/api/phone.go | 4 +++ internal/models/hooks.go | 35 +++++++++++++++++++ ... => 20230913081932_add_hook_config.up.sql} | 4 +-- 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 internal/models/hooks.go rename migrations/{20230913081932_add_hooks_table.up.sql => 20230913081932_add_hook_config.up.sql} (63%) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index c5378af54c..d6d870aa7d 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -153,6 +153,12 @@ func closeBody(rsp *http.Response) { } } +// func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig HookConfig, user *models.User, config *conf.GlobalConfiguration) error { +// // TODO: fetch extensibility point + +// return nil +// } + func triggerEventHooks(ctx context.Context, conn *storage.Connection, event HookEvent, user *models.User, config *conf.GlobalConfiguration) error { if config.Webhook.URL != "" { hookURL, err := url.Parse(config.Webhook.URL) diff --git a/internal/api/phone.go b/internal/api/phone.go index 4599e8f2ae..aa4aa304a1 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -88,10 +88,14 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, return "", internalServerError("error generating otp").WithInternalError(err) } + // if hooks.CustomSMSProvider != null { message, err := generateSMSFromTemplate(config.Sms.SMSTemplate, otp) if err != nil { return "", err } + // } else { + // triggerHooks + // } messageID, err = smsProvider.SendMessage(phone, message, channel) if err != nil { diff --git a/internal/models/hooks.go b/internal/models/hooks.go new file mode 100644 index 0000000000..123f4cde4e --- /dev/null +++ b/internal/models/hooks.go @@ -0,0 +1,35 @@ +package models + +import ( + "github.com/gobuffalo/pop/v6" +) + +type HookConfig struct { + Name string `json:"name" db:"name"` + HookURI string `json:"hook_uri" db:"hook_uri"` + Secret string `json:"secret" db:"secret"` + ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` + Metadata JSONMap `json:"metadata" db:"metadata"` +} + +// TableName overrides the table name used by pop +func (HookConfig) TableName() string { + tableName := "hook_config" + return tableName +} + +func (h *HookConfig) BeforeSave(tx *pop.Connection) error { + // TODO: Encrypt the Secret + return nil +} + +func NewHookConfig(name, hookURI, secret, extensibilityPoint string, metadata map[string]interface{}) (*HookConfig, error) { + hookConfig := &HookConfig{ + Name: name, + HookURI: hookURI, + Secret: secret, + ExtensibilityPoint: extensibilityPoint, + Metadata: metadata, + } + return hookConfig, nil +} diff --git a/migrations/20230913081932_add_hooks_table.up.sql b/migrations/20230913081932_add_hook_config.up.sql similarity index 63% rename from migrations/20230913081932_add_hooks_table.up.sql rename to migrations/20230913081932_add_hook_config.up.sql index 820da89a62..5be4017245 100644 --- a/migrations/20230913081932_add_hooks_table.up.sql +++ b/migrations/20230913081932_add_hook_config.up.sql @@ -1,6 +1,6 @@ -- auth.hooks definition -create table if not exists {{ index .Options "Namespace" }}.hooks( +create table if not exists {{ index .Options "Namespace" }}.hook_config( name text null, hook_uri text not null, secret text not null, @@ -9,4 +9,4 @@ create table if not exists {{ index .Options "Namespace" }}.hooks( constraint extensibility_point_pkey primary key (extensibility_point) ); -comment on table {{ index .Options "Namespace" }}.hooks is 'Auth: Store of hook configuration - can be used to customize hooks for given extensibility points.'; +comment on table {{ index .Options "Namespace" }}.hook_config is 'Auth: Store of hook configuration - can be used to customize hooks for given extensibility points.'; From e850b02b1ca5d26bd8cd858a369080c9a8b1de78 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 13 Sep 2023 17:30:03 +0800 Subject: [PATCH 03/36] feat: add trigger mechanism --- internal/api/hooks.go | 75 +++++++++++++++++++++++++++++++++++++--- internal/models/hooks.go | 3 +- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index d6d870aa7d..1e50ef1ea6 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -153,12 +153,79 @@ func closeBody(rsp *http.Response) { } } -// func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig HookConfig, user *models.User, config *conf.GlobalConfiguration) error { -// // TODO: fetch extensibility point +func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { + // TODO: fetch extensibility point + return _triggerAuthHook(ctx, hookConfig.HookURI, hookConfig.Secret, conn, hookConfig.ExtensibilityPoint, user, config) +} + +// TODO: rename this +func _triggerAuthHook(ctx context.Context, hookURL string, secret string, conn *storage.Connection, extensibilityPoint string, user *models.User, config *conf.GlobalConfiguration) error { + + // TODO: Change this to take in the various fields that need to be passed + payload := struct { + User *models.User `json:"user"` + }{ + User: user, + } + data, err := json.Marshal(&payload) + if err != nil { + return internalServerError("Failed to serialize the data for signup webhook").WithInternalError(err) + } + + sha, err := checksum(data) + if err != nil { + return internalServerError("Failed to checksum the data for signup webhook").WithInternalError(err) + } + + claims := webhookClaims{ + StandardClaims: jwt.StandardClaims{ + IssuedAt: time.Now().Unix(), + Subject: uuid.Nil.String(), + Issuer: gotrueIssuer, + }, + SHA256: sha, + } + + w := Webhook{ + WebhookConfig: &config.Webhook, + jwtSecret: secret, + claims: claims, + payload: data, + } + + w.URL = hookURL -// return nil -// } + body, err := w.trigger() + if body != nil { + defer utilities.SafeClose(body) + } + if err == nil && body != nil { + // TODO: figure out how we can dictate the response + webhookRsp := &WebhookResponse{} + decoder := json.NewDecoder(body) + if err = decoder.Decode(webhookRsp); err != nil { + return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + } + return conn.Transaction(func(tx *storage.Connection) error { + if webhookRsp.UserMetaData != nil { + user.UserMetaData = nil + if terr := user.UpdateUserMetaData(tx, webhookRsp.UserMetaData); terr != nil { + return terr + } + } + if webhookRsp.AppMetaData != nil { + user.AppMetaData = nil + if terr := user.UpdateAppMetaData(tx, webhookRsp.AppMetaData); terr != nil { + return terr + } + } + return nil + }) + } + return err +} +// Deprecate this func triggerEventHooks(ctx context.Context, conn *storage.Connection, event HookEvent, user *models.User, config *conf.GlobalConfiguration) error { if config.Webhook.URL != "" { hookURL, err := url.Parse(config.Webhook.URL) diff --git a/internal/models/hooks.go b/internal/models/hooks.go index 123f4cde4e..ce5ce7ec2f 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -5,7 +5,8 @@ import ( ) type HookConfig struct { - Name string `json:"name" db:"name"` + Name string `json:"name" db:"name"` + // TODO: change this t o just URI HookURI string `json:"hook_uri" db:"hook_uri"` Secret string `json:"secret" db:"secret"` ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` From 0c138aee91a1fc894241c92bc62fa608b98a8802 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 13 Sep 2023 20:11:32 +0800 Subject: [PATCH 04/36] feat: add corresponding errors --- internal/api/phone.go | 10 +++++++--- internal/models/errors.go | 9 +++++++++ internal/models/hooks.go | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/internal/api/phone.go b/internal/api/phone.go index aa4aa304a1..09ea6bd696 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -42,6 +42,7 @@ func formatPhoneNumber(phone string) string { // sendPhoneConfirmation sends an otp to the user's phone number func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, user *models.User, phone, otpType string, smsProvider sms_provider.SmsProvider, channel string) (string, error) { + config := a.config var token *string @@ -87,14 +88,17 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, if err != nil { return "", internalServerError("error generating otp").WithInternalError(err) } + // Extensibility Point - initialize + // TODO: I guesss this could be presented as a struct above - // if hooks.CustomSMSProvider != null { + // hookConfiguration = fetchHookConfiguration("custom-sms-provider") + // if hookConfiguration != nil{ + // trigger hooks + // } else { message, err := generateSMSFromTemplate(config.Sms.SMSTemplate, otp) if err != nil { return "", err } - // } else { - // triggerHooks // } messageID, err = smsProvider.SendMessage(phone, message, channel) diff --git a/internal/models/errors.go b/internal/models/errors.go index 381517b6cb..395e8fcab4 100644 --- a/internal/models/errors.go +++ b/internal/models/errors.go @@ -23,6 +23,8 @@ func IsNotFoundError(err error) bool { return true case FlowStateNotFoundError, *FlowStateNotFoundError: return true + case HookConfigNotFoundError, *HookConfigNotFoundError: + return true } return false } @@ -105,3 +107,10 @@ type FlowStateNotFoundError struct{} func (e FlowStateNotFoundError) Error() string { return "Flow State not found" } + +// HookConfigNotFoundError represents when a HookConfig is not found. +type HookConfigNotFoundError struct{} + +func (e HookConfigNotFoundError) Error() string { + return "Hook Config not Found" +} diff --git a/internal/models/hooks.go b/internal/models/hooks.go index ce5ce7ec2f..3cca801a05 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -1,7 +1,10 @@ package models import ( + "database/sql" "github.com/gobuffalo/pop/v6" + "github.com/pkg/errors" + "github.com/supabase/gotrue/internal/storage" ) type HookConfig struct { @@ -34,3 +37,15 @@ func NewHookConfig(name, hookURI, secret, extensibilityPoint string, metadata ma } return hookConfig, nil } + +func fetchHookConfiguration(tx *storage.Connection, query string, args ...interface{}) (*HookConfig, error) { + obj := &HookConfig{} + if err := tx.Eager().Q().Where(query, args...).First(obj); err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, HookConfigNotFoundError{} + } + return nil, errors.Wrap(err, "error finding user") + } + + return obj, nil +} From 496a0739956b264fa65a90d144c2aea4a907c33c Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 13 Sep 2023 20:23:43 +0800 Subject: [PATCH 05/36] feat: add logic for fetching hook configuration --- internal/api/phone.go | 28 +++++++++++++++++----------- internal/models/hooks.go | 2 +- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/internal/api/phone.go b/internal/api/phone.go index 09ea6bd696..0423df1be7 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -3,6 +3,7 @@ package api import ( "bytes" "context" + "fmt" "regexp" "strings" "text/template" @@ -88,22 +89,27 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, if err != nil { return "", internalServerError("error generating otp").WithInternalError(err) } + // Extensibility Point - initialize // TODO: I guesss this could be presented as a struct above - // hookConfiguration = fetchHookConfiguration("custom-sms-provider") - // if hookConfiguration != nil{ - // trigger hooks - // } else { - message, err := generateSMSFromTemplate(config.Sms.SMSTemplate, otp) - if err != nil { + hookConfiguration, err := models.FetchHookConfiguration(tx, "extensibility_point = ?", "custom-sms-provider") + if (err != nil && err != models.HookConfigNotFoundError{}) { return "", err } - // } - - messageID, err = smsProvider.SendMessage(phone, message, channel) - if err != nil { - return messageID, err + if (hookConfiguration != nil && err == models.HookConfigNotFoundError{}) { + // TODO: Placeholder + fmt.Println("execute stuff") + } else { + message, err := generateSMSFromTemplate(config.Sms.SMSTemplate, otp) + if err != nil { + return "", err + } + + messageID, err = smsProvider.SendMessage(phone, message, channel) + if err != nil { + return messageID, err + } } } diff --git a/internal/models/hooks.go b/internal/models/hooks.go index 3cca801a05..db09c2fd5c 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -38,7 +38,7 @@ func NewHookConfig(name, hookURI, secret, extensibilityPoint string, metadata ma return hookConfig, nil } -func fetchHookConfiguration(tx *storage.Connection, query string, args ...interface{}) (*HookConfig, error) { +func FetchHookConfiguration(tx *storage.Connection, query string, args ...interface{}) (*HookConfig, error) { obj := &HookConfig{} if err := tx.Eager().Q().Where(query, args...).First(obj); err != nil { if errors.Cause(err) == sql.ErrNoRows { From 759b5e425e5c6f78116a1068c74b76dd283df834 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 13 Sep 2023 20:37:04 +0800 Subject: [PATCH 06/36] feat: link sms sending to trigger --- internal/api/phone.go | 15 +++++++++------ internal/models/hooks.go | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/api/phone.go b/internal/api/phone.go index 0423df1be7..27a5486bf9 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -3,7 +3,6 @@ package api import ( "bytes" "context" - "fmt" "regexp" "strings" "text/template" @@ -94,13 +93,15 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, // TODO: I guesss this could be presented as a struct above hookConfiguration, err := models.FetchHookConfiguration(tx, "extensibility_point = ?", "custom-sms-provider") - if (err != nil && err != models.HookConfigNotFoundError{}) { + if err != nil && models.IsNotFoundError(err) { return "", err } - if (hookConfiguration != nil && err == models.HookConfigNotFoundError{}) { - // TODO: Placeholder - fmt.Println("execute stuff") - } else { + if hookConfiguration != nil && models.IsNotFoundError(err) { + // TODO: find a way to wrap and properly Fetch the resp + if terr := triggerAuthHook(ctx, tx, *hookConfiguration, user, config); terr != nil { + return "", terr + } + } else if models.IsNotFoundError(err) { message, err := generateSMSFromTemplate(config.Sms.SMSTemplate, otp) if err != nil { return "", err @@ -110,6 +111,8 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, if err != nil { return messageID, err } + } else { + return "", err } } diff --git a/internal/models/hooks.go b/internal/models/hooks.go index db09c2fd5c..b854b91b89 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -38,6 +38,7 @@ func NewHookConfig(name, hookURI, secret, extensibilityPoint string, metadata ma return hookConfig, nil } +// TODO: Make this into smaller function and add wrapper func FetchHookConfiguration(tx *storage.Connection, query string, args ...interface{}) (*HookConfig, error) { obj := &HookConfig{} if err := tx.Eager().Q().Where(query, args...).First(obj); err != nil { From 8c878610c0f1b2e154260e373d6d03f4b5c94f10 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 13 Sep 2023 23:12:58 +0800 Subject: [PATCH 07/36] fix: update schemas to include request response --- internal/api/hooks.go | 31 ++++--------------- internal/models/hooks.go | 28 +++++++++-------- .../20230913081932_add_hook_config.up.sql | 4 ++- 3 files changed, 24 insertions(+), 39 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 1e50ef1ea6..fd009dd5d2 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -154,14 +154,13 @@ func closeBody(rsp *http.Response) { } func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { - // TODO: fetch extensibility point - return _triggerAuthHook(ctx, hookConfig.HookURI, hookConfig.Secret, conn, hookConfig.ExtensibilityPoint, user, config) + return _triggerAuthHook(ctx, hookConfig.URI, hookConfig.Secret, conn, hookConfig.ExtensibilityPoint, user, config) } // TODO: rename this func _triggerAuthHook(ctx context.Context, hookURL string, secret string, conn *storage.Connection, extensibilityPoint string, user *models.User, config *conf.GlobalConfiguration) error { - // TODO: Change this to take in the various fields that need to be passed + // TODO: Need also to filter the relevant user fields payload := struct { User *models.User `json:"user"` }{ @@ -169,21 +168,17 @@ func _triggerAuthHook(ctx context.Context, hookURL string, secret string, conn * } data, err := json.Marshal(&payload) if err != nil { - return internalServerError("Failed to serialize the data for signup webhook").WithInternalError(err) - } - - sha, err := checksum(data) - if err != nil { - return internalServerError("Failed to checksum the data for signup webhook").WithInternalError(err) + // TODO: include name of hook that failed + return internalServerError("Failed to serialize the data for hook").WithInternalError(err) } + // TODO: Sign the payload here claims := webhookClaims{ StandardClaims: jwt.StandardClaims{ IssuedAt: time.Now().Unix(), Subject: uuid.Nil.String(), Issuer: gotrueIssuer, }, - SHA256: sha, } w := Webhook{ @@ -199,6 +194,7 @@ func _triggerAuthHook(ctx context.Context, hookURL string, secret string, conn * if body != nil { defer utilities.SafeClose(body) } + // TODO: this should return webhook response and we should modify the method signature if err == nil && body != nil { // TODO: figure out how we can dictate the response webhookRsp := &WebhookResponse{} @@ -206,21 +202,6 @@ func _triggerAuthHook(ctx context.Context, hookURL string, secret string, conn * if err = decoder.Decode(webhookRsp); err != nil { return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) } - return conn.Transaction(func(tx *storage.Connection) error { - if webhookRsp.UserMetaData != nil { - user.UserMetaData = nil - if terr := user.UpdateUserMetaData(tx, webhookRsp.UserMetaData); terr != nil { - return terr - } - } - if webhookRsp.AppMetaData != nil { - user.AppMetaData = nil - if terr := user.UpdateAppMetaData(tx, webhookRsp.AppMetaData); terr != nil { - return terr - } - } - return nil - }) } return err } diff --git a/internal/models/hooks.go b/internal/models/hooks.go index b854b91b89..d06878802c 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -8,11 +8,12 @@ import ( ) type HookConfig struct { - Name string `json:"name" db:"name"` - // TODO: change this t o just URI - HookURI string `json:"hook_uri" db:"hook_uri"` + Name string `json:"name" db:"name"` + URI string `json:"hook_uri" db:"hook_uri"` Secret string `json:"secret" db:"secret"` ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` + RequestSchema JSONMap `json:"request_schema" db:"request_schema"` + ResponseSchema JSONMap `json:"response_schema" db:"response_schema"` Metadata JSONMap `json:"metadata" db:"metadata"` } @@ -27,16 +28,17 @@ func (h *HookConfig) BeforeSave(tx *pop.Connection) error { return nil } -func NewHookConfig(name, hookURI, secret, extensibilityPoint string, metadata map[string]interface{}) (*HookConfig, error) { - hookConfig := &HookConfig{ - Name: name, - HookURI: hookURI, - Secret: secret, - ExtensibilityPoint: extensibilityPoint, - Metadata: metadata, - } - return hookConfig, nil -} +// Shouldn't need to create a new hook config unless admin, can be implemented later +// func NewHookConfig(name, URI, secret, extensibilityPoint string, metadata map[string]interface{}) (*HookConfig, error) { +// hookConfig := &HookConfig{ +// Name: name, +// URI: URI, +// Secret: secret, +// ExtensibilityPoint: extensibilityPoint, +// Metadata: metadata, +// } +// return hookConfig, nil +// } // TODO: Make this into smaller function and add wrapper func FetchHookConfiguration(tx *storage.Connection, query string, args ...interface{}) (*HookConfig, error) { diff --git a/migrations/20230913081932_add_hook_config.up.sql b/migrations/20230913081932_add_hook_config.up.sql index 5be4017245..dabbea93d1 100644 --- a/migrations/20230913081932_add_hook_config.up.sql +++ b/migrations/20230913081932_add_hook_config.up.sql @@ -2,9 +2,11 @@ create table if not exists {{ index .Options "Namespace" }}.hook_config( name text null, - hook_uri text not null, + uri text not null, secret text not null, extensibility_point text not null, + request_schema jsonb not null, + response_schema jsonb not null, metadata json null, constraint extensibility_point_pkey primary key (extensibility_point) ); From e5da60a46e7d9b434f1455fe6cd3d6cf0ada5ee1 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 13 Sep 2023 23:50:20 +0800 Subject: [PATCH 08/36] fix: update the name --- internal/api/phone.go | 2 +- internal/models/hooks.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/api/phone.go b/internal/api/phone.go index 27a5486bf9..52dcc532eb 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -93,7 +93,7 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, // TODO: I guesss this could be presented as a struct above hookConfiguration, err := models.FetchHookConfiguration(tx, "extensibility_point = ?", "custom-sms-provider") - if err != nil && models.IsNotFoundError(err) { + if err != nil && !models.IsNotFoundError(err) { return "", err } if hookConfiguration != nil && models.IsNotFoundError(err) { diff --git a/internal/models/hooks.go b/internal/models/hooks.go index d06878802c..c45dbb08bb 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -9,7 +9,7 @@ import ( type HookConfig struct { Name string `json:"name" db:"name"` - URI string `json:"hook_uri" db:"hook_uri"` + URI string `json:"uri" db:"uri"` Secret string `json:"secret" db:"secret"` ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` RequestSchema JSONMap `json:"request_schema" db:"request_schema"` From 5c50c78d0204962b58d5c7d80e2bbd9484b0f67c Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Tue, 19 Sep 2023 10:28:44 +0800 Subject: [PATCH 09/36] refactor: condense function --- internal/api/hooks.go | 25 ++++++++++++++++--------- internal/api/phone.go | 6 ++---- internal/models/hooks.go | 20 ++++++-------------- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index fd009dd5d2..5ccb5170e6 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -26,6 +26,16 @@ import ( type HookEvent string +type ExtensibilityPoint struct { + Name string +} + +func NewExtensibilityPoint(name string) *ExtensibilityPoint { + return &ExtensibilityPoint{ + Name: name, + } +} + const ( headerHookSignature = "x-webhook-signature" defaultHookRetries = 3 @@ -36,6 +46,10 @@ const ( LoginEvent = "login" ) +const ( + PhoneProviderExtensibilityPoint = "phone-provider" +) + var defaultTimeout = time.Second * 5 type webhookClaims struct { @@ -154,13 +168,6 @@ func closeBody(rsp *http.Response) { } func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { - return _triggerAuthHook(ctx, hookConfig.URI, hookConfig.Secret, conn, hookConfig.ExtensibilityPoint, user, config) -} - -// TODO: rename this -func _triggerAuthHook(ctx context.Context, hookURL string, secret string, conn *storage.Connection, extensibilityPoint string, user *models.User, config *conf.GlobalConfiguration) error { - // TODO: Change this to take in the various fields that need to be passed - // TODO: Need also to filter the relevant user fields payload := struct { User *models.User `json:"user"` }{ @@ -183,12 +190,12 @@ func _triggerAuthHook(ctx context.Context, hookURL string, secret string, conn * w := Webhook{ WebhookConfig: &config.Webhook, - jwtSecret: secret, + jwtSecret: hookConfig.Secret, claims: claims, payload: data, } - w.URL = hookURL + w.URL = hookConfig.URI body, err := w.trigger() if body != nil { diff --git a/internal/api/phone.go b/internal/api/phone.go index 52dcc532eb..d6d2bb2fe0 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -42,7 +42,6 @@ func formatPhoneNumber(phone string) string { // sendPhoneConfirmation sends an otp to the user's phone number func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, user *models.User, phone, otpType string, smsProvider sms_provider.SmsProvider, channel string) (string, error) { - config := a.config var token *string @@ -90,9 +89,8 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, } // Extensibility Point - initialize - // TODO: I guesss this could be presented as a struct above - - hookConfiguration, err := models.FetchHookConfiguration(tx, "extensibility_point = ?", "custom-sms-provider") + phoneExtensibilityPoint := NewExtensibilityPoint("custom-sms-provider") + hookConfiguration, err := models.FindHookByExtensibilityPoint(tx, phoneExtensibilityPoint.Name) if err != nil && !models.IsNotFoundError(err) { return "", err } diff --git a/internal/models/hooks.go b/internal/models/hooks.go index c45dbb08bb..24d4cfb6d3 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -28,20 +28,12 @@ func (h *HookConfig) BeforeSave(tx *pop.Connection) error { return nil } -// Shouldn't need to create a new hook config unless admin, can be implemented later -// func NewHookConfig(name, URI, secret, extensibilityPoint string, metadata map[string]interface{}) (*HookConfig, error) { -// hookConfig := &HookConfig{ -// Name: name, -// URI: URI, -// Secret: secret, -// ExtensibilityPoint: extensibilityPoint, -// Metadata: metadata, -// } -// return hookConfig, nil -// } - -// TODO: Make this into smaller function and add wrapper -func FetchHookConfiguration(tx *storage.Connection, query string, args ...interface{}) (*HookConfig, error) { +func FindHookByExtensibilityPoint(tx *storage.Connection, name string) (*HookConfig, error) { + return findHookConfiguration(tx, "extensibility_point = ?", name) + +} + +func findHookConfiguration(tx *storage.Connection, query string, args ...interface{}) (*HookConfig, error) { obj := &HookConfig{} if err := tx.Eager().Q().Where(query, args...).First(obj); err != nil { if errors.Cause(err) == sql.ErrNoRows { From 493f65515e9b300b86e47d832f36e1f58cf9c2e2 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 20 Sep 2023 00:16:03 +0800 Subject: [PATCH 10/36] feat: add separate auth hook logic --- internal/api/hooks.go | 149 +++++++++++++++++++++++++++++++++++------- 1 file changed, 126 insertions(+), 23 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 5ccb5170e6..79ea662683 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -40,10 +40,12 @@ const ( headerHookSignature = "x-webhook-signature" defaultHookRetries = 3 gotrueIssuer = "gotrue" - ValidateEvent = "validate" - SignupEvent = "signup" - EmailChangeEvent = "email_change" - LoginEvent = "login" + // TODO (Joel): Properly substitute this + authHookIssuer = "auth" + ValidateEvent = "validate" + SignupEvent = "signup" + EmailChangeEvent = "email_change" + LoginEvent = "login" ) const ( @@ -70,6 +72,100 @@ type WebhookResponse struct { UserMetaData map[string]interface{} `json:"user_metadata,omitempty"` } +// Duplicate of Webhook, should eventually modify the fields passed +type AuthHook struct { + *conf.WebhookConfig + // Decide what should go here + jwtSecret string + claims jwt.Claims + payload []byte +} + +func (w *AuthHook) trigger() (io.ReadCloser, error) { + timeout := defaultTimeout + if w.TimeoutSec > 0 { + timeout = time.Duration(w.TimeoutSec) * time.Second + } + + if w.Retries == 0 { + w.Retries = defaultHookRetries + } + + hooklog := logrus.WithFields(logrus.Fields{ + "component": "webhook", + "url": w.URL, + "signed": w.jwtSecret != "", + "instance_id": uuid.Nil.String(), + }) + client := http.Client{ + Timeout: timeout, + } + payload, jwtErr := w.generateSignature() + if jwtErr != nil { + return nil, jwtErr + } + + for i := 0; i < w.Retries; i++ { + hooklog = hooklog.WithField("attempt", i+1) + hooklog.Info("Starting to perform signup hook request") + + req, err := http.NewRequest(http.MethodPost, w.URL, bytes.NewBuffer(payload)) + if err != nil { + return nil, internalServerError("Failed to make request object").WithInternalError(err) + } + req.Header.Set("Content-Type", "application/json") + watcher, req := watchForConnection(req) + + start := time.Now() + rsp, err := client.Do(req) + if err != nil { + if terr, ok := err.(net.Error); ok && terr.Timeout() { + // timed out - try again? + if i == w.Retries-1 { + closeBody(rsp) + return nil, httpError(http.StatusGatewayTimeout, "Failed to perform webhook in time frame (%v seconds)", timeout.Seconds()) + } + hooklog.Info("Request timed out") + continue + } else if watcher.gotConn { + closeBody(rsp) + return nil, internalServerError("Failed to trigger webhook to %s", w.URL).WithInternalError(err) + } else { + closeBody(rsp) + return nil, httpError(http.StatusBadGateway, "Failed to connect to %s", w.URL) + } + } + dur := time.Since(start) + rspLog := hooklog.WithFields(logrus.Fields{ + "status_code": rsp.StatusCode, + "dur": dur.Nanoseconds(), + }) + switch rsp.StatusCode { + case http.StatusOK, http.StatusNoContent, http.StatusAccepted: + rspLog.Infof("Finished processing webhook in %s", dur) + var body io.ReadCloser + if rsp.ContentLength > 0 { + body = rsp.Body + } + return body, nil + default: + rspLog.Infof("Bad response for webhook %d in %s", rsp.StatusCode, dur) + } + } + + hooklog.Infof("Failed to process webhook for %s after %d attempts", w.URL, w.Retries) + return nil, unprocessableEntityError("Failed to handle signup webhook") +} + +func (a *AuthHook) generateSignature() ([]byte, error) { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, a.claims) + tokenString, err := token.SignedString([]byte(a.jwtSecret)) + if err != nil { + return []byte(""), internalServerError("Failed build signing string").WithInternalError(err) + } + return []byte(tokenString), nil +} + func (w *Webhook) trigger() (io.ReadCloser, error) { timeout := defaultTimeout if w.TimeoutSec > 0 { @@ -168,49 +264,55 @@ func closeBody(rsp *http.Response) { } func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { + // TODO: these should be filtered but I'm not sure how payload := struct { User *models.User `json:"user"` }{ User: user, } + data, err := json.Marshal(&payload) if err != nil { // TODO: include name of hook that failed return internalServerError("Failed to serialize the data for hook").WithInternalError(err) } - // TODO: Sign the payload here - claims := webhookClaims{ - StandardClaims: jwt.StandardClaims{ - IssuedAt: time.Now().Unix(), - Subject: uuid.Nil.String(), - Issuer: gotrueIssuer, - }, + // TODO: substitute with a custom Claims intrface + claims := jwt.MapClaims{ + "IssuedAt": time.Now().Unix(), + "Subject": uuid.Nil.String(), + "Issuer": authHookIssuer, + "Data": data, } - w := Webhook{ + a := AuthHook{ WebhookConfig: &config.Webhook, jwtSecret: hookConfig.Secret, claims: claims, - payload: data, } - w.URL = hookConfig.URI + // Works out because this is a http hook - eventually needs to change + a.URL = hookConfig.URI - body, err := w.trigger() + body, err := a.trigger() if body != nil { defer utilities.SafeClose(body) } + // TODO: this should return webhook response and we should modify the method signature - if err == nil && body != nil { - // TODO: figure out how we can dictate the response - webhookRsp := &WebhookResponse{} - decoder := json.NewDecoder(body) - if err = decoder.Decode(webhookRsp); err != nil { - return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) - } + //if err == nil && body != nil { + // TODO: figure out how we can dictate the response + // Also need to validate. For now this is fine because the expected is to return nothing + // webhookRsp := &WebhookResponse{} + // decoder := json.NewDecoder(body) + // if err = decoder.Decode(webhookRsp); err != nil { + // return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + // } + // } + if err != nil { + return err } - return err + return nil } // Deprecate this @@ -302,6 +404,7 @@ func triggerHook(ctx context.Context, hookURL *url.URL, secret string, conn *sto if err = decoder.Decode(webhookRsp); err != nil { return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) } + return conn.Transaction(func(tx *storage.Connection) error { if webhookRsp.UserMetaData != nil { user.UserMetaData = nil From 50e1fa801bf039200a37ef250081949b7e06cb6c Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 20 Sep 2023 01:16:32 +0800 Subject: [PATCH 11/36] refactor: change types --- internal/api/hooks.go | 25 +++++++++++++++++++++---- internal/api/phone.go | 4 +++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 79ea662683..8f6e149c86 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -8,6 +8,7 @@ import ( "encoding/json" "io" "net" + "fmt" "net/http" "net/http/httptrace" "net/url" @@ -104,12 +105,24 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { if jwtErr != nil { return nil, jwtErr } + fmt.Println(w.URL) + jsonString := struct { + JWT string `json:"jwt"` + }{ + JWT: payload, + } + + // Marshal the JSON object to JSON format + load, err := json.Marshal(jsonString) + if err != nil { + return nil, err + } for i := 0; i < w.Retries; i++ { hooklog = hooklog.WithField("attempt", i+1) hooklog.Info("Starting to perform signup hook request") - req, err := http.NewRequest(http.MethodPost, w.URL, bytes.NewBuffer(payload)) + req, err := http.NewRequest(http.MethodPost, w.URL, bytes.NewBuffer(load)) if err != nil { return nil, internalServerError("Failed to make request object").WithInternalError(err) } @@ -157,13 +170,13 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { return nil, unprocessableEntityError("Failed to handle signup webhook") } -func (a *AuthHook) generateSignature() ([]byte, error) { +func (a *AuthHook) generateSignature() (string, error) { token := jwt.NewWithClaims(jwt.SigningMethodHS256, a.claims) tokenString, err := token.SignedString([]byte(a.jwtSecret)) if err != nil { - return []byte(""), internalServerError("Failed build signing string").WithInternalError(err) + return "", internalServerError("Failed build signing string").WithInternalError(err) } - return []byte(tokenString), nil + return tokenString, nil } func (w *Webhook) trigger() (io.ReadCloser, error) { @@ -265,6 +278,7 @@ func closeBody(rsp *http.Response) { func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { // TODO: these should be filtered but I'm not sure how + fmt.Println("auth hook") payload := struct { User *models.User `json:"user"` }{ @@ -277,6 +291,8 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m return internalServerError("Failed to serialize the data for hook").WithInternalError(err) } + fmt.Println("more data") + // TODO: substitute with a custom Claims intrface claims := jwt.MapClaims{ "IssuedAt": time.Now().Unix(), @@ -298,6 +314,7 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m if body != nil { defer utilities.SafeClose(body) } + fmt.Println("triggered") // TODO: this should return webhook response and we should modify the method signature //if err == nil && body != nil { diff --git a/internal/api/phone.go b/internal/api/phone.go index d6d2bb2fe0..04f39bb31e 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -7,6 +7,7 @@ import ( "strings" "text/template" "time" + "fmt" "github.com/pkg/errors" "github.com/supabase/gotrue/internal/api/sms_provider" @@ -94,7 +95,8 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, if err != nil && !models.IsNotFoundError(err) { return "", err } - if hookConfiguration != nil && models.IsNotFoundError(err) { + fmt.Println(hookConfiguration) + if hookConfiguration != nil { // TODO: find a way to wrap and properly Fetch the resp if terr := triggerAuthHook(ctx, tx, *hookConfiguration, user, config); terr != nil { return "", terr From b6bbbff6df8a2f5fd67f93faae133f386d2f77a9 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Mon, 25 Sep 2023 14:37:34 +0800 Subject: [PATCH 12/36] deps: add jsonschema --- go.mod | 3 +++ go.sum | 6 ++++++ internal/api/hooks.go | 40 +++++++++++++++++----------------------- internal/api/phone.go | 1 - 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index d9176ffe8d..58705b4857 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,9 @@ require ( require ( github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/gobuffalo/nulls v0.4.2 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect ) require ( diff --git a/go.sum b/go.sum index 66af8890b3..b232dff5a8 100644 --- a/go.sum +++ b/go.sum @@ -491,6 +491,12 @@ github.com/supabase/mailme v0.0.0-20230628061017-01f68480c747 h1:FIUdLV4o5JLsJno github.com/supabase/mailme v0.0.0-20230628061017-01f68480c747/go.mod h1:kWsnmPfUBZTavlXYkfJrE9unzmmRAIi/kqsxXfEWEY8= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 8f6e149c86..5d969c01a6 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -8,7 +8,6 @@ import ( "encoding/json" "io" "net" - "fmt" "net/http" "net/http/httptrace" "net/url" @@ -105,16 +104,15 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { if jwtErr != nil { return nil, jwtErr } - fmt.Println(w.URL) jsonString := struct { - JWT string `json:"jwt"` - }{ - JWT: payload, - } + JWT string `json:"jwt"` + }{ + JWT: payload, + } - // Marshal the JSON object to JSON format - load, err := json.Marshal(jsonString) + // Marshal the JSON object to JSON format + load, err := json.Marshal(jsonString) if err != nil { return nil, err } @@ -278,27 +276,24 @@ func closeBody(rsp *http.Response) { func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { // TODO: these should be filtered but I'm not sure how - fmt.Println("auth hook") - payload := struct { - User *models.User `json:"user"` - }{ - User: user, - } - - data, err := json.Marshal(&payload) - if err != nil { - // TODO: include name of hook that failed - return internalServerError("Failed to serialize the data for hook").WithInternalError(err) - } + // payload := struct { + // User *models.User `json:"user"` + // }{ + // User: user, + // } - fmt.Println("more data") + // data, err := json.Marshal(&payload) + // if err != nil { + // // TODO: include name of hook that failed + // return internalServerError("Failed to serialize the data for hook").WithInternalError(err) + // } // TODO: substitute with a custom Claims intrface claims := jwt.MapClaims{ "IssuedAt": time.Now().Unix(), "Subject": uuid.Nil.String(), "Issuer": authHookIssuer, - "Data": data, + "Data": user, } a := AuthHook{ @@ -314,7 +309,6 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m if body != nil { defer utilities.SafeClose(body) } - fmt.Println("triggered") // TODO: this should return webhook response and we should modify the method signature //if err == nil && body != nil { diff --git a/internal/api/phone.go b/internal/api/phone.go index 04f39bb31e..3de43a3284 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -7,7 +7,6 @@ import ( "strings" "text/template" "time" - "fmt" "github.com/pkg/errors" "github.com/supabase/gotrue/internal/api/sms_provider" From abd73c8670c4d41be3acfedd34ee6692475f61fd Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Tue, 26 Sep 2023 15:44:35 +0800 Subject: [PATCH 13/36] refactor: remove stray statement --- internal/api/phone.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/api/phone.go b/internal/api/phone.go index 3de43a3284..f0aa144121 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -94,7 +94,6 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, if err != nil && !models.IsNotFoundError(err) { return "", err } - fmt.Println(hookConfiguration) if hookConfiguration != nil { // TODO: find a way to wrap and properly Fetch the resp if terr := triggerAuthHook(ctx, tx, *hookConfiguration, user, config); terr != nil { From 33f1debfcb21b91cbcc8736868b909934906d69f Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 27 Sep 2023 14:21:23 +0800 Subject: [PATCH 14/36] refactor: uncomment more sectoins --- internal/api/hooks.go | 49 ++++++++++++++++++++----------------------- internal/api/phone.go | 2 +- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 5d969c01a6..f89874b813 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" "io" "net" "net/http" @@ -100,7 +101,7 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { client := http.Client{ Timeout: timeout, } - payload, jwtErr := w.generateSignature() + signedPayload, jwtErr := w.generateSignature() if jwtErr != nil { return nil, jwtErr } @@ -108,11 +109,11 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { jsonString := struct { JWT string `json:"jwt"` }{ - JWT: payload, + JWT: signedPayload, } // Marshal the JSON object to JSON format - load, err := json.Marshal(jsonString) + requestLoad, err := json.Marshal(jsonString) if err != nil { return nil, err } @@ -120,7 +121,7 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { hooklog = hooklog.WithField("attempt", i+1) hooklog.Info("Starting to perform signup hook request") - req, err := http.NewRequest(http.MethodPost, w.URL, bytes.NewBuffer(load)) + req, err := http.NewRequest(http.MethodPost, w.URL, bytes.NewBuffer(requestLoad)) if err != nil { return nil, internalServerError("Failed to make request object").WithInternalError(err) } @@ -129,6 +130,8 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { start := time.Now() rsp, err := client.Do(req) + fmt.Println("check here") + fmt.Println(err) if err != nil { if terr, ok := err.(net.Error); ok && terr.Timeout() { // timed out - try again? @@ -274,26 +277,21 @@ func closeBody(rsp *http.Response) { } } +// TODO: Add an additional metadata param func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { - // TODO: these should be filtered but I'm not sure how - // payload := struct { - // User *models.User `json:"user"` - // }{ - // User: user, - // } - - // data, err := json.Marshal(&payload) - // if err != nil { - // // TODO: include name of hook that failed - // return internalServerError("Failed to serialize the data for hook").WithInternalError(err) - // } + // TODO: filterJSONPayload(user, metadata) + payload := struct { + User *models.User `json:"user"` + }{ + User: user, + } // TODO: substitute with a custom Claims intrface claims := jwt.MapClaims{ "IssuedAt": time.Now().Unix(), "Subject": uuid.Nil.String(), "Issuer": authHookIssuer, - "Data": user, + "Events": payload, } a := AuthHook{ @@ -311,15 +309,14 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m } // TODO: this should return webhook response and we should modify the method signature - //if err == nil && body != nil { - // TODO: figure out how we can dictate the response - // Also need to validate. For now this is fine because the expected is to return nothing - // webhookRsp := &WebhookResponse{} - // decoder := json.NewDecoder(body) - // if err = decoder.Decode(webhookRsp); err != nil { - // return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) - // } - // } + if err == nil && body != nil { + // TODO: Fetch the output from hook config and then validate against it + webhookRsp := &WebhookResponse{} + decoder := json.NewDecoder(body) + if err = decoder.Decode(webhookRsp); err != nil { + return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + } + } if err != nil { return err } diff --git a/internal/api/phone.go b/internal/api/phone.go index f0aa144121..7b1ce45d8e 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -89,7 +89,7 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, } // Extensibility Point - initialize - phoneExtensibilityPoint := NewExtensibilityPoint("custom-sms-provider") + phoneExtensibilityPoint := NewExtensibilityPoint("custom-sms-sender") hookConfiguration, err := models.FindHookByExtensibilityPoint(tx, phoneExtensibilityPoint.Name) if err != nil && !models.IsNotFoundError(err) { return "", err From 7dabdcf66290ab9ff10bec1db4346adce2cd9b4d Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 27 Sep 2023 15:23:35 +0800 Subject: [PATCH 15/36] refactor: add jsonschema and some validation --- internal/api/hooks.go | 61 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index f89874b813..42c30dce13 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -23,6 +23,7 @@ import ( "github.com/supabase/gotrue/internal/models" "github.com/supabase/gotrue/internal/storage" "github.com/supabase/gotrue/internal/utilities" + "github.com/xeipuuv/gojsonschema" ) type HookEvent string @@ -130,8 +131,6 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { start := time.Now() rsp, err := client.Do(req) - fmt.Println("check here") - fmt.Println(err) if err != nil { if terr, ok := err.(net.Error); ok && terr.Timeout() { // timed out - try again? @@ -279,11 +278,10 @@ func closeBody(rsp *http.Response) { // TODO: Add an additional metadata param func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { - // TODO: filterJSONPayload(user, metadata) - payload := struct { - User *models.User `json:"user"` - }{ - User: user, + // TODO: Modify this so it takes in metadata and geeneralizes or we pass through hook + inp, err := TransformInput(user, hookConfig) + if err != nil { + return err } // TODO: substitute with a custom Claims intrface @@ -291,7 +289,7 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m "IssuedAt": time.Now().Unix(), "Subject": uuid.Nil.String(), "Issuer": authHookIssuer, - "Events": payload, + "Events": inp, } a := AuthHook{ @@ -459,3 +457,50 @@ type connectionWatcher struct { func (c *connectionWatcher) GotConn(_ httptrace.GotConnInfo) { c.gotConn = true } + +// TODO: should take in metadata as well +func TransformInput(user *models.User, hookConfig models.HookConfig) (map[string]interface{}, error) { + // Create an empty map to store the result + result := make(map[string]interface{}) + + // Check if the user is not nil and has a phone number + if user != nil && user.Phone != "" { + // Add the phone number to the result map + result["phone"] = string(user.Phone) + } + // No switch statement for now but based on the type we can decide what to check + requestSchema := hookConfig.RequestSchema + requestJSON, err := json.Marshal(requestSchema) + if err != nil { + return nil, err + } + + finalJSON, err := json.Marshal(requestJSON) + if err != nil { + return nil, err + } + + schemaLoader := gojsonschema.NewStringLoader(string(finalJSON)) + + jsonData, err := json.Marshal(result) + if err != nil { + return nil, err + } + jsonLoader := gojsonschema.NewStringLoader(string(jsonData)) + + validationResult, err := gojsonschema.Validate(schemaLoader, jsonLoader) + if err != nil { + fmt.Printf("Error loading JSON data: %s\n", err.Error()) + return nil, err + } + if validationResult.Valid() { + fmt.Println("JSON data is valid against the schema.") + } else { + fmt.Println("JSON data is not valid against the schema.") + for _, desc := range validationResult.Errors() { + fmt.Printf("- %s\n", desc) + } + } + + return result, nil +} From 0077a2ec1856e5ed19560d7e9a499999d8d7bd98 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 27 Sep 2023 16:26:34 +0800 Subject: [PATCH 16/36] refactor: add validation --- internal/api/hooks.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 42c30dce13..8ecc69d8ed 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -475,12 +475,7 @@ func TransformInput(user *models.User, hookConfig models.HookConfig) (map[string return nil, err } - finalJSON, err := json.Marshal(requestJSON) - if err != nil { - return nil, err - } - - schemaLoader := gojsonschema.NewStringLoader(string(finalJSON)) + schemaLoader := gojsonschema.NewStringLoader(string(requestJSON)) jsonData, err := json.Marshal(result) if err != nil { From 480b8305099c98261e28f89ad7ffdbe1e066888c Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 27 Sep 2023 22:44:03 +0800 Subject: [PATCH 17/36] refactor: add response validation --- internal/api/hooks.go | 71 ++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 8ecc69d8ed..dbeeb8be89 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -74,13 +74,21 @@ type WebhookResponse struct { UserMetaData map[string]interface{} `json:"user_metadata,omitempty"` } +// TODO: this should eventually go into one large file or we vendor a librar ywhich can generate the struct that we wish to marshal into. Or we can use CLI and maintain copy somewhere +type CustomSmsHookResponse struct { + Status int `json:"status"` + Message string `json:"message"` + Code string `json:"code"` + MoreInfo string `json:"more_info"` + Data interface{} `json:"data,omitempty"` +} + // Duplicate of Webhook, should eventually modify the fields passed type AuthHook struct { *conf.WebhookConfig // Decide what should go here jwtSecret string claims jwt.Claims - payload []byte } func (w *AuthHook) trigger() (io.ReadCloser, error) { @@ -160,6 +168,8 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { if rsp.ContentLength > 0 { body = rsp.Body } + fmt.Printf("%v", rsp) + fmt.Println(body) return body, nil default: rspLog.Infof("Bad response for webhook %d in %s", rsp.StatusCode, dur) @@ -279,6 +289,7 @@ func closeBody(rsp *http.Response) { // TODO: Add an additional metadata param func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { // TODO: Modify this so it takes in metadata and geeneralizes or we pass through hook + inp, err := TransformInput(user, hookConfig) if err != nil { return err @@ -308,12 +319,13 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m // TODO: this should return webhook response and we should modify the method signature if err == nil && body != nil { - // TODO: Fetch the output from hook config and then validate against it - webhookRsp := &WebhookResponse{} - decoder := json.NewDecoder(body) - if err = decoder.Decode(webhookRsp); err != nil { - return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + + resp, err := DecodeAndValidateResponse(hookConfig.ResponseSchema, body) + if err != nil { + return err } + // TODO: modify function so that it returns the response. In this case it's not needed + fmt.Println(resp) } if err != nil { return err @@ -462,40 +474,65 @@ func (c *connectionWatcher) GotConn(_ httptrace.GotConnInfo) { func TransformInput(user *models.User, hookConfig models.HookConfig) (map[string]interface{}, error) { // Create an empty map to store the result result := make(map[string]interface{}) - // Check if the user is not nil and has a phone number if user != nil && user.Phone != "" { // Add the phone number to the result map result["phone"] = string(user.Phone) } + // No switch statement for now but based on the type we can decide what to check - requestSchema := hookConfig.RequestSchema - requestJSON, err := json.Marshal(requestSchema) + jsonData, err := json.Marshal(result) if err != nil { return nil, err } - schemaLoader := gojsonschema.NewStringLoader(string(requestJSON)) + if err := validateSchema(hookConfig.RequestSchema, string(jsonData)); err != nil { + return nil, err + } - jsonData, err := json.Marshal(result) + return result, nil +} + +func DecodeAndValidateResponse(outputSchema map[string]interface{}, resp io.ReadCloser) (output map[string]interface{}, err error) { + // TODO: Fetch the output from hook config and then validate against it + // Switch based on different response types + // TODO: Move this into separate file for validation + customSmsResponse := &CustomSmsHookResponse{} + decoder := json.NewDecoder(resp) + if err = decoder.Decode(customSmsResponse); err != nil { + return nil, internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + } + jsonData, err := json.Marshal(customSmsResponse) if err != nil { return nil, err } - jsonLoader := gojsonschema.NewStringLoader(string(jsonData)) + if validationErr := validateSchema(outputSchema, string(jsonData)); validationErr != nil { + return nil, validationErr + } + // Validate Response against schema + return nil, nil +} +func validateSchema(schema map[string]interface{}, jsonDataAsString string) error { + jsonLoader := gojsonschema.NewStringLoader(jsonDataAsString) + requestJSON, err := json.Marshal(schema) + if err != nil { + return err + } + + schemaLoader := gojsonschema.NewStringLoader(string(requestJSON)) validationResult, err := gojsonschema.Validate(schemaLoader, jsonLoader) if err != nil { fmt.Printf("Error loading JSON data: %s\n", err.Error()) - return nil, err + return err } if validationResult.Valid() { - fmt.Println("JSON data is valid against the schema.") + return nil } else { - fmt.Println("JSON data is not valid against the schema.") + return errors.New("JSON data is not valid against the schema.") for _, desc := range validationResult.Errors() { fmt.Printf("- %s\n", desc) } } - - return result, nil + return nil } From 52a6eb6c88e6dcf39bb76d64d5edede211160ffe Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 27 Sep 2023 22:52:35 +0800 Subject: [PATCH 18/36] refactor: move inputs and outputs to separate file --- internal/api/hook_inputs.go | 6 ++++++ internal/api/hook_outputs.go | 10 ++++++++++ internal/api/hooks.go | 12 +----------- 3 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 internal/api/hook_inputs.go create mode 100644 internal/api/hook_outputs.go diff --git a/internal/api/hook_inputs.go b/internal/api/hook_inputs.go new file mode 100644 index 0000000000..8c9e780c51 --- /dev/null +++ b/internal/api/hook_inputs.go @@ -0,0 +1,6 @@ +package api + +// TODO: Get staticcheck to ignore this +// type CustomSMSSenderRequest struct { +// phone string +// } diff --git a/internal/api/hook_outputs.go b/internal/api/hook_outputs.go new file mode 100644 index 0000000000..fe55ea7b7d --- /dev/null +++ b/internal/api/hook_outputs.go @@ -0,0 +1,10 @@ +package api + +// TODO: Document how to generate the jsonschema to insert into the DB from this and/or add a make command which quickly does this +type CustomSmsHookResponse struct { + Status int `json:"status"` + Message string `json:"message"` + Code string `json:"code"` + MoreInfo string `json:"more_info"` + Data interface{} `json:"data,omitempty"` +} diff --git a/internal/api/hooks.go b/internal/api/hooks.go index dbeeb8be89..26dce12439 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -74,15 +74,6 @@ type WebhookResponse struct { UserMetaData map[string]interface{} `json:"user_metadata,omitempty"` } -// TODO: this should eventually go into one large file or we vendor a librar ywhich can generate the struct that we wish to marshal into. Or we can use CLI and maintain copy somewhere -type CustomSmsHookResponse struct { - Status int `json:"status"` - Message string `json:"message"` - Code string `json:"code"` - MoreInfo string `json:"more_info"` - Data interface{} `json:"data,omitempty"` -} - // Duplicate of Webhook, should eventually modify the fields passed type AuthHook struct { *conf.WebhookConfig @@ -529,10 +520,9 @@ func validateSchema(schema map[string]interface{}, jsonDataAsString string) erro if validationResult.Valid() { return nil } else { - return errors.New("JSON data is not valid against the schema.") for _, desc := range validationResult.Errors() { fmt.Printf("- %s\n", desc) } + return errors.New("JSON data is not valid against the schema.") } - return nil } From 64a3af35f5f9c09b2b2f12d54e52228befcb4284 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 4 Oct 2023 14:40:04 +0800 Subject: [PATCH 19/36] refactor: patch some of the TODOs --- internal/api/hooks.go | 27 ++++++++++----------------- internal/api/phone.go | 5 ++++- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 26dce12439..f3f2d39461 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -277,16 +277,13 @@ func closeBody(rsp *http.Response) { } } -// TODO: Add an additional metadata param -func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration) error { - // TODO: Modify this so it takes in metadata and geeneralizes or we pass through hook - - inp, err := TransformInput(user, hookConfig) +func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig models.HookConfig, user *models.User, config *conf.GlobalConfiguration, metadata map[string]interface{}) (map[string]interface{}, error) { + inp, err := EncodeAndValidateInput(user, hookConfig, metadata) if err != nil { - return err + return nil, err } - // TODO: substitute with a custom Claims intrface + // TODO: substitute with a custom claims interface claims := jwt.MapClaims{ "IssuedAt": time.Now().Unix(), "Subject": uuid.Nil.String(), @@ -308,20 +305,17 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m defer utilities.SafeClose(body) } - // TODO: this should return webhook response and we should modify the method signature if err == nil && body != nil { - resp, err := DecodeAndValidateResponse(hookConfig.ResponseSchema, body) if err != nil { - return err + return resp, err } - // TODO: modify function so that it returns the response. In this case it's not needed - fmt.Println(resp) + return resp, nil } if err != nil { - return err + return nil, err } - return nil + return nil, err } // Deprecate this @@ -462,7 +456,7 @@ func (c *connectionWatcher) GotConn(_ httptrace.GotConnInfo) { } // TODO: should take in metadata as well -func TransformInput(user *models.User, hookConfig models.HookConfig) (map[string]interface{}, error) { +func EncodeAndValidateInput(user *models.User, hookConfig models.HookConfig, metadata map[string]interface{}) (map[string]interface{}, error) { // Create an empty map to store the result result := make(map[string]interface{}) // Check if the user is not nil and has a phone number @@ -485,9 +479,8 @@ func TransformInput(user *models.User, hookConfig models.HookConfig) (map[string } func DecodeAndValidateResponse(outputSchema map[string]interface{}, resp io.ReadCloser) (output map[string]interface{}, err error) { - // TODO: Fetch the output from hook config and then validate against it // Switch based on different response types - // TODO: Move this into separate file for validation + // TODO: Fetch the output from hook config and then validate against it customSmsResponse := &CustomSmsHookResponse{} decoder := json.NewDecoder(resp) if err = decoder.Decode(customSmsResponse); err != nil { diff --git a/internal/api/phone.go b/internal/api/phone.go index 7b1ce45d8e..a7ca1ec48d 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -96,7 +96,10 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, } if hookConfiguration != nil { // TODO: find a way to wrap and properly Fetch the resp - if terr := triggerAuthHook(ctx, tx, *hookConfiguration, user, config); terr != nil { + // TODO: Find a way to properly pass in interface + // TODO: Change and make use of the _ + metadata := make(map[string]interface{}) + if _, terr := triggerAuthHook(ctx, tx, *hookConfiguration, user, config, metadata); terr != nil { return "", terr } } else if models.IsNotFoundError(err) { From 9a95a577e5e254fb839164df3e59764051a0b11c Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 4 Oct 2023 15:37:38 +0800 Subject: [PATCH 20/36] refactor: patch some typing issues --- internal/api/hooks.go | 50 +++++++++++++++++++++---------------------- internal/api/phone.go | 3 +-- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index f3f2d39461..bb5a84ee75 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -28,16 +28,6 @@ import ( type HookEvent string -type ExtensibilityPoint struct { - Name string -} - -func NewExtensibilityPoint(name string) *ExtensibilityPoint { - return &ExtensibilityPoint{ - Name: name, - } -} - const ( headerHookSignature = "x-webhook-signature" defaultHookRetries = 3 @@ -50,8 +40,9 @@ const ( LoginEvent = "login" ) +// ExtensibilityPoints const ( - PhoneProviderExtensibilityPoint = "phone-provider" + CustomSMSExtensibilityPoint = "custom-sms-sender" ) var defaultTimeout = time.Second * 5 @@ -455,7 +446,6 @@ func (c *connectionWatcher) GotConn(_ httptrace.GotConnInfo) { c.gotConn = true } -// TODO: should take in metadata as well func EncodeAndValidateInput(user *models.User, hookConfig models.HookConfig, metadata map[string]interface{}) (map[string]interface{}, error) { // Create an empty map to store the result result := make(map[string]interface{}) @@ -478,23 +468,33 @@ func EncodeAndValidateInput(user *models.User, hookConfig models.HookConfig, met return result, nil } -func DecodeAndValidateResponse(outputSchema map[string]interface{}, resp io.ReadCloser) (output map[string]interface{}, err error) { - // Switch based on different response types - // TODO: Fetch the output from hook config and then validate against it - customSmsResponse := &CustomSmsHookResponse{} - decoder := json.NewDecoder(resp) - if err = decoder.Decode(customSmsResponse); err != nil { +func DecodeAndValidateResponse(hookConfig models.HookConfig, resp io.ReadCloser) (output interface{}, err error) { + var jsonData []byte + var decodedResponse interface{} + switch hookConfig.ExtensibilityPoint { + // Repeat for all possible Hook types + case CustomSMSExtensibilityPoint: + var outputs CustomSmsHookResponse + decoder := json.NewDecoder(resp) + if err = decoder.Decode(outputs); err != nil { + // TODO: Refactor this into a single error somewhere + return nil, internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + } + decodedResponse = outputs + + default: return nil, internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) } - jsonData, err := json.Marshal(customSmsResponse) - if err != nil { - return nil, err - } - if validationErr := validateSchema(outputSchema, string(jsonData)); validationErr != nil { + + if validationErr := validateSchema(hookConfig.ResponseSchema, string(jsonData)); validationErr != nil { return nil, validationErr } - // Validate Response against schema - return nil, nil + + jsonData, err = json.Marshal(decodedResponse) + if err != nil { + return nil, internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + } + return decodedResponse, nil } func validateSchema(schema map[string]interface{}, jsonDataAsString string) error { diff --git a/internal/api/phone.go b/internal/api/phone.go index a7ca1ec48d..1bb922a0d6 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -89,8 +89,7 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, } // Extensibility Point - initialize - phoneExtensibilityPoint := NewExtensibilityPoint("custom-sms-sender") - hookConfiguration, err := models.FindHookByExtensibilityPoint(tx, phoneExtensibilityPoint.Name) + hookConfiguration, err := models.FindHookByExtensibilityPoint(tx, CustomSMSExtensibilityPoint) if err != nil && !models.IsNotFoundError(err) { return "", err } From 737a71ce920edd01826d983f0d25631da8b6e2de Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 4 Oct 2023 16:16:34 +0800 Subject: [PATCH 21/36] fix: cast response --- internal/api/hooks.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index bb5a84ee75..704e718af5 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -297,11 +297,12 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m } if err == nil && body != nil { - resp, err := DecodeAndValidateResponse(hookConfig.ResponseSchema, body) + resp, err := DecodeAndValidateResponse(hookConfig, body) if err != nil { - return resp, err + // TODO: Figure out if there's a way to not lose typing here + return resp.(map[string]interface{}), err } - return resp, nil + return resp.(map[string]interface{}), nil } if err != nil { return nil, err From c129103f868607522f5aeea7f4e1f67de20b3c37 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 4 Oct 2023 16:34:41 +0800 Subject: [PATCH 22/36] refactor: add validation for encode --- internal/api/hook_inputs.go | 14 +++++++++++ internal/api/hooks.go | 25 +++++++++++-------- internal/models/hooks.go | 16 ++++++------ .../20230913081932_add_hook_config.up.sql | 2 +- 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/internal/api/hook_inputs.go b/internal/api/hook_inputs.go index 8c9e780c51..b5b6d058f6 100644 --- a/internal/api/hook_inputs.go +++ b/internal/api/hook_inputs.go @@ -1,6 +1,20 @@ package api +import "github.com/supabase/gotrue/internal/models" + // TODO: Get staticcheck to ignore this // type CustomSMSSenderRequest struct { // phone string // } + +func TransformCustomSMSExtensibilityPointInputs(user *models.User, metadata map[string]interface{}) (request interface{}, err error) { + // Check if the user is not nil and has a phone number + result := make(map[string]interface{}) + + if user != nil && user.Phone != "" { + // Add the phone number to the result map + result["phone"] = string(user.Phone) + } + return result, nil + +} diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 704e718af5..377aa158e4 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -447,17 +447,22 @@ func (c *connectionWatcher) GotConn(_ httptrace.GotConnInfo) { c.gotConn = true } -func EncodeAndValidateInput(user *models.User, hookConfig models.HookConfig, metadata map[string]interface{}) (map[string]interface{}, error) { +func EncodeAndValidateInput(user *models.User, hookConfig models.HookConfig, metadata map[string]interface{}) (interface{}, error) { // Create an empty map to store the result - result := make(map[string]interface{}) - // Check if the user is not nil and has a phone number - if user != nil && user.Phone != "" { - // Add the phone number to the result map - result["phone"] = string(user.Phone) + var request interface{} + var err error + switch hookConfig.ExtensibilityPoint { + case CustomSMSExtensibilityPoint: + request, err = TransformCustomSMSExtensibilityPointInputs(user, metadata) + default: + return nil, internalServerError("failed to encode webhook").WithInternalError(err) + } + if err != nil { + return nil, internalServerError("failed to encode webhook").WithInternalError(err) } // No switch statement for now but based on the type we can decide what to check - jsonData, err := json.Marshal(result) + jsonData, err := json.Marshal(request) if err != nil { return nil, err } @@ -466,7 +471,7 @@ func EncodeAndValidateInput(user *models.User, hookConfig models.HookConfig, met return nil, err } - return result, nil + return jsonData, nil } func DecodeAndValidateResponse(hookConfig models.HookConfig, resp io.ReadCloser) (output interface{}, err error) { @@ -475,7 +480,7 @@ func DecodeAndValidateResponse(hookConfig models.HookConfig, resp io.ReadCloser) switch hookConfig.ExtensibilityPoint { // Repeat for all possible Hook types case CustomSMSExtensibilityPoint: - var outputs CustomSmsHookResponse + var outputs *CustomSmsHookResponse decoder := json.NewDecoder(resp) if err = decoder.Decode(outputs); err != nil { // TODO: Refactor this into a single error somewhere @@ -495,7 +500,7 @@ func DecodeAndValidateResponse(hookConfig models.HookConfig, resp io.ReadCloser) if err != nil { return nil, internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) } - return decodedResponse, nil + return jsonData, nil } func validateSchema(schema map[string]interface{}, jsonDataAsString string) error { diff --git a/internal/models/hooks.go b/internal/models/hooks.go index 24d4cfb6d3..e0a6c317aa 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -2,19 +2,21 @@ package models import ( "database/sql" + "github.com/gobuffalo/pop/v6" + "github.com/gofrs/uuid" "github.com/pkg/errors" "github.com/supabase/gotrue/internal/storage" ) type HookConfig struct { - Name string `json:"name" db:"name"` - URI string `json:"uri" db:"uri"` - Secret string `json:"secret" db:"secret"` - ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` - RequestSchema JSONMap `json:"request_schema" db:"request_schema"` - ResponseSchema JSONMap `json:"response_schema" db:"response_schema"` - Metadata JSONMap `json:"metadata" db:"metadata"` + ID uuid.UUID `json:"id" db:"id"` + URI string `json:"uri" db:"uri"` + Secret string `json:"secret" db:"secret"` + ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` + RequestSchema JSONMap `json:"request_schema" db:"request_schema"` + ResponseSchema JSONMap `json:"response_schema" db:"response_schema"` + Metadata JSONMap `json:"metadata" db:"metadata"` } // TableName overrides the table name used by pop diff --git a/migrations/20230913081932_add_hook_config.up.sql b/migrations/20230913081932_add_hook_config.up.sql index dabbea93d1..486a89127b 100644 --- a/migrations/20230913081932_add_hook_config.up.sql +++ b/migrations/20230913081932_add_hook_config.up.sql @@ -1,7 +1,7 @@ -- auth.hooks definition create table if not exists {{ index .Options "Namespace" }}.hook_config( - name text null, + id uuid not null, uri text not null, secret text not null, extensibility_point text not null, From 9e0702691304ec090edf80597c0365b94e6ce76a Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Fri, 6 Oct 2023 17:53:22 +0800 Subject: [PATCH 23/36] fix: continue to exclude structs --- internal/api/hook_inputs.go | 9 +++++---- internal/api/hooks.go | 2 -- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/api/hook_inputs.go b/internal/api/hook_inputs.go index b5b6d058f6..7f943fbe33 100644 --- a/internal/api/hook_inputs.go +++ b/internal/api/hook_inputs.go @@ -2,10 +2,11 @@ package api import "github.com/supabase/gotrue/internal/models" -// TODO: Get staticcheck to ignore this -// type CustomSMSSenderRequest struct { -// phone string -// } +// TODO: Find a way to exclude all structs in this file from checks +type CustomSMSSenderRequest struct { + //lint:ignore U1000 This struct's fields are intentionally unused. They are used for generation of jsonschema which is stored in the database. + phone string +} func TransformCustomSMSExtensibilityPointInputs(user *models.User, metadata map[string]interface{}) (request interface{}, err error) { // Check if the user is not nil and has a phone number diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 377aa158e4..c6f33bdf1b 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -448,7 +448,6 @@ func (c *connectionWatcher) GotConn(_ httptrace.GotConnInfo) { } func EncodeAndValidateInput(user *models.User, hookConfig models.HookConfig, metadata map[string]interface{}) (interface{}, error) { - // Create an empty map to store the result var request interface{} var err error switch hookConfig.ExtensibilityPoint { @@ -461,7 +460,6 @@ func EncodeAndValidateInput(user *models.User, hookConfig models.HookConfig, met return nil, internalServerError("failed to encode webhook").WithInternalError(err) } - // No switch statement for now but based on the type we can decide what to check jsonData, err := json.Marshal(request) if err != nil { return nil, err From 4152025e730fe245bf55960089f444ef87b2c3bf Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Mon, 9 Oct 2023 15:45:10 +0800 Subject: [PATCH 24/36] refactor: move transforms to separate file --- internal/api/hook_inputs.go | 14 -------------- internal/api/hook_transforms.go | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 14 deletions(-) create mode 100644 internal/api/hook_transforms.go diff --git a/internal/api/hook_inputs.go b/internal/api/hook_inputs.go index 7f943fbe33..c2d9135f79 100644 --- a/internal/api/hook_inputs.go +++ b/internal/api/hook_inputs.go @@ -1,21 +1,7 @@ package api -import "github.com/supabase/gotrue/internal/models" - // TODO: Find a way to exclude all structs in this file from checks type CustomSMSSenderRequest struct { //lint:ignore U1000 This struct's fields are intentionally unused. They are used for generation of jsonschema which is stored in the database. phone string } - -func TransformCustomSMSExtensibilityPointInputs(user *models.User, metadata map[string]interface{}) (request interface{}, err error) { - // Check if the user is not nil and has a phone number - result := make(map[string]interface{}) - - if user != nil && user.Phone != "" { - // Add the phone number to the result map - result["phone"] = string(user.Phone) - } - return result, nil - -} diff --git a/internal/api/hook_transforms.go b/internal/api/hook_transforms.go new file mode 100644 index 0000000000..0e58b34066 --- /dev/null +++ b/internal/api/hook_transforms.go @@ -0,0 +1,15 @@ +package api + +import "github.com/supabase/gotrue/internal/models" + +func TransformCustomSMSExtensibilityPointInputs(user *models.User, metadata map[string]interface{}) (request interface{}, err error) { + // Check if the user is not nil and has a phone number + result := make(map[string]interface{}) + + if user != nil && user.Phone != "" { + // Add the phone number to the result map + result["phone"] = string(user.Phone) + } + return result, nil + +} From e4a4b13151729f94ae793a7988061f71f0b760aa Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Mon, 9 Oct 2023 16:05:29 +0800 Subject: [PATCH 25/36] refacotr: remove dated request.json --- internal/hooks/custom_sms_request.json | 30 ++++++++++++++++++++ internal/hooks/custom_sms_response.json | 37 +++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 internal/hooks/custom_sms_request.json create mode 100644 internal/hooks/custom_sms_response.json diff --git a/internal/hooks/custom_sms_request.json b/internal/hooks/custom_sms_request.json new file mode 100644 index 0000000000..a04a39ebee --- /dev/null +++ b/internal/hooks/custom_sms_request.json @@ -0,0 +1,30 @@ +{ + "description": "CustomSMSRequest", + "type": "object", + "properties": { + "api_version": { + "description": "version of the api", + "type": "string" + }, + "user": { + "type": "object", + "properties": { + "phone": { + "type": "string", + "description": "Store when a phone has been confirmed. Null if unconfirmed." + }, + "app_metadata": { + "type": "object" + }, + "confirmed_at": { + "type": "string", + "description": "Stores user attributes which do not impact core functionality" + } + }, + "required": ["phone", "app_metadata", "confirmed_at"] + } + }, + "required": ["api_version", "user"], + "additionalProperties": false +} + diff --git a/internal/hooks/custom_sms_response.json b/internal/hooks/custom_sms_response.json new file mode 100644 index 0000000000..55f467b6bf --- /dev/null +++ b/internal/hooks/custom_sms_response.json @@ -0,0 +1,37 @@ +{ + "type": "object", + "title": "CustomSMSProviderResponse", + "description": "Custom SMS provider Request", + "properties": { + "status": { + "description": "HTTP Status code (e.g. 400)", + "type": "number" + }, + "message": { + "description": "Short Description of the error message", + "type": "string" + }, + "code": { + "description": "Internal Error code reference", + "type": "string" + }, + "more_info": { + "description": "Detailed elaboration and possibly link to an error page in the future.", + "type": "string" + }, + "data": { + "description": "Response data returned by the hook" + }, + "api_version": { + "description": "API Version", + "type": "string" + }, + "user": { + "description": "User information", + "type": "object" + } + }, + "required": ["api_version", "user"], + "additionalProperties": false +} + From f02fd39b68ccf5f70cb507fcbde3ef917896ff6a Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Mon, 9 Oct 2023 17:38:05 +0800 Subject: [PATCH 26/36] refactor: use generated inputs instead --- internal/api/hook_inputs.go | 199 ++++++++++++++++++++++++- internal/api/hook_outputs.go | 189 ++++++++++++++++++++++- internal/hooks/custom_sms_request.json | 2 +- 3 files changed, 378 insertions(+), 12 deletions(-) diff --git a/internal/api/hook_inputs.go b/internal/api/hook_inputs.go index c2d9135f79..86a04f19ae 100644 --- a/internal/api/hook_inputs.go +++ b/internal/api/hook_inputs.go @@ -1,7 +1,198 @@ package api -// TODO: Find a way to exclude all structs in this file from checks -type CustomSMSSenderRequest struct { - //lint:ignore U1000 This struct's fields are intentionally unused. They are used for generation of jsonschema which is stored in the database. - phone string +import ( + "bytes" + "errors" + "encoding/json" + "fmt" +) + +// AppMetadata +type AppMetadata struct { +} + +// CustomSMSRequest +type CustomSMSRequest struct { + + // version of the api + ApiVersion string `json:"api_version"` + UserData *UserData `json:"user"` +} + +// UserData +type UserData struct { + AppMetadata *AppMetadata `json:"app_metadata"` + + // Stores user attributes which do not impact core functionality + ConfirmedAt string `json:"confirmed_at"` + + // Store when a phone has been confirmed. Null if unconfirmed. + Phone string `json:"phone"` +} + +func (strct *CustomSMSRequest) MarshalJSON() ([]byte, error) { + buf := bytes.NewBuffer(make([]byte, 0)) + buf.WriteString("{") + comma := false + // "ApiVersion" field is required + // only required object types supported for marshal checking (for now) + // Marshal the "api_version" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"api_version\": ") + if tmp, err := json.Marshal(strct.ApiVersion); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // "UserData" field is required + if strct.UserData == nil { + return nil, errors.New("user is a required field") + } + // Marshal the "user" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"user\": ") + if tmp, err := json.Marshal(strct.UserData); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + + buf.WriteString("}") + rv := buf.Bytes() + return rv, nil +} + +func (strct *CustomSMSRequest) UnmarshalJSON(b []byte) error { + api_versionReceived := false + userReceived := false + var jsonMap map[string]json.RawMessage + if err := json.Unmarshal(b, &jsonMap); err != nil { + return err + } + // parse all the defined properties + for k, v := range jsonMap { + switch k { + case "api_version": + if err := json.Unmarshal([]byte(v), &strct.ApiVersion); err != nil { + return err + } + api_versionReceived = true + case "user": + if err := json.Unmarshal([]byte(v), &strct.User); err != nil { + return err + } + userReceived = true + default: + return fmt.Errorf("additional property not allowed: \"" + k + "\"") + } + } + // check if api_version (a required property) was received + if !api_versionReceived { + return errors.New("\"api_version\" is required but was not present") + } + // check if user (a required property) was received + if !userReceived { + return errors.New("\"user\" is required but was not present") + } + return nil +} + +func (strct *UserData) MarshalJSON() ([]byte, error) { + buf := bytes.NewBuffer(make([]byte, 0)) + buf.WriteString("{") + comma := false + // "AppMetadata" field is required + if strct.AppMetadata == nil { + return nil, errors.New("app_metadata is a required field") + } + // Marshal the "app_metadata" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"app_metadata\": ") + if tmp, err := json.Marshal(strct.AppMetadata); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // "ConfirmedAt" field is required + // only required object types supported for marshal checking (for now) + // Marshal the "confirmed_at" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"confirmed_at\": ") + if tmp, err := json.Marshal(strct.ConfirmedAt); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // "Phone" field is required + // only required object types supported for marshal checking (for now) + // Marshal the "phone" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"phone\": ") + if tmp, err := json.Marshal(strct.Phone); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + + buf.WriteString("}") + rv := buf.Bytes() + return rv, nil +} + +func (strct *UserData) UnmarshalJSON(b []byte) error { + app_metadataReceived := false + confirmed_atReceived := false + phoneReceived := false + var jsonMap map[string]json.RawMessage + if err := json.Unmarshal(b, &jsonMap); err != nil { + return err + } + // parse all the defined properties + for k, v := range jsonMap { + switch k { + case "app_metadata": + if err := json.Unmarshal([]byte(v), &strct.AppMetadata); err != nil { + return err + } + app_metadataReceived = true + case "confirmed_at": + if err := json.Unmarshal([]byte(v), &strct.ConfirmedAt); err != nil { + return err + } + confirmed_atReceived = true + case "phone": + if err := json.Unmarshal([]byte(v), &strct.Phone); err != nil { + return err + } + phoneReceived = true + } + } + // check if app_metadata (a required property) was received + if !app_metadataReceived { + return errors.New("\"app_metadata\" is required but was not present") + } + // check if confirmed_at (a required property) was received + if !confirmed_atReceived { + return errors.New("\"confirmed_at\" is required but was not present") + } + // check if phone (a required property) was received + if !phoneReceived { + return errors.New("\"phone\" is required but was not present") + } + return nil } diff --git a/internal/api/hook_outputs.go b/internal/api/hook_outputs.go index fe55ea7b7d..524d498700 100644 --- a/internal/api/hook_outputs.go +++ b/internal/api/hook_outputs.go @@ -1,10 +1,185 @@ package api -// TODO: Document how to generate the jsonschema to insert into the DB from this and/or add a make command which quickly does this -type CustomSmsHookResponse struct { - Status int `json:"status"` - Message string `json:"message"` - Code string `json:"code"` - MoreInfo string `json:"more_info"` - Data interface{} `json:"data,omitempty"` +import ( + "errors" + "encoding/json" + "fmt" + "bytes" +) + +// CustomSMSProviderResponse Custom SMS provider Request +type CustomSMSProviderResponse struct { + + // API Version + ApiVersion string `json:"api_version"` + + // Internal Error code reference + Code string `json:"code,omitempty"` + + // Response data returned by the hook + Data interface{} `json:"data,omitempty"` + + // Short Description of the error message + Message string `json:"message,omitempty"` + + // Detailed elaboration and possibly link to an error page in the future. + MoreInfo string `json:"more_info,omitempty"` + + // HTTP Status code (e.g. 400) + Status float64 `json:"status,omitempty"` + + // User information + User *User `json:"user"` +} + +// User User information +type User struct { +} + +func (strct *CustomSMSProviderResponse) MarshalJSON() ([]byte, error) { + buf := bytes.NewBuffer(make([]byte, 0)) + buf.WriteString("{") + comma := false + // "ApiVersion" field is required + // only required object types supported for marshal checking (for now) + // Marshal the "api_version" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"api_version\": ") + if tmp, err := json.Marshal(strct.ApiVersion); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // Marshal the "code" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"code\": ") + if tmp, err := json.Marshal(strct.Code); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // Marshal the "data" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"data\": ") + if tmp, err := json.Marshal(strct.Data); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // Marshal the "message" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"message\": ") + if tmp, err := json.Marshal(strct.Message); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // Marshal the "more_info" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"more_info\": ") + if tmp, err := json.Marshal(strct.MoreInfo); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // Marshal the "status" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"status\": ") + if tmp, err := json.Marshal(strct.Status); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + // "User" field is required + if strct.User == nil { + return nil, errors.New("user is a required field") + } + // Marshal the "user" field + if comma { + buf.WriteString(",") + } + buf.WriteString("\"user\": ") + if tmp, err := json.Marshal(strct.User); err != nil { + return nil, err + } else { + buf.Write(tmp) + } + comma = true + + buf.WriteString("}") + rv := buf.Bytes() + return rv, nil +} + +func (strct *CustomSMSProviderResponse) UnmarshalJSON(b []byte) error { + api_versionReceived := false + userReceived := false + var jsonMap map[string]json.RawMessage + if err := json.Unmarshal(b, &jsonMap); err != nil { + return err + } + // parse all the defined properties + for k, v := range jsonMap { + switch k { + case "api_version": + if err := json.Unmarshal([]byte(v), &strct.ApiVersion); err != nil { + return err + } + api_versionReceived = true + case "code": + if err := json.Unmarshal([]byte(v), &strct.Code); err != nil { + return err + } + case "data": + if err := json.Unmarshal([]byte(v), &strct.Data); err != nil { + return err + } + case "message": + if err := json.Unmarshal([]byte(v), &strct.Message); err != nil { + return err + } + case "more_info": + if err := json.Unmarshal([]byte(v), &strct.MoreInfo); err != nil { + return err + } + case "status": + if err := json.Unmarshal([]byte(v), &strct.Status); err != nil { + return err + } + case "user": + if err := json.Unmarshal([]byte(v), &strct.User); err != nil { + return err + } + userReceived = true + default: + return fmt.Errorf("additional property not allowed: \"" + k + "\"") + } + } + // check if api_version (a required property) was received + if !api_versionReceived { + return errors.New("\"api_version\" is required but was not present") + } + // check if user (a required property) was received + if !userReceived { + return errors.New("\"user\" is required but was not present") + } + return nil } diff --git a/internal/hooks/custom_sms_request.json b/internal/hooks/custom_sms_request.json index a04a39ebee..87c04a0acc 100644 --- a/internal/hooks/custom_sms_request.json +++ b/internal/hooks/custom_sms_request.json @@ -1,5 +1,5 @@ { - "description": "CustomSMSRequest", + "title": "CustomSMSRequest", "type": "object", "properties": { "api_version": { From f7fb2375e6fa523cbb73d97fa98b45de368746c5 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Mon, 9 Oct 2023 17:41:57 +0800 Subject: [PATCH 27/36] refactor: generate instead of manually typing --- internal/api/hook_inputs.go | 2 +- internal/api/hook_outputs.go | 12 ++++++------ internal/api/hooks.go | 2 +- internal/hooks/custom_sms_request.json | 2 +- internal/hooks/custom_sms_response.json | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/api/hook_inputs.go b/internal/api/hook_inputs.go index 86a04f19ae..91f94879bf 100644 --- a/internal/api/hook_inputs.go +++ b/internal/api/hook_inputs.go @@ -84,7 +84,7 @@ func (strct *CustomSMSRequest) UnmarshalJSON(b []byte) error { } api_versionReceived = true case "user": - if err := json.Unmarshal([]byte(v), &strct.User); err != nil { + if err := json.Unmarshal([]byte(v), &strct.UserData); err != nil { return err } userReceived = true diff --git a/internal/api/hook_outputs.go b/internal/api/hook_outputs.go index 524d498700..c2faae504d 100644 --- a/internal/api/hook_outputs.go +++ b/internal/api/hook_outputs.go @@ -1,14 +1,14 @@ package api import ( - "errors" "encoding/json" "fmt" "bytes" + "errors" ) -// CustomSMSProviderResponse Custom SMS provider Request -type CustomSMSProviderResponse struct { +// CustomSMSHookResponse Custom SMS provider Request +type CustomSMSHookResponse struct { // API Version ApiVersion string `json:"api_version"` @@ -36,7 +36,7 @@ type CustomSMSProviderResponse struct { type User struct { } -func (strct *CustomSMSProviderResponse) MarshalJSON() ([]byte, error) { +func (strct *CustomSMSHookResponse) MarshalJSON() ([]byte, error) { buf := bytes.NewBuffer(make([]byte, 0)) buf.WriteString("{") comma := false @@ -85,7 +85,7 @@ func (strct *CustomSMSProviderResponse) MarshalJSON() ([]byte, error) { } else { buf.Write(tmp) } - comma = true + comma = true // Marshal the "more_info" field if comma { buf.WriteString(",") @@ -129,7 +129,7 @@ func (strct *CustomSMSProviderResponse) MarshalJSON() ([]byte, error) { return rv, nil } -func (strct *CustomSMSProviderResponse) UnmarshalJSON(b []byte) error { +func (strct *CustomSMSHookResponse) UnmarshalJSON(b []byte) error { api_versionReceived := false userReceived := false var jsonMap map[string]json.RawMessage diff --git a/internal/api/hooks.go b/internal/api/hooks.go index c6f33bdf1b..6f3c30574b 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -478,7 +478,7 @@ func DecodeAndValidateResponse(hookConfig models.HookConfig, resp io.ReadCloser) switch hookConfig.ExtensibilityPoint { // Repeat for all possible Hook types case CustomSMSExtensibilityPoint: - var outputs *CustomSmsHookResponse + var outputs *CustomSMSHookResponse decoder := json.NewDecoder(resp) if err = decoder.Decode(outputs); err != nil { // TODO: Refactor this into a single error somewhere diff --git a/internal/hooks/custom_sms_request.json b/internal/hooks/custom_sms_request.json index 87c04a0acc..e554adb475 100644 --- a/internal/hooks/custom_sms_request.json +++ b/internal/hooks/custom_sms_request.json @@ -6,7 +6,7 @@ "description": "version of the api", "type": "string" }, - "user": { + "userg": { "type": "object", "properties": { "phone": { diff --git a/internal/hooks/custom_sms_response.json b/internal/hooks/custom_sms_response.json index 55f467b6bf..4c6c169d29 100644 --- a/internal/hooks/custom_sms_response.json +++ b/internal/hooks/custom_sms_response.json @@ -1,6 +1,6 @@ { "type": "object", - "title": "CustomSMSProviderResponse", + "title": "CustomSMSHookResponse", "description": "Custom SMS provider Request", "properties": { "status": { From b5b52772631ccc1d5d3c524ff564ce421614a46d Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Tue, 10 Oct 2023 10:00:41 +0800 Subject: [PATCH 28/36] refactor: add webhookResponseError --- internal/api/errors.go | 3 +++ internal/api/hooks.go | 9 ++++----- internal/api/phone.go | 3 +-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/api/errors.go b/internal/api/errors.go index 8efdbe811e..474a1c0093 100644 --- a/internal/api/errors.go +++ b/internal/api/errors.go @@ -123,6 +123,9 @@ func tooManyRequestsError(fmtString string, args ...interface{}) *HTTPError { func conflictError(fmtString string, args ...interface{}) *HTTPError { return httpError(http.StatusConflict, fmtString, args...) } +func webhookResponseError(fmtString string, args ...interface{}) *HTTPError { + return internalServerError(fmt.Sprintf("Webhook returned malformed JSON: %v"), args) +} // HTTPError is an error with a message and an HTTP status code. type HTTPError struct { diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 6f3c30574b..2ea264ff87 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -397,7 +397,7 @@ func triggerHook(ctx context.Context, hookURL *url.URL, secret string, conn *sto webhookRsp := &WebhookResponse{} decoder := json.NewDecoder(body) if err = decoder.Decode(webhookRsp); err != nil { - return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + return webhookResponseError(err.Error()).WithInternalError(err) } return conn.Transaction(func(tx *storage.Connection) error { @@ -481,13 +481,12 @@ func DecodeAndValidateResponse(hookConfig models.HookConfig, resp io.ReadCloser) var outputs *CustomSMSHookResponse decoder := json.NewDecoder(resp) if err = decoder.Decode(outputs); err != nil { - // TODO: Refactor this into a single error somewhere - return nil, internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + return nil, webhookResponseError(err.Error()).WithInternalError(err) } decodedResponse = outputs default: - return nil, internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + return nil, webhookResponseError(err.Error()).WithInternalError(err) } if validationErr := validateSchema(hookConfig.ResponseSchema, string(jsonData)); validationErr != nil { @@ -496,7 +495,7 @@ func DecodeAndValidateResponse(hookConfig models.HookConfig, resp io.ReadCloser) jsonData, err = json.Marshal(decodedResponse) if err != nil { - return nil, internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err) + return nil, webhookResponseError(err.Error()).WithInternalError(err) } return jsonData, nil } diff --git a/internal/api/phone.go b/internal/api/phone.go index 1bb922a0d6..6b6622e4b4 100644 --- a/internal/api/phone.go +++ b/internal/api/phone.go @@ -95,9 +95,8 @@ func (a *API) sendPhoneConfirmation(ctx context.Context, tx *storage.Connection, } if hookConfiguration != nil { // TODO: find a way to wrap and properly Fetch the resp - // TODO: Find a way to properly pass in interface - // TODO: Change and make use of the _ metadata := make(map[string]interface{}) + // Response not needed for SMS Hook if _, terr := triggerAuthHook(ctx, tx, *hookConfiguration, user, config, metadata); terr != nil { return "", terr } From 089c509ff1f6827935139134aa17c7dcd1efae8d Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Tue, 10 Oct 2023 21:51:41 +0800 Subject: [PATCH 29/36] refactor: update transforms --- internal/api/hook_transforms.go | 8 +++++++- internal/api/hooks.go | 2 +- internal/hooks/README.md | 8 ++++++++ internal/hooks/custom_sms_request.json | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 internal/hooks/README.md diff --git a/internal/api/hook_transforms.go b/internal/api/hook_transforms.go index 0e58b34066..0e23383878 100644 --- a/internal/api/hook_transforms.go +++ b/internal/api/hook_transforms.go @@ -5,10 +5,16 @@ import "github.com/supabase/gotrue/internal/models" func TransformCustomSMSExtensibilityPointInputs(user *models.User, metadata map[string]interface{}) (request interface{}, err error) { // Check if the user is not nil and has a phone number result := make(map[string]interface{}) + userMap := make(map[string]interface{}) + if user != nil && user.Phone != "" { // Add the phone number to the result map - result["phone"] = string(user.Phone) + result["user"] = userMap + result["api_version"] = "1.0" + userMap["app_metadata"] = make(map[string]interface{}) + userMap["confirmed_at"] = "" + userMap["phone"] = string(user.Phone) } return result, nil diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 6d97063402..55b606ddae 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -42,7 +42,7 @@ const ( // ExtensibilityPoints const ( - CustomSMSExtensibilityPoint = "custom-sms-sender" + CustomSMSExtensibilityPoint = "custom-sms-provider" ) var defaultTimeout = time.Second * 5 diff --git a/internal/hooks/README.md b/internal/hooks/README.md new file mode 100644 index 0000000000..a0d02370a2 --- /dev/null +++ b/internal/hooks/README.md @@ -0,0 +1,8 @@ +## Overview + +This directory contains the JSONSchema's of each extensibility point. To introduce a new extensibility point, please do the following: + +1. File a Pull Request with the `_request.json` and `_response.json` +2. Run `schema-generate _request.json` and `schema-generate _response.json` and place the `hook_inputs` and `hook_outputs` respectively +3. Add a new sql statement into `hook.sql` +4. Update the switch statements in `hooks.go` diff --git a/internal/hooks/custom_sms_request.json b/internal/hooks/custom_sms_request.json index e554adb475..87c04a0acc 100644 --- a/internal/hooks/custom_sms_request.json +++ b/internal/hooks/custom_sms_request.json @@ -6,7 +6,7 @@ "description": "version of the api", "type": "string" }, - "userg": { + "user": { "type": "object", "properties": { "phone": { From 1556ff71cdf504b0fe9baf5bad459e0d6cca22f5 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Tue, 10 Oct 2023 23:07:33 +0800 Subject: [PATCH 30/36] fix: update proposed readme --- internal/api/errors.go | 2 +- internal/hooks/README.md | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/internal/api/errors.go b/internal/api/errors.go index 474a1c0093..13b53214fd 100644 --- a/internal/api/errors.go +++ b/internal/api/errors.go @@ -124,7 +124,7 @@ func conflictError(fmtString string, args ...interface{}) *HTTPError { return httpError(http.StatusConflict, fmtString, args...) } func webhookResponseError(fmtString string, args ...interface{}) *HTTPError { - return internalServerError(fmt.Sprintf("Webhook returned malformed JSON: %v"), args) + return internalServerError(fmt.Sprintf("Webhook returned malformed JSON: %v", fmtString), args) } // HTTPError is an error with a message and an HTTP status code. diff --git a/internal/hooks/README.md b/internal/hooks/README.md index a0d02370a2..19120dc0f1 100644 --- a/internal/hooks/README.md +++ b/internal/hooks/README.md @@ -1,8 +1,20 @@ -## Overview +## Proposed Developer's Guide to adding a new extensibility point -This directory contains the JSONSchema's of each extensibility point. To introduce a new extensibility point, please do the following: +This directory contains the JSONSchema's of each extensibility point. To introduce a new extensibility point, do the following: -1. File a Pull Request with the `_request.json` and `_response.json` -2. Run `schema-generate _request.json` and `schema-generate _response.json` and place the `hook_inputs` and `hook_outputs` respectively -3. Add a new sql statement into `hook.sql` -4. Update the switch statements in `hooks.go` +1. Install [`schema-generate`](https://github.com/a-h/generate) +2. In `internal/hooks`, create two JSON Schema files for the inputs and outputs: `_request.json` and `_response.json` +3. Run `schema-generate _request.json` and `schema-generate _response.json` and place the generated outputs in `internal/api/hook_inputs` and `internal/api/hook_outputs.go` respectively. +4. Run the following SQL statement: + +``` sql +INSERT INTO auth.hook_config (id, uri, secret, extensibility_point, request_schema, response_schema, metadata) +VALUES + ('your_uuid_value', 'your_uri_value', 'my_supa_secret', 'custom-sms-provider', + 'your_request_jsonschema', + 'your_response_jsonschema', + '{}'::jsonb); +``` + +5. Edit `internal/api/transforms.go` so that it moulds the fields provided by GoTrue into a suitable request for the Hook. +6. Update the `EncodeAndValidateInput` as well as `DecodeAndValidateResponse` function in `hooks.go` to handle the new request and response. From a52299de143e637e223ac707809bd3c3c8453a8d Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Tue, 10 Oct 2023 23:10:12 +0800 Subject: [PATCH 31/36] chore: run gofmt --- internal/api/hook_transforms.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/api/hook_transforms.go b/internal/api/hook_transforms.go index 0e23383878..6851b56ccf 100644 --- a/internal/api/hook_transforms.go +++ b/internal/api/hook_transforms.go @@ -7,14 +7,13 @@ func TransformCustomSMSExtensibilityPointInputs(user *models.User, metadata map[ result := make(map[string]interface{}) userMap := make(map[string]interface{}) - if user != nil && user.Phone != "" { // Add the phone number to the result map result["user"] = userMap result["api_version"] = "1.0" userMap["app_metadata"] = make(map[string]interface{}) userMap["confirmed_at"] = "" - userMap["phone"] = string(user.Phone) + userMap["phone"] = string(user.Phone) } return result, nil From 099f87a875a4ab7d23e042ca4bbf14a7661bc5a3 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 11 Oct 2023 15:11:37 +0800 Subject: [PATCH 32/36] fix: update README --- internal/api/api.go | 5 +-- internal/api/hooks.go | 31 +++++++++++++++++-- internal/hooks/README.md | 18 ++++++----- internal/models/hooks.go | 2 +- .../20230913081932_add_hook_config.up.sql | 3 +- 5 files changed, 44 insertions(+), 15 deletions(-) diff --git a/internal/api/api.go b/internal/api/api.go index a55df2733f..18dceac160 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -229,8 +229,9 @@ func NewAPIWithVersion(ctx context.Context, globalConfig *conf.GlobalConfigurati }) corsHandler := cors.New(cors.Options{ - AllowedMethods: []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete}, - AllowedHeaders: globalConfig.CORS.AllAllowedHeaders([]string{"Accept", "Authorization", "Content-Type", "X-Client-IP", "X-Client-Info", audHeaderName, useCookieHeader}), + AllowedMethods: []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete}, + AllowedHeaders: globalConfig.CORS.AllAllowedHeaders([]string{"Accept", "Authorization", "Content-Type", "X-Client-IP", "X-Client-Info", audHeaderName, useCookieHeader, webhookSignatureHeader, webhookTimestampHeader, webhookIDHeader}), + // TODO: Add check if headers need to be exposed as well ExposedHeaders: []string{"X-Total-Count", "Link"}, AllowCredentials: true, }) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 55b606ddae..02718a9229 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -40,11 +40,22 @@ const ( LoginEvent = "login" ) +const ( + webhookSignatureHeader = "webhook-signature" + webhookTimestampHeader = "webhook-timestamp" + webhookIDHeader = "webhook-id" +) + // ExtensibilityPoints const ( CustomSMSExtensibilityPoint = "custom-sms-provider" ) +// Event names +// const ( +// CustomSMSEvent = fmt.Sprintf("auth.%v", CustomSMSExtensibilityPoint) +// ) + var defaultTimeout = time.Second * 5 type webhookClaims struct { @@ -73,6 +84,12 @@ type AuthHook struct { claims jwt.Claims } +// func setWebhookHeaders(req *http.Request) { +// req.Header.Set("webhook-id", "") +// req.Header.Set("webhook-timestamp", "") +// req.Header.Set("webhook-signature", "") +// } + func (w *AuthHook) trigger() (io.ReadCloser, error) { timeout := defaultTimeout if w.TimeoutSec > 0 { @@ -116,7 +133,11 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { if err != nil { return nil, internalServerError("Failed to make request object").WithInternalError(err) } + + // setWebhookHeaders(req) + req.Header.Set("Content-Type", "application/json") + watcher, req := watchForConnection(req) start := time.Now() @@ -163,6 +184,7 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { } func (a *AuthHook) generateSignature() (string, error) { + // TODO: change this to {msg_id}.{timestamp}.{payload}. token := jwt.NewWithClaims(jwt.SigningMethodHS256, a.claims) tokenString, err := token.SignedString([]byte(a.jwtSecret)) if err != nil { @@ -279,13 +301,16 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m "IssuedAt": time.Now().Unix(), "Subject": uuid.Nil.String(), "Issuer": authHookIssuer, - "Events": inp, + //"Type": "", + // "Timestamp": "now", + "Events": inp, } a := AuthHook{ WebhookConfig: &config.Webhook, - jwtSecret: hookConfig.Secret, - claims: claims, + // TODO: Add logic to support JWT secret selection + jwtSecret: hookConfig.Secret[0], + claims: claims, } // Works out because this is a http hook - eventually needs to change diff --git a/internal/hooks/README.md b/internal/hooks/README.md index 19120dc0f1..0089b38af3 100644 --- a/internal/hooks/README.md +++ b/internal/hooks/README.md @@ -1,20 +1,22 @@ -## Proposed Developer's Guide to adding a new extensibility point +## How To Add a Hook -This directory contains the JSONSchema's of each extensibility point. To introduce a new extensibility point, do the following: +To introduce a new hook at an extensibility point, do the following: -1. Install [`schema-generate`](https://github.com/a-h/generate) -2. In `internal/hooks`, create two JSON Schema files for the inputs and outputs: `_request.json` and `_response.json` +1. Clone [`schema-generate`](https://github.com/J0/generate) and run `make` +2. In `internal/hooks`, create two separate JSON Schema files for the inputs and outputs. They should follow the format `_request.json` and `_response.json` 3. Run `schema-generate _request.json` and `schema-generate _response.json` and place the generated outputs in `internal/api/hook_inputs` and `internal/api/hook_outputs.go` respectively. -4. Run the following SQL statement: - +4. Run the following SQL statement to register your hook: ``` sql INSERT INTO auth.hook_config (id, uri, secret, extensibility_point, request_schema, response_schema, metadata) VALUES - ('your_uuid_value', 'your_uri_value', 'my_supa_secret', 'custom-sms-provider', + ('your_uuid_value', 'your_uri_value', ARRAY['webhook_jwt_secret'], 'custom-sms-provider', 'your_request_jsonschema', 'your_response_jsonschema', '{}'::jsonb); ``` -5. Edit `internal/api/transforms.go` so that it moulds the fields provided by GoTrue into a suitable request for the Hook. +This will be superceded by an endpoint in the near future. + +5. Edit `internal/api/transforms.go` so that it transforms the fields provided by GoTrue into a suitable request for the Hook. 6. Update the `EncodeAndValidateInput` as well as `DecodeAndValidateResponse` function in `hooks.go` to handle the new request and response. +7. Add an event name in `hooks.go` to describe what happens when the Hook fires diff --git a/internal/models/hooks.go b/internal/models/hooks.go index e0a6c317aa..c2d5abb906 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -12,7 +12,7 @@ import ( type HookConfig struct { ID uuid.UUID `json:"id" db:"id"` URI string `json:"uri" db:"uri"` - Secret string `json:"secret" db:"secret"` + Secret []string `json:"secret" db:"secret"` ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` RequestSchema JSONMap `json:"request_schema" db:"request_schema"` ResponseSchema JSONMap `json:"response_schema" db:"response_schema"` diff --git a/migrations/20230913081932_add_hook_config.up.sql b/migrations/20230913081932_add_hook_config.up.sql index 486a89127b..7b870b3c64 100644 --- a/migrations/20230913081932_add_hook_config.up.sql +++ b/migrations/20230913081932_add_hook_config.up.sql @@ -3,7 +3,8 @@ create table if not exists {{ index .Options "Namespace" }}.hook_config( id uuid not null, uri text not null, - secret text not null, + -- This is an array in order to allow for low downtime JWT secret rotation + secret text[] not null, extensibility_point text not null, request_schema jsonb not null, response_schema jsonb not null, From fb3a37c25cf0a82c4730f714ad1b62291e14f5a4 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 11 Oct 2023 20:30:32 +0800 Subject: [PATCH 33/36] refactor: drop unused logged fields --- internal/api/hooks.go | 67 +++++++++++++------ internal/models/hooks.go | 1 + .../20230913081932_add_hook_config.up.sql | 1 + 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 02718a9229..3e0494a206 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -84,32 +84,48 @@ type AuthHook struct { claims jwt.Claims } -// func setWebhookHeaders(req *http.Request) { -// req.Header.Set("webhook-id", "") -// req.Header.Set("webhook-timestamp", "") +// func setWebhookHeaders(req *http.Request, hookID, timestamp string) { +// req.Header.Set("webhook-id", hookID) +// req.Header.Set("webhook-timestamp", timeStamp) // req.Header.Set("webhook-signature", "") // } -func (w *AuthHook) trigger() (io.ReadCloser, error) { +func generateHookCompliantTimestamp(timestamp time.Time) string { + + // Timeformat taken from Webhooks standard + timeFormat := "2022-11-03T20:26:10.344522Z" + formattedTime := timestamp.Format(timeFormat) + return formattedTime +} + +func (a *AuthHook) trigger() (io.ReadCloser, error) { timeout := defaultTimeout - if w.TimeoutSec > 0 { - timeout = time.Duration(w.TimeoutSec) * time.Second + if a.TimeoutSec > 0 { + timeout = time.Duration(a.TimeoutSec) * time.Second } - if w.Retries == 0 { - w.Retries = defaultHookRetries + if a.Retries == 0 { + a.Retries = defaultHookRetries + } + hookID := uuid.Must(uuid.NewV4()) + timestamp := time.Now().Unix() + signature, err := a.generateSignature() + if err != nil { + return nil, err } hooklog := logrus.WithFields(logrus.Fields{ "component": "webhook", - "url": w.URL, - "signed": w.jwtSecret != "", + "uri": a.URL, "instance_id": uuid.Nil.String(), + "hook_id": hookID, + "timestamp": timestamp, + "signature": signature, }) client := http.Client{ Timeout: timeout, } - signedPayload, jwtErr := w.generateSignature() + signedPayload, jwtErr := a.generateBody() if jwtErr != nil { return nil, jwtErr } @@ -125,16 +141,16 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { if err != nil { return nil, err } - for i := 0; i < w.Retries; i++ { + for i := 0; i < a.Retries; i++ { hooklog = hooklog.WithField("attempt", i+1) hooklog.Info("Starting to perform signup hook request") - req, err := http.NewRequest(http.MethodPost, w.URL, bytes.NewBuffer(requestLoad)) + req, err := http.NewRequest(http.MethodPost, a.URL, bytes.NewBuffer(requestLoad)) if err != nil { return nil, internalServerError("Failed to make request object").WithInternalError(err) } - // setWebhookHeaders(req) + // setWebhookHeaders(req, hookID, timestamp) req.Header.Set("Content-Type", "application/json") @@ -145,7 +161,7 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { if err != nil { if terr, ok := err.(net.Error); ok && terr.Timeout() { // timed out - try again? - if i == w.Retries-1 { + if i == a.Retries-1 { closeBody(rsp) return nil, httpError(http.StatusGatewayTimeout, "Failed to perform webhook in time frame (%v seconds)", timeout.Seconds()) } @@ -153,10 +169,10 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { continue } else if watcher.gotConn { closeBody(rsp) - return nil, internalServerError("Failed to trigger webhook to %s", w.URL).WithInternalError(err) + return nil, internalServerError("Failed to trigger webhook to %s", a.URL).WithInternalError(err) } else { closeBody(rsp) - return nil, httpError(http.StatusBadGateway, "Failed to connect to %s", w.URL) + return nil, httpError(http.StatusBadGateway, "Failed to connect to %s", a.URL) } } dur := time.Since(start) @@ -179,9 +195,17 @@ func (w *AuthHook) trigger() (io.ReadCloser, error) { } } - hooklog.Infof("Failed to process webhook for %s after %d attempts", w.URL, w.Retries) + hooklog.Infof("Failed to process webhook for %s after %d attempts", a.URL, a.Retries) return nil, unprocessableEntityError("Failed to handle signup webhook") } +func (a *AuthHook) generateBody() (string, error) { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, a.claims) + tokenString, err := token.SignedString([]byte(a.jwtSecret)) + if err != nil { + return "", internalServerError("Failed build signing string").WithInternalError(err) + } + return tokenString, nil +} func (a *AuthHook) generateSignature() (string, error) { // TODO: change this to {msg_id}.{timestamp}.{payload}. @@ -301,9 +325,10 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m "IssuedAt": time.Now().Unix(), "Subject": uuid.Nil.String(), "Issuer": authHookIssuer, - //"Type": "", - // "Timestamp": "now", - "Events": inp, + "Type": hookConfig.EventName, + // TODO: For readbility, kind of duplicate of issuedAt. Check if we need this + "Timestamp": generateHookCompliantTimestamp(time.Now().UTC()), + "Data": inp, } a := AuthHook{ diff --git a/internal/models/hooks.go b/internal/models/hooks.go index c2d5abb906..0ffa3e5f5e 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -14,6 +14,7 @@ type HookConfig struct { URI string `json:"uri" db:"uri"` Secret []string `json:"secret" db:"secret"` ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` + EventName string `json:"event_name" db:"event_name"` RequestSchema JSONMap `json:"request_schema" db:"request_schema"` ResponseSchema JSONMap `json:"response_schema" db:"response_schema"` Metadata JSONMap `json:"metadata" db:"metadata"` diff --git a/migrations/20230913081932_add_hook_config.up.sql b/migrations/20230913081932_add_hook_config.up.sql index 7b870b3c64..20d58aec16 100644 --- a/migrations/20230913081932_add_hook_config.up.sql +++ b/migrations/20230913081932_add_hook_config.up.sql @@ -3,6 +3,7 @@ create table if not exists {{ index .Options "Namespace" }}.hook_config( id uuid not null, uri text not null, + event_name text not null, -- This is an array in order to allow for low downtime JWT secret rotation secret text[] not null, extensibility_point text not null, From eacf6894746e88e38ae90d02bd583eafda38823e Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 11 Oct 2023 20:36:17 +0800 Subject: [PATCH 34/36] feat: set headers --- internal/api/hooks.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 3e0494a206..e1053ad520 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -51,11 +51,6 @@ const ( CustomSMSExtensibilityPoint = "custom-sms-provider" ) -// Event names -// const ( -// CustomSMSEvent = fmt.Sprintf("auth.%v", CustomSMSExtensibilityPoint) -// ) - var defaultTimeout = time.Second * 5 type webhookClaims struct { @@ -84,14 +79,13 @@ type AuthHook struct { claims jwt.Claims } -// func setWebhookHeaders(req *http.Request, hookID, timestamp string) { -// req.Header.Set("webhook-id", hookID) -// req.Header.Set("webhook-timestamp", timeStamp) -// req.Header.Set("webhook-signature", "") -// } +func setWebhookHeaders(req *http.Request, hookID uuid.UUID, timestamp int64) { + req.Header.Set("webhook-id", hookID.String()) + req.Header.Set("webhook-timestamp", fmt.Sprintf("%v", timestamp)) + // req.Header.Set("webhook-signature", "") +} func generateHookCompliantTimestamp(timestamp time.Time) string { - // Timeformat taken from Webhooks standard timeFormat := "2022-11-03T20:26:10.344522Z" formattedTime := timestamp.Format(timeFormat) @@ -150,7 +144,7 @@ func (a *AuthHook) trigger() (io.ReadCloser, error) { return nil, internalServerError("Failed to make request object").WithInternalError(err) } - // setWebhookHeaders(req, hookID, timestamp) + setWebhookHeaders(req, hookID, timestamp) req.Header.Set("Content-Type", "application/json") From cb3cba2e044726dfff16f0efa2de36a0b5a8239e Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Wed, 11 Oct 2023 23:04:03 +0800 Subject: [PATCH 35/36] feat: add character set checks on migration --- internal/api/hooks.go | 44 ++++++++++++------- .../20230913081932_add_hook_config.up.sql | 6 ++- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/internal/api/hooks.go b/internal/api/hooks.go index e1053ad520..67f1249aa8 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -3,7 +3,9 @@ package api import ( "bytes" "context" + "crypto/hmac" "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" @@ -73,8 +75,9 @@ type WebhookResponse struct { // Duplicate of Webhook, should eventually modify the fields passed type AuthHook struct { + // Deprecate this *conf.WebhookConfig - // Decide what should go here + // TODO: Decide whether to replace this with hookConfig instead jwtSecret string claims jwt.Claims } @@ -109,17 +112,16 @@ func (a *AuthHook) trigger() (io.ReadCloser, error) { } hooklog := logrus.WithFields(logrus.Fields{ - "component": "webhook", - "uri": a.URL, - "instance_id": uuid.Nil.String(), - "hook_id": hookID, - "timestamp": timestamp, - "signature": signature, + "component": "webhook", + "uri": a.URL, + "hook_id": hookID, + "timestamp": timestamp, + "signature": signature, }) client := http.Client{ Timeout: timeout, } - signedPayload, jwtErr := a.generateBody() + signedPayload, jwtErr := a.generateBody(a.jwtSecret) if jwtErr != nil { return nil, jwtErr } @@ -192,17 +194,27 @@ func (a *AuthHook) trigger() (io.ReadCloser, error) { hooklog.Infof("Failed to process webhook for %s after %d attempts", a.URL, a.Retries) return nil, unprocessableEntityError("Failed to handle signup webhook") } -func (a *AuthHook) generateBody() (string, error) { - token := jwt.NewWithClaims(jwt.SigningMethodHS256, a.claims) - tokenString, err := token.SignedString([]byte(a.jwtSecret)) - if err != nil { - return "", internalServerError("Failed build signing string").WithInternalError(err) - } - return tokenString, nil +func (a *AuthHook) generateBody(secret string) (string, error) { + // TODO: Probably do logging here so we can use the field + msgID := uuid.Must(uuid.NewV4()) + timestamp := time.Now().Unix() + // payload + msg := []byte(fmt.Sprintf("msg_%v.%v.%v", msgID, timestamp, a.claims)) + hasher := hmac.New(sha256.New, []byte(secret)) + // Write the data to the hasher + hasher.Write(msg) + + // Get the HMAC-SHA256 signature + signature := hasher.Sum(nil) + + // Encode the signature as a hex string + signatureHex := hex.EncodeToString(signature) + + return signatureHex, nil } func (a *AuthHook) generateSignature() (string, error) { - // TODO: change this to {msg_id}.{timestamp}.{payload}. + // TODO: change this to {msg_id}.{timestamp}.{payload} and also to use hmac-sha256 token := jwt.NewWithClaims(jwt.SigningMethodHS256, a.claims) tokenString, err := token.SignedString([]byte(a.jwtSecret)) if err != nil { diff --git a/migrations/20230913081932_add_hook_config.up.sql b/migrations/20230913081932_add_hook_config.up.sql index 20d58aec16..77aff7d87c 100644 --- a/migrations/20230913081932_add_hook_config.up.sql +++ b/migrations/20230913081932_add_hook_config.up.sql @@ -4,13 +4,15 @@ create table if not exists {{ index .Options "Namespace" }}.hook_config( id uuid not null, uri text not null, event_name text not null, - -- This is an array in order to allow for low downtime JWT secret rotation + -- TODO: This is an array in order to allow for low downtime JWT secret rotation. Decide whether to revert back to space separated string secret text[] not null, extensibility_point text not null, request_schema jsonb not null, response_schema jsonb not null, metadata json null, - constraint extensibility_point_pkey primary key (extensibility_point) + constraint extensibility_point_pkey primary key (extensibility_point), + constraint event_name_charset_check check (event_name ~ '^[a-zA-Z0-9_]+$') ); + comment on table {{ index .Options "Namespace" }}.hook_config is 'Auth: Store of hook configuration - can be used to customize hooks for given extensibility points.'; From b8f903cc6c631addfd63d0585712d982ce9f3896 Mon Sep 17 00:00:00 2001 From: "joel@joellee.org" Date: Mon, 30 Oct 2023 19:01:50 +0800 Subject: [PATCH 36/36] reafactor: convert claims to custom claims --- internal/api/hooks.go | 37 ++++++++++++------- internal/api/otp.go | 2 +- internal/hooks/README.md | 19 ++++------ internal/hooks/auth_hooks.csv | 2 + internal/models/hooks.go | 2 +- .../20230913081932_add_hook_config.up.sql | 3 +- 6 files changed, 36 insertions(+), 29 deletions(-) create mode 100644 internal/hooks/auth_hooks.csv diff --git a/internal/api/hooks.go b/internal/api/hooks.go index 67f1249aa8..3305d0d51b 100644 --- a/internal/api/hooks.go +++ b/internal/api/hooks.go @@ -73,13 +73,20 @@ type WebhookResponse struct { UserMetaData map[string]interface{} `json:"user_metadata,omitempty"` } +type AuthHookCustomClaims struct { + Data interface{} `json:"data"` + EventType string `json:"event_type"` + Timestamp string `json:"timestamp"` + jwt.StandardClaims +} + // Duplicate of Webhook, should eventually modify the fields passed type AuthHook struct { // Deprecate this *conf.WebhookConfig // TODO: Decide whether to replace this with hookConfig instead jwtSecret string - claims jwt.Claims + claims AuthHookCustomClaims } func setWebhookHeaders(req *http.Request, hookID uuid.UUID, timestamp int64) { @@ -106,7 +113,8 @@ func (a *AuthHook) trigger() (io.ReadCloser, error) { } hookID := uuid.Must(uuid.NewV4()) timestamp := time.Now().Unix() - signature, err := a.generateSignature() + // TODO: Double check this against the standard + signature, err := a.generateSignature(a.jwtSecret) if err != nil { return nil, err } @@ -154,6 +162,7 @@ func (a *AuthHook) trigger() (io.ReadCloser, error) { start := time.Now() rsp, err := client.Do(req) + if err != nil { if terr, ok := err.(net.Error); ok && terr.Timeout() { // timed out - try again? @@ -194,7 +203,7 @@ func (a *AuthHook) trigger() (io.ReadCloser, error) { hooklog.Infof("Failed to process webhook for %s after %d attempts", a.URL, a.Retries) return nil, unprocessableEntityError("Failed to handle signup webhook") } -func (a *AuthHook) generateBody(secret string) (string, error) { +func (a *AuthHook) generateSignature(secret string) (string, error) { // TODO: Probably do logging here so we can use the field msgID := uuid.Must(uuid.NewV4()) timestamp := time.Now().Unix() @@ -213,8 +222,7 @@ func (a *AuthHook) generateBody(secret string) (string, error) { return signatureHex, nil } -func (a *AuthHook) generateSignature() (string, error) { - // TODO: change this to {msg_id}.{timestamp}.{payload} and also to use hmac-sha256 +func (a *AuthHook) generateBody(secret string) (string, error) { token := jwt.NewWithClaims(jwt.SigningMethodHS256, a.claims) tokenString, err := token.SignedString([]byte(a.jwtSecret)) if err != nil { @@ -327,20 +335,21 @@ func triggerAuthHook(ctx context.Context, conn *storage.Connection, hookConfig m } // TODO: substitute with a custom claims interface - claims := jwt.MapClaims{ - "IssuedAt": time.Now().Unix(), - "Subject": uuid.Nil.String(), - "Issuer": authHookIssuer, - "Type": hookConfig.EventName, - // TODO: For readbility, kind of duplicate of issuedAt. Check if we need this - "Timestamp": generateHookCompliantTimestamp(time.Now().UTC()), - "Data": inp, + claims := AuthHookCustomClaims{ + inp, + hookConfig.EventName, + generateHookCompliantTimestamp(time.Now().UTC()), + jwt.StandardClaims{ + IssuedAt: time.Now().Unix(), + Subject: uuid.Nil.String(), + Issuer: authHookIssuer, + }, } a := AuthHook{ WebhookConfig: &config.Webhook, // TODO: Add logic to support JWT secret selection - jwtSecret: hookConfig.Secret[0], + jwtSecret: hookConfig.Secret, claims: claims, } diff --git a/internal/api/otp.go b/internal/api/otp.go index 9cb26262a6..75922598f8 100644 --- a/internal/api/otp.go +++ b/internal/api/otp.go @@ -205,7 +205,7 @@ func (a *API) SmsOtp(w http.ResponseWriter, r *http.Request) error { } mID, serr := a.sendPhoneConfirmation(ctx, tx, user, params.Phone, phoneConfirmationOtp, smsProvider, params.Channel) if serr != nil { - return badRequestError("Error sending sms OTP: %v", err) + return badRequestError("Error sending sms OTP: %v", serr) } messageID = mID return nil diff --git a/internal/hooks/README.md b/internal/hooks/README.md index 0089b38af3..5bc1fa1898 100644 --- a/internal/hooks/README.md +++ b/internal/hooks/README.md @@ -3,20 +3,17 @@ To introduce a new hook at an extensibility point, do the following: 1. Clone [`schema-generate`](https://github.com/J0/generate) and run `make` -2. In `internal/hooks`, create two separate JSON Schema files for the inputs and outputs. They should follow the format `_request.json` and `_response.json` -3. Run `schema-generate _request.json` and `schema-generate _response.json` and place the generated outputs in `internal/api/hook_inputs` and `internal/api/hook_outputs.go` respectively. -4. Run the following SQL statement to register your hook: -``` sql -INSERT INTO auth.hook_config (id, uri, secret, extensibility_point, request_schema, response_schema, metadata) -VALUES - ('your_uuid_value', 'your_uri_value', ARRAY['webhook_jwt_secret'], 'custom-sms-provider', - 'your_request_jsonschema', - 'your_response_jsonschema', - '{}'::jsonb); -``` +2. Update the `hooks.csv` with a new entry with appropriate `request_schema` and `repsonse_schema` in JSON format. +3. Run `schema-generate _request.json` and `schema-generate _response.json` and place the generated outputs in `internal/api/hook_inputs` and `internal/api/hook_outputs.go` respectively. See the entry for custom SMS provider as an example. +2. Copy hook information into container: `docker cp auth_hooks.csv gotrue_postgres:/tmp/auth_hooks.csv` +3. Load the hook config in: `COPY auth.hook_config FROM '/tmp/auth_hooks.csv' delimiter ',' csv header;` This will be superceded by an endpoint in the near future. 5. Edit `internal/api/transforms.go` so that it transforms the fields provided by GoTrue into a suitable request for the Hook. 6. Update the `EncodeAndValidateInput` as well as `DecodeAndValidateResponse` function in `hooks.go` to handle the new request and response. 7. Add an event name in `hooks.go` to describe what happens when the Hook fires + +### Note On Developing Locally + +If developing locally you will probably need to `http://host.docker.internal:54321/functions/v1/` or similar to access the external port. diff --git a/internal/hooks/auth_hooks.csv b/internal/hooks/auth_hooks.csv new file mode 100644 index 0000000000..f8c192fb44 --- /dev/null +++ b/internal/hooks/auth_hooks.csv @@ -0,0 +1,2 @@ +id,uri,event_name,secret,extensibility_point,request_schema,response_schema,metadata +c7088173-d2be-490c-b150-1b66b737408d,https://rstggefpkhuepnqqtmwq.supabase.co/functions/v1/custom-sms-sender,auth_custom_sms_provider,"my_supa_secret",custom-sms-provider,"{""title"":""CustomSMSRequest"",""type"":""object"",""properties"":{""api_version"":{""description"":""version of the api"",""type"":""string""},""user"":{""type"":""object"",""properties"":{""phone"":{""type"":""string"",""description"":""Store when a phone has been confirmed. Null if unconfirmed.""},""app_metadata"":{""type"":""object""},""confirmed_at"":{""type"":""string"",""description"":""Stores user attributes which do not impact core functionality""}},""required"":[""phone"",""app_metadata"",""confirmed_at""]}},""required"":[""api_version"",""user""],""additionalProperties"":false}","{""type"":""object"",""title"":""CustomSMSHookResponse"",""description"":""Custom SMS provider Request"",""properties"":{""status"":{""description"":""HTTP Status code (e.g. 400)"",""type"":""number""},""message"":{""description"":""Short Description of the error message"",""type"":""string""},""code"":{""description"":""Internal Error code reference"",""type"":""string""},""more_info"":{""description"":""Detailed elaboration and possibly link to an error page in the future."",""type"":""string""},""data"":{""description"":""Response data returned by the hook""},""api_version"":{""description"":""API Version"",""type"":""string""},""user"":{""description"":""User information"",""type"":""object""}},""required"":[""api_version"",""user""],""additionalProperties"":false}",{} diff --git a/internal/models/hooks.go b/internal/models/hooks.go index 0ffa3e5f5e..8ebb4eed2a 100644 --- a/internal/models/hooks.go +++ b/internal/models/hooks.go @@ -12,7 +12,7 @@ import ( type HookConfig struct { ID uuid.UUID `json:"id" db:"id"` URI string `json:"uri" db:"uri"` - Secret []string `json:"secret" db:"secret"` + Secret string `json:"secret" db:"secret"` ExtensibilityPoint string `json:"extensibility_point" db:"extensibility_point"` EventName string `json:"event_name" db:"event_name"` RequestSchema JSONMap `json:"request_schema" db:"request_schema"` diff --git a/migrations/20230913081932_add_hook_config.up.sql b/migrations/20230913081932_add_hook_config.up.sql index 77aff7d87c..d4deacb9c3 100644 --- a/migrations/20230913081932_add_hook_config.up.sql +++ b/migrations/20230913081932_add_hook_config.up.sql @@ -4,8 +4,7 @@ create table if not exists {{ index .Options "Namespace" }}.hook_config( id uuid not null, uri text not null, event_name text not null, - -- TODO: This is an array in order to allow for low downtime JWT secret rotation. Decide whether to revert back to space separated string - secret text[] not null, + secret text not null, extensibility_point text not null, request_schema jsonb not null, response_schema jsonb not null,