diff --git a/backend/controller/controller.go b/backend/controller/controller.go index eb4981ecd4..ea1f134288 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -640,11 +640,11 @@ func (s *Service) GetModuleContext(ctx context.Context, req *connect.Request[ftl if !ok { return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("module %q not found", req.Msg.Module)) } - moduleContext, err := modulecontext.FromEnvironment(ctx, module.Name) + moduleContext, err := modulecontext.FromEnvironment(ctx, module.Name, false) if err != nil { return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get module context: %w", err)) } - response, err := moduleContext.ToProto(ctx) + response, err := moduleContext.ToProto(module.Name) if err != nil { return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not marshal module context: %w", err)) } diff --git a/common/configuration/in_memory_resolver.go b/common/configuration/in_memory_resolver.go deleted file mode 100644 index a6644fdafc..0000000000 --- a/common/configuration/in_memory_resolver.go +++ /dev/null @@ -1,44 +0,0 @@ -package configuration - -import ( - "context" - "net/url" -) - -type InMemoryResolver[R Role] struct { - keyMap map[Ref]*url.URL -} - -var _ Resolver[Configuration] = InMemoryResolver[Configuration]{} -var _ Resolver[Secrets] = InMemoryResolver[Secrets]{} - -func NewInMemoryResolver[R Role]() *InMemoryResolver[R] { - return &InMemoryResolver[R]{keyMap: map[Ref]*url.URL{}} -} - -func (k InMemoryResolver[R]) Role() R { var r R; return r } - -func (k InMemoryResolver[R]) Get(ctx context.Context, ref Ref) (*url.URL, error) { - if key, found := k.keyMap[ref]; found { - return key, nil - } - return nil, ErrNotFound -} - -func (k InMemoryResolver[R]) List(ctx context.Context) ([]Entry, error) { - entries := []Entry{} - for ref, url := range k.keyMap { - entries = append(entries, Entry{Ref: ref, Accessor: url}) - } - return entries, nil -} - -func (k InMemoryResolver[R]) Set(ctx context.Context, ref Ref, key *url.URL) error { - k.keyMap[ref] = key - return nil -} - -func (k InMemoryResolver[R]) Unset(ctx context.Context, ref Ref) error { - delete(k.keyMap, ref) - return nil -} diff --git a/common/configuration/manager.go b/common/configuration/manager.go index 93759de13d..9961dcfae8 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -44,9 +44,7 @@ func configFromEnvironment() []string { func NewDefaultSecretsManagerFromEnvironment(ctx context.Context) (*Manager[Secrets], error) { var cr Resolver[Secrets] = ProjectConfigResolver[Secrets]{Config: configFromEnvironment()} return DefaultSecretsMixin{ - InlineProvider: InlineProvider[Secrets]{ - Inline: true, - }, + InlineProvider: InlineProvider[Secrets]{}, }.NewSecretsManager(ctx, cr) } @@ -55,9 +53,7 @@ func NewDefaultSecretsManagerFromEnvironment(ctx context.Context) (*Manager[Secr func NewDefaultConfigurationManagerFromEnvironment(ctx context.Context) (*Manager[Configuration], error) { cr := ProjectConfigResolver[Configuration]{Config: configFromEnvironment()} return DefaultConfigMixin{ - InlineProvider: InlineProvider[Configuration]{ - Inline: true, - }, + InlineProvider: InlineProvider[Configuration]{}, }.NewConfigurationManager(ctx, cr) } diff --git a/common/configuration/manager_test.go b/common/configuration/manager_test.go index 042391af3c..0e6848d20a 100644 --- a/common/configuration/manager_test.go +++ b/common/configuration/manager_test.go @@ -2,6 +2,7 @@ package configuration import ( "context" + "fmt" "net/url" "os" "path/filepath" @@ -16,13 +17,7 @@ import ( func TestManager(t *testing.T) { keyring.MockInit() // There's no way to undo this :\ - - config := filepath.Join(t.TempDir(), "ftl-project.toml") - existing, err := os.ReadFile("testdata/ftl-project.toml") - assert.NoError(t, err) - err = os.WriteFile(config, existing, 0600) - assert.NoError(t, err) - + config := tempConfigPath(t, "testdata/ftl-project.toml", "manager") ctx := log.ContextWithNewDefaultLogger(context.Background()) t.Run("Secrets", func(t *testing.T) { @@ -60,6 +55,64 @@ func TestManager(t *testing.T) { }) } +// TestMapPriority checks that module specific configs beat global configs when flattening for a module +func TestMapPriority(t *testing.T) { + ctx := log.ContextWithNewDefaultLogger(context.Background()) + config := tempConfigPath(t, "", "map") + cm, err := New(ctx, + ProjectConfigResolver[Configuration]{Config: []string{config}}, + []Provider[Configuration]{ + InlineProvider[Configuration]{ + Inline: true, + }, + }) + assert.NoError(t, err) + moduleName := "test" + + // Set 50 configs and 50 global configs + // It's hard to tell if module config beats global configs because we are dealing with unordered maps, or because the logic is correct + // Repeating it 50 times hopefully gives us a good chance of catching inconsistencies + for i := range 50 { + key := fmt.Sprintf("key%d", i) + + strValue := "HelloWorld" + globalStrValue := "GlobalHelloWorld" + if i%2 == 0 { + // sometimes try setting the module config first + assert.NoError(t, cm.Set(ctx, Ref{Module: optional.Some(moduleName), Name: key}, strValue)) + assert.NoError(t, cm.Set(ctx, Ref{Module: optional.None[string](), Name: key}, globalStrValue)) + } else { + // other times try setting the global config first + assert.NoError(t, cm.Set(ctx, Ref{Module: optional.None[string](), Name: key}, globalStrValue)) + assert.NoError(t, cm.Set(ctx, Ref{Module: optional.Some(moduleName), Name: key}, strValue)) + } + } + result, err := cm.MapForModule(ctx, moduleName) + assert.NoError(t, err) + + for i := range 50 { + key := fmt.Sprintf("key%d", i) + assert.Equal(t, `"HelloWorld"`, string(result[key]), "module configs should beat global configs") + } +} + +func tempConfigPath(t *testing.T, existingPath string, prefix string) string { + t.Helper() + + config := filepath.Join(t.TempDir(), fmt.Sprintf("%s-ftl-project.toml", prefix)) + var existing []byte + var err error + if existingPath == "" { + existing = []byte{} + } else { + existing, err = os.ReadFile(existingPath) + assert.NoError(t, err) + } + err = os.WriteFile(config, existing, 0600) + assert.NoError(t, err) + return config +} + // nolint func testManager[R Role]( t *testing.T, diff --git a/go-runtime/ftl/call.go b/go-runtime/ftl/call.go index f46f0b6d0f..57d6e8ded7 100644 --- a/go-runtime/ftl/call.go +++ b/go-runtime/ftl/call.go @@ -10,6 +10,7 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/go-runtime/encoding" + "github.com/TBD54566975/ftl/go-runtime/modulecontext" "github.com/TBD54566975/ftl/internal/rpc" ) @@ -19,37 +20,44 @@ func call[Req, Resp any](ctx context.Context, callee Ref, req Req) (resp Resp, e return resp, fmt.Errorf("%s: failed to marshal request: %w", callee, err) } - if overrider, ok := CallOverriderFromContext(ctx); ok { - override, uncheckedResp, err := overrider.OverrideCall(ctx, callee, req) + behavior, err := modulecontext.FromContext(ctx).BehaviorForVerb(modulecontext.Ref(callee)) + if err != nil { + return resp, fmt.Errorf("%s: %w", callee, err) + } + switch behavior := behavior.(type) { + case modulecontext.MockBehavior: + uncheckedResp, err := behavior.Mock(ctx, req) if err != nil { return resp, fmt.Errorf("%s: %w", callee, err) } - if override { - if resp, ok = uncheckedResp.(Resp); ok { - return resp, nil - } - return resp, fmt.Errorf("%s: overridden verb had invalid response type %T, expected %v", callee, uncheckedResp, reflect.TypeFor[Resp]()) + if r, ok := uncheckedResp.(Resp); ok { + return r, nil } - } - - client := rpc.ClientFromContext[ftlv1connect.VerbServiceClient](ctx) - cresp, err := client.Call(ctx, connect.NewRequest(&ftlv1.CallRequest{Verb: callee.ToProto(), Body: reqData})) - if err != nil { - return resp, fmt.Errorf("%s: failed to call Verb: %w", callee, err) - } - switch cresp := cresp.Msg.Response.(type) { - case *ftlv1.CallResponse_Error_: - return resp, fmt.Errorf("%s: %s", callee, cresp.Error.Message) - - case *ftlv1.CallResponse_Body: - err = encoding.Unmarshal(cresp.Body, &resp) + return resp, fmt.Errorf("%s: overridden verb had invalid response type %T, expected %v", callee, uncheckedResp, reflect.TypeFor[Resp]()) + case modulecontext.DirectBehavior: + panic("not implemented") + case modulecontext.StandardBehavior: + client := rpc.ClientFromContext[ftlv1connect.VerbServiceClient](ctx) + cresp, err := client.Call(ctx, connect.NewRequest(&ftlv1.CallRequest{Verb: callee.ToProto(), Body: reqData})) if err != nil { - return resp, fmt.Errorf("%s: failed to decode response: %w", callee, err) + return resp, fmt.Errorf("%s: failed to call Verb: %w", callee, err) } - return resp, nil + switch cresp := cresp.Msg.Response.(type) { + case *ftlv1.CallResponse_Error_: + return resp, fmt.Errorf("%s: %s", callee, cresp.Error.Message) + case *ftlv1.CallResponse_Body: + err = encoding.Unmarshal(cresp.Body, &resp) + if err != nil { + return resp, fmt.Errorf("%s: failed to decode response: %w", callee, err) + } + return resp, nil + + default: + panic(fmt.Sprintf("%s: invalid response type %T", callee, cresp)) + } default: - panic(fmt.Sprintf("%s: invalid response type %T", callee, cresp)) + panic(fmt.Sprintf("unknown behavior: %s", behavior)) } } diff --git a/go-runtime/ftl/call_overrider.go b/go-runtime/ftl/call_overrider.go deleted file mode 100644 index 48694dbed1..0000000000 --- a/go-runtime/ftl/call_overrider.go +++ /dev/null @@ -1,21 +0,0 @@ -package ftl - -import ( - "context" -) - -type CallOverrider interface { - OverrideCall(ctx context.Context, callee Ref, req any) (override bool, resp any, err error) -} -type contextCallOverriderKey struct{} - -func ApplyCallOverriderToContext(ctx context.Context, overrider CallOverrider) context.Context { - return context.WithValue(ctx, contextCallOverriderKey{}, overrider) -} - -func CallOverriderFromContext(ctx context.Context) (CallOverrider, bool) { - if overrider, ok := ctx.Value(contextCallOverriderKey{}).(CallOverrider); ok { - return overrider, true - } - return nil, false -} diff --git a/go-runtime/ftl/config.go b/go-runtime/ftl/config.go index ae0f93b9f2..7657345d79 100644 --- a/go-runtime/ftl/config.go +++ b/go-runtime/ftl/config.go @@ -6,7 +6,7 @@ import ( "runtime" "strings" - "github.com/TBD54566975/ftl/common/configuration" + "github.com/TBD54566975/ftl/go-runtime/modulecontext" ) // ConfigType is a type that can be used as a configuration value. @@ -32,9 +32,7 @@ func (c ConfigValue[T]) GoString() string { // Get returns the value of the configuration key from FTL. func (c ConfigValue[T]) Get(ctx context.Context) (out T) { - cm := configuration.ConfigFromContext(ctx) - ref := configuration.NewRef(c.Module, c.Name) - err := cm.Get(ctx, ref, &out) + err := modulecontext.FromContext(ctx).GetConfig(c.Name, &out) if err != nil { panic(fmt.Errorf("failed to get %s: %w", c, err)) } diff --git a/go-runtime/ftl/config_test.go b/go-runtime/ftl/config_test.go index 4b968804a3..9149703950 100644 --- a/go-runtime/ftl/config_test.go +++ b/go-runtime/ftl/config_test.go @@ -6,22 +6,21 @@ import ( "github.com/alecthomas/assert/v2" - "github.com/TBD54566975/ftl/common/configuration" - "github.com/TBD54566975/ftl/common/projectconfig" + "github.com/TBD54566975/ftl/go-runtime/modulecontext" "github.com/TBD54566975/ftl/internal/log" ) func TestConfig(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) - cr := configuration.ProjectConfigResolver[configuration.Configuration]{Config: []string{"testdata/ftl-project.toml"}} - assert.Equal(t, []string{"testdata/ftl-project.toml"}, projectconfig.ConfigPaths(cr.Config)) - cm, err := configuration.NewConfigurationManager(ctx, cr) - assert.NoError(t, err) - ctx = configuration.ContextWithConfig(ctx, cm) + + moduleCtx := modulecontext.New() + ctx = moduleCtx.ApplyToContext(ctx) + type C struct { One string Two string } config := Config[C]("test") + assert.NoError(t, moduleCtx.SetConfig("test", C{"one", "two"})) assert.Equal(t, C{"one", "two"}, config.Get(ctx)) } diff --git a/go-runtime/ftl/database.go b/go-runtime/ftl/database.go index d1ee62182e..8b7253f78c 100644 --- a/go-runtime/ftl/database.go +++ b/go-runtime/ftl/database.go @@ -27,8 +27,8 @@ func (d Database) String() string { return fmt.Sprintf("database %q", d.Name) } // Get returns the sql db connection for the database. func (d Database) Get(ctx context.Context) *sql.DB { - provider := modulecontext.DBProviderFromContext(ctx) - db, err := provider.Get(d.Name) + provider := modulecontext.FromContext(ctx) + db, err := provider.GetDatabase(d.Name, d.DBType) if err != nil { panic(err.Error()) } diff --git a/go-runtime/ftl/ftltest/ftltest.go b/go-runtime/ftl/ftltest/ftltest.go index 791729a770..24349b18be 100644 --- a/go-runtime/ftl/ftltest/ftltest.go +++ b/go-runtime/ftl/ftltest/ftltest.go @@ -6,9 +6,6 @@ import ( "fmt" "reflect" - "github.com/alecthomas/types/optional" - - cf "github.com/TBD54566975/ftl/common/configuration" "github.com/TBD54566975/ftl/go-runtime/ftl" "github.com/TBD54566975/ftl/go-runtime/modulecontext" "github.com/TBD54566975/ftl/internal/log" @@ -17,15 +14,12 @@ import ( // Context suitable for use in testing FTL verbs with provided options func Context(options ...func(context.Context) error) context.Context { ctx := log.ContextWithNewDefaultLogger(context.Background()) - context, err := modulecontext.FromEnvironment(ctx, ftl.Module()) + context, err := modulecontext.FromEnvironment(ctx, ftl.Module(), true) if err != nil { panic(err) } ctx = context.ApplyToContext(ctx) - mockProvider := newMockVerbProvider() - ctx = ftl.ApplyCallOverriderToContext(ctx, mockProvider) - for _, option := range options { err = option(ctx) if err != nil { @@ -49,8 +43,7 @@ func WithConfig[T ftl.ConfigType](config ftl.ConfigValue[T], value T) func(conte if config.Module != ftl.Module() { return fmt.Errorf("config %v does not match current module %s", config.Module, ftl.Module()) } - cm := cf.ConfigFromContext(ctx) - return cm.Set(ctx, cf.Ref{Module: optional.Some(config.Module), Name: config.Name}, value) + return modulecontext.FromContext(ctx).SetConfig(config.Name, value) } } @@ -68,8 +61,7 @@ func WithSecret[T ftl.SecretType](secret ftl.SecretValue[T], value T) func(conte if secret.Module != ftl.Module() { return fmt.Errorf("secret %v does not match current module %s", secret.Module, ftl.Module()) } - sm := cf.SecretsFromContext(ctx) - return sm.Set(ctx, cf.Ref{Module: optional.Some(secret.Module), Name: secret.Name}, value) + return modulecontext.FromContext(ctx).SetSecret(secret.Name, value) } } @@ -87,21 +79,13 @@ func WithSecret[T ftl.SecretType](secret ftl.SecretValue[T], value T) func(conte func WhenVerb[Req any, Resp any](verb ftl.Verb[Req, Resp], fake func(ctx context.Context, req Req) (resp Resp, err error)) func(context.Context) error { return func(ctx context.Context) error { ref := ftl.FuncRef(verb) - overrider, ok := ftl.CallOverriderFromContext(ctx) - if !ok { - return fmt.Errorf("could not override %v with a fake, context not set up with call overrider", ref) - } - mockProvider, ok := overrider.(*mockVerbProvider) - if !ok { - return fmt.Errorf("could not override %v with a fake, call overrider is not a MockProvider", ref) - } - mockProvider.mocks[ref] = func(ctx context.Context, req any) (resp any, err error) { + modulecontext.FromContext(ctx).SetMockVerb(modulecontext.Ref(ref), func(ctx context.Context, req any) (resp any, err error) { request, ok := req.(Req) if !ok { return nil, fmt.Errorf("invalid request type %T for %v, expected %v", req, ref, reflect.TypeFor[Req]()) } return fake(ctx, request) - } + }) return nil } } diff --git a/go-runtime/ftl/ftltest/mock.go b/go-runtime/ftl/ftltest/mock.go deleted file mode 100644 index 21a7448deb..0000000000 --- a/go-runtime/ftl/ftltest/mock.go +++ /dev/null @@ -1,41 +0,0 @@ -package ftltest - -import ( - "context" - "fmt" - - "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" - "github.com/TBD54566975/ftl/go-runtime/ftl" - "github.com/TBD54566975/ftl/internal/rpc" -) - -type mockFunc func(ctx context.Context, req any) (resp any, err error) - -// mockVerbProvider keeps a mapping of verb references to mock functions. -// -// It implements the CallOverrider interface to intercept calls with the mock functions. -type mockVerbProvider struct { - mocks map[ftl.Ref]mockFunc -} - -var _ = (ftl.CallOverrider)(&mockVerbProvider{}) - -func newMockVerbProvider() *mockVerbProvider { - provider := &mockVerbProvider{ - mocks: map[ftl.Ref]mockFunc{}, - } - return provider -} - -func (m *mockVerbProvider) OverrideCall(ctx context.Context, ref ftl.Ref, req any) (override bool, resp any, err error) { - mock, ok := m.mocks[ref] - if ok { - resp, err = mock(ctx, req) - return true, resp, err - } - if rpc.IsClientAvailableInContext[ftlv1connect.VerbServiceClient](ctx) { - return false, nil, nil - } - // Return a clean error for testing because we know the client is not available to make real calls - return false, nil, fmt.Errorf("no mock found") -} diff --git a/go-runtime/ftl/secrets.go b/go-runtime/ftl/secrets.go index d381e04ef9..978d6593f9 100644 --- a/go-runtime/ftl/secrets.go +++ b/go-runtime/ftl/secrets.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/TBD54566975/ftl/common/configuration" + "github.com/TBD54566975/ftl/go-runtime/modulecontext" ) // SecretType is a type that can be used as a secret value. @@ -30,8 +30,7 @@ func (s SecretValue[T]) GoString() string { // Get returns the value of the secret from FTL. func (s SecretValue[T]) Get(ctx context.Context) (out T) { - sm := configuration.SecretsFromContext(ctx) - if err := sm.Get(ctx, configuration.NewRef(s.Module, s.Name), &out); err != nil { + if err := modulecontext.FromContext(ctx).GetSecret(s.Name, &out); err != nil { panic(fmt.Errorf("failed to get %s: %w", s, err)) } return diff --git a/go-runtime/ftl/secrets_test.go b/go-runtime/ftl/secrets_test.go index 47ed08d0e0..b1079c856e 100644 --- a/go-runtime/ftl/secrets_test.go +++ b/go-runtime/ftl/secrets_test.go @@ -6,20 +6,21 @@ import ( "github.com/alecthomas/assert/v2" - "github.com/TBD54566975/ftl/common/configuration" + "github.com/TBD54566975/ftl/go-runtime/modulecontext" "github.com/TBD54566975/ftl/internal/log" ) func TestSecret(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) - sr := configuration.ProjectConfigResolver[configuration.Secrets]{Config: []string{"testdata/ftl-project.toml"}} - sm, err := configuration.NewSecretsManager(ctx, sr) - assert.NoError(t, err) - ctx = configuration.ContextWithSecrets(ctx, sm) + + moduleCtx := modulecontext.New() + ctx = moduleCtx.ApplyToContext(ctx) + type C struct { One string Two string } - config := Secret[C]("secret") - assert.Equal(t, C{"one", "two"}, config.Get(ctx)) + secret := Secret[C]("test") + assert.NoError(t, moduleCtx.SetSecret("test", C{"one", "two"})) + assert.Equal(t, C{"one", "two"}, secret.Get(ctx)) } diff --git a/go-runtime/ftl/testdata/ftl-project.toml b/go-runtime/ftl/testdata/ftl-project.toml deleted file mode 100644 index 71ef4bf259..0000000000 --- a/go-runtime/ftl/testdata/ftl-project.toml +++ /dev/null @@ -1,8 +0,0 @@ -[global] - -[modules] - [modules.testing] - [modules.testing.configuration] - test = "inline://eyJvbmUiOiJvbmUiLCJ0d28iOiJ0d28ifQ" - [modules.testing.secrets] - secret = "inline://eyJvbmUiOiJvbmUiLCJ0d28iOiJ0d28ifQ" diff --git a/go-runtime/modulecontext/db_provider.go b/go-runtime/modulecontext/db_provider.go deleted file mode 100644 index 93bdefb67f..0000000000 --- a/go-runtime/modulecontext/db_provider.go +++ /dev/null @@ -1,116 +0,0 @@ -package modulecontext - -import ( - "context" - "database/sql" - "fmt" - "os" - "strconv" - "strings" - - _ "github.com/jackc/pgx/v5/stdlib" // SQL driver - - ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" -) - -type DBType ftlv1.ModuleContextResponse_DBType - -const ( - DBTypePostgres = DBType(ftlv1.ModuleContextResponse_POSTGRES) -) - -func (x DBType) String() string { - switch x { - case DBTypePostgres: - return "Postgres" - default: - panic(fmt.Sprintf("unknown DB type: %s", strconv.Itoa(int(x)))) - } -} - -type dbEntry struct { - dsn string - dbType DBType - db *sql.DB -} - -// DBProvider takes in DSNs and holds a *sql.DB for each -// this allows us to: -// - pool db connections, rather than initializing anew each time -// - validate DSNs at startup, rather than returning errors or panicking at Database.Get() -type DBProvider struct { - entries map[string]dbEntry -} - -type contextKeyDSNProvider struct{} - -func NewDBProvider() *DBProvider { - return &DBProvider{ - entries: map[string]dbEntry{}, - } -} - -// NewDBProviderFromEnvironment creates a new DBProvider from environment variables. -// -// This is a temporary measure until we have a way to load DSNs from the ftl-project.toml file. -func NewDBProviderFromEnvironment(module string) (*DBProvider, error) { - // TODO: Replace this with loading DSNs from ftl-project.toml. - dbProvider := NewDBProvider() - for _, entry := range os.Environ() { - if !strings.HasPrefix(entry, "FTL_POSTGRES_DSN_") { - continue - } - parts := strings.SplitN(entry, "=", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("invalid DSN environment variable: %s", entry) - } - key := parts[0] - value := parts[1] - // FTL_POSTGRES_DSN_MODULE_DBNAME - parts = strings.Split(key, "_") - if len(parts) != 5 { - return nil, fmt.Errorf("invalid DSN environment variable: %s", entry) - } - moduleName := parts[3] - dbName := parts[4] - if !strings.EqualFold(moduleName, module) { - continue - } - if err := dbProvider.Add(strings.ToLower(dbName), DBTypePostgres, value); err != nil { - return nil, err - } - } - return dbProvider, nil -} - -func ContextWithDBProvider(ctx context.Context, provider *DBProvider) context.Context { - return context.WithValue(ctx, contextKeyDSNProvider{}, provider) -} - -func DBProviderFromContext(ctx context.Context) *DBProvider { - m, ok := ctx.Value(contextKeyDSNProvider{}).(*DBProvider) - if !ok { - panic("no db provider in context") - } - return m -} - -func (d *DBProvider) Add(name string, dbType DBType, dsn string) error { - db, err := sql.Open("pgx", dsn) - if err != nil { - return err - } - d.entries[name] = dbEntry{ - dsn: dsn, - db: db, - dbType: dbType, - } - return nil -} - -func (d *DBProvider) Get(name string) (*sql.DB, error) { - if entry, ok := d.entries[name]; ok { - return entry.db, nil - } - return nil, fmt.Errorf("missing DSN for database %s", name) -} diff --git a/go-runtime/modulecontext/db_provider_test.go b/go-runtime/modulecontext/db_provider_test.go deleted file mode 100644 index 66cf14b157..0000000000 --- a/go-runtime/modulecontext/db_provider_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package modulecontext - -import ( - "context" - "testing" - - "github.com/TBD54566975/ftl/internal/log" - "github.com/alecthomas/assert/v2" -) - -func TestValidDSN(t *testing.T) { - dbProvider := NewDBProvider() - dsn := "postgres://localhost:54320/echo?sslmode=disable&user=postgres&password=secret" - err := dbProvider.Add("test", DBTypePostgres, dsn) - assert.NoError(t, err, "expected no error for valid DSN") - assert.Equal(t, dbProvider.entries["test"].dsn, dsn, "expected DSN to be set and unmodified") -} - -func TestGettingAndSettingFromContext(t *testing.T) { - ctx := log.ContextWithNewDefaultLogger(context.Background()) - dbProvider := NewDBProvider() - ctx = ContextWithDBProvider(ctx, dbProvider) - assert.Equal(t, dbProvider, DBProviderFromContext(ctx), "expected dbProvider to be set and retrieved correctly") -} diff --git a/go-runtime/modulecontext/from_environment.go b/go-runtime/modulecontext/from_environment.go new file mode 100644 index 0000000000..d3d1a12fcb --- /dev/null +++ b/go-runtime/modulecontext/from_environment.go @@ -0,0 +1,75 @@ +package modulecontext + +import ( + "context" + "fmt" + "os" + "strings" + + cf "github.com/TBD54566975/ftl/common/configuration" +) + +// FromEnvironment creates a ModuleContext from the local environment. +// +// This is useful for testing and development, where the environment is used to provide +// configurations, secrets and DSNs. The context is built from a combination of +// the ftl-project.toml file and (for now) environment variables. +func FromEnvironment(ctx context.Context, module string, isTesting bool) (*ModuleContext, error) { + // TODO: split this func into separate purposes: explicitly reading a particular project file, and reading DSNs from environment + var moduleCtx *ModuleContext + if isTesting { + moduleCtx = NewForTesting() + } else { + moduleCtx = New() + } + + cm, err := cf.NewDefaultConfigurationManagerFromEnvironment(ctx) + if err != nil { + return nil, err + } + configs, err := cm.MapForModule(ctx, module) + if err != nil { + return nil, err + } + for name, data := range configs { + moduleCtx.SetConfigData(name, data) + } + + sm, err := cf.NewDefaultSecretsManagerFromEnvironment(ctx) + if err != nil { + return nil, err + } + secrets, err := sm.MapForModule(ctx, module) + if err != nil { + return nil, err + } + for name, data := range secrets { + moduleCtx.SetSecretData(name, data) + } + + for _, entry := range os.Environ() { + if !strings.HasPrefix(entry, "FTL_POSTGRES_DSN_") { + continue + } + parts := strings.SplitN(entry, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid DSN environment variable: %s", entry) + } + key := parts[0] + value := parts[1] + // FTL_POSTGRES_DSN_MODULE_DBNAME + parts = strings.Split(key, "_") + if len(parts) != 5 { + return nil, fmt.Errorf("invalid DSN environment variable: %s", entry) + } + moduleName := parts[3] + dbName := parts[4] + if !strings.EqualFold(moduleName, module) { + continue + } + if err := moduleCtx.AddDatabase(strings.ToLower(dbName), DBTypePostgres, value); err != nil { + return nil, err + } + } + return moduleCtx, nil +} diff --git a/go-runtime/modulecontext/from_environment_test.go b/go-runtime/modulecontext/from_environment_test.go new file mode 100644 index 0000000000..42eb324522 --- /dev/null +++ b/go-runtime/modulecontext/from_environment_test.go @@ -0,0 +1,54 @@ +package modulecontext + +import ( + "context" + "os" + "os/exec" //nolint:depguard + "path/filepath" + "testing" + + ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" + "github.com/TBD54566975/ftl/internal/log" + "github.com/alecthomas/assert/v2" +) + +func TestFromEnvironment(t *testing.T) { + // Setup a git repo with a ftl-project.toml file with known values. + dir := t.TempDir() + cmd := exec.Command("git", "init", dir) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + assert.NoError(t, err) + + data, err := os.ReadFile("testdata/ftl-project.toml") + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, "ftl-project.toml"), data, 0600) + assert.NoError(t, err) + + t.Setenv("FTL_POSTGRES_DSN_ECHO_ECHO", "postgres://echo:echo@localhost:5432/echo") + + // Move into the temp git repo. + oldwd, err := os.Getwd() + assert.NoError(t, err) + + assert.NoError(t, os.Chdir(dir)) + t.Cleanup(func() { assert.NoError(t, os.Chdir(oldwd)) }) + + ctx := log.ContextWithNewDefaultLogger(context.Background()) + + moduleContext, err := FromEnvironment(ctx, "echo", false) + assert.NoError(t, err) + + response, err := moduleContext.ToProto("echo") + assert.NoError(t, err) + + assert.Equal(t, &ftlv1.ModuleContextResponse{ + Module: "echo", + Configs: map[string][]uint8{"foo": []byte(`"bar"`)}, + Secrets: map[string][]uint8{"foo": []byte(`"bar"`)}, + Databases: []*ftlv1.ModuleContextResponse_DSN{ + {Name: "echo", Dsn: "postgres://echo:echo@localhost:5432/echo"}, + }, + }, response) +} diff --git a/go-runtime/modulecontext/from_proto.go b/go-runtime/modulecontext/from_proto.go index 4271c16b2f..aea15e6787 100644 --- a/go-runtime/modulecontext/from_proto.go +++ b/go-runtime/modulecontext/from_proto.go @@ -1,60 +1,21 @@ package modulecontext import ( - "context" - "net/url" - ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" - cf "github.com/TBD54566975/ftl/common/configuration" ) -func FromProto(ctx context.Context, response *ftlv1.ModuleContextResponse) (*ModuleContext, error) { - cm, err := newInMemoryConfigManager[cf.Configuration](ctx, response.Configs) - if err != nil { - return nil, err - } - sm, err := newInMemoryConfigManager[cf.Secrets](ctx, response.Secrets) - if err != nil { - return nil, err +func FromProto(response *ftlv1.ModuleContextResponse) (*ModuleContext, error) { + moduleCtx := New() + for name, data := range response.Configs { + moduleCtx.configs[name] = data } - moduleCtx := &ModuleContext{ - module: response.Module, - configManager: cm, - secretsManager: sm, - dbProvider: NewDBProvider(), + for name, data := range response.Secrets { + moduleCtx.secrets[name] = data } - for _, entry := range response.Databases { - if err = moduleCtx.dbProvider.Add(entry.Name, DBType(entry.Type), entry.Dsn); err != nil { + if err := moduleCtx.AddDatabase(entry.Name, DBType(entry.Type), entry.Dsn); err != nil { return nil, err } } return moduleCtx, nil } - -func newInMemoryConfigManager[R cf.Role](ctx context.Context, config map[string][]byte) (*cf.Manager[R], error) { - provider := cf.InlineProvider[R]{ - Inline: true, - } - refs := map[cf.Ref]*url.URL{} - for name, data := range config { - ref := cf.Ref{Name: name} - u, err := provider.Store(ctx, ref, data) - if err != nil { - return nil, err - } - refs[ref] = u - } - resolver := cf.NewInMemoryResolver[R]() - for ref, u := range refs { - err := resolver.Set(ctx, ref, u) - if err != nil { - return nil, err - } - } - manager, err := cf.New(ctx, resolver, []cf.Provider[R]{provider}) - if err != nil { - return nil, err - } - return manager, nil -} diff --git a/go-runtime/modulecontext/module_context.go b/go-runtime/modulecontext/module_context.go index 84d2719a79..c1a41ae354 100644 --- a/go-runtime/modulecontext/module_context.go +++ b/go-runtime/modulecontext/module_context.go @@ -2,59 +2,206 @@ package modulecontext import ( "context" + "database/sql" + "encoding/json" + "fmt" + "strconv" - cf "github.com/TBD54566975/ftl/common/configuration" + ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" + _ "github.com/jackc/pgx/v5/stdlib" // SQL driver ) +type Ref struct { + Module string + Name string +} + +type MockVerb func(ctx context.Context, req any) (resp any, err error) + +type dbEntry struct { + dsn string + dbType DBType + db *sql.DB +} + +type DBType ftlv1.ModuleContextResponse_DBType + +const ( + DBTypePostgres = DBType(ftlv1.ModuleContextResponse_POSTGRES) +) + +func (x DBType) String() string { + switch x { + case DBTypePostgres: + return "Postgres" + default: + panic(fmt.Sprintf("unknown DB type: %s", strconv.Itoa(int(x)))) + } +} + // ModuleContext holds the context needed for a module, including configs, secrets and DSNs type ModuleContext struct { - module string - configManager *cf.Manager[cf.Configuration] - secretsManager *cf.Manager[cf.Secrets] - dbProvider *DBProvider + isTesting bool + configs map[string][]byte + secrets map[string][]byte + databases map[string]dbEntry + mockVerbs map[Ref]MockVerb } -func New(module string, cm *cf.Manager[cf.Configuration], sm *cf.Manager[cf.Secrets], dbp *DBProvider) *ModuleContext { +type contextKeyModuleContext struct{} + +func New() *ModuleContext { return &ModuleContext{ - module: module, - configManager: cm, - secretsManager: sm, - dbProvider: dbp, + configs: map[string][]byte{}, + secrets: map[string][]byte{}, + databases: map[string]dbEntry{}, + mockVerbs: map[Ref]MockVerb{}, + } +} + +func NewForTesting() *ModuleContext { + moduleCtx := New() + moduleCtx.isTesting = true + return moduleCtx +} + +func FromContext(ctx context.Context) *ModuleContext { + m, ok := ctx.Value(contextKeyModuleContext{}).(*ModuleContext) + if !ok { + panic("no ModuleContext in context") } + return m } -// ApplyToContext returns a Go context.Context that includes configurations, -// secrets and DSNs can be retreived Each of these components have accessors to -// get a manager back from the context +// ApplyToContext returns a Go context.Context with ModuleContext added. func (m *ModuleContext) ApplyToContext(ctx context.Context) context.Context { - ctx = ContextWithDBProvider(ctx, m.dbProvider) - ctx = cf.ContextWithConfig(ctx, m.configManager) - ctx = cf.ContextWithSecrets(ctx, m.secretsManager) - return ctx + return context.WithValue(ctx, contextKeyModuleContext{}, m) } -// FromEnvironment creates a ModuleContext from the local environment. +// GetConfig reads a configuration value for the module. // -// This is useful for testing and development, where the environment is used to provide -// configurations, secrets and DSNs. The context is built from a combination of -// the ftl-project.toml file and (for now) environment variables. -func FromEnvironment(ctx context.Context, module string) (*ModuleContext, error) { - cm, err := cf.NewDefaultConfigurationManagerFromEnvironment(ctx) +// "value" must be a pointer to a Go type that can be unmarshalled from JSON. +func (m *ModuleContext) GetConfig(name string, value any) error { + data, ok := m.configs[name] + if !ok { + return fmt.Errorf("no config value for %q", name) + } + return json.Unmarshal(data, value) +} + +// SetConfig sets a configuration value for the module. +func (m *ModuleContext) SetConfig(name string, value any) error { + data, err := json.Marshal(value) if err != nil { - return nil, err + return err } - sm, err := cf.NewDefaultSecretsManagerFromEnvironment(ctx) + m.SetConfigData(name, data) + return nil +} + +// SetConfigData sets a configuration value with raw bytes +func (m *ModuleContext) SetConfigData(name string, data []byte) { + m.configs[name] = data +} + +// GetSecret reads a secret value for the module. +// +// "value" must be a pointer to a Go type that can be unmarshalled from JSON. +func (m *ModuleContext) GetSecret(name string, value any) error { + data, ok := m.secrets[name] + if !ok { + return fmt.Errorf("no secret value for %q", name) + } + return json.Unmarshal(data, value) +} + +// SetSecret sets a secret value for the module. +func (m *ModuleContext) SetSecret(name string, value any) error { + data, err := json.Marshal(value) if err != nil { - return nil, err + return err } - dbp, err := NewDBProviderFromEnvironment(module) + m.SetSecretData(name, data) + return nil +} + +// SetSecretData sets a secret value with raw bytes +func (m *ModuleContext) SetSecretData(name string, data []byte) { + m.secrets[name] = data +} + +// AddDatabase adds a database connection +func (m *ModuleContext) AddDatabase(name string, dbType DBType, dsn string) error { + db, err := sql.Open("pgx", dsn) if err != nil { - return nil, err + return err } - return &ModuleContext{ - module: module, - configManager: cm, - secretsManager: sm, - dbProvider: dbp, - }, nil + m.databases[name] = dbEntry{ + dsn: dsn, + db: db, + dbType: dbType, + } + return nil } + +// GetDatabase gets a database connection +// +// Returns an error if no database with that name is found or it is not the expected type +func (m *ModuleContext) GetDatabase(name string, dbType DBType) (*sql.DB, error) { + entry, ok := m.databases[name] + if !ok { + return nil, fmt.Errorf("missing DSN for database %s", name) + } + if entry.dbType != dbType { + return nil, fmt.Errorf("database %s does not match expected type of %s", name, dbType) + } + return entry.db, nil +} + +// BehaviorForVerb returns what to do to execute a verb +// +// This allows module context to dictate behavior based on testing options +func (m *ModuleContext) BehaviorForVerb(ref Ref) (VerbBehavior, error) { + if mock, ok := m.mockVerbs[ref]; ok { + return MockBehavior{Mock: mock}, nil + } + // TODO: add logic here for when to do direct behavior + if m.isTesting { + return StandardBehavior{}, fmt.Errorf("no mock found") + } + return StandardBehavior{}, nil +} + +func (m *ModuleContext) SetMockVerb(ref Ref, mock MockVerb) { + m.mockVerbs[ref] = mock +} + +// VerbBehavior indicates how to execute a verb +// +//sumtype:decl +type VerbBehavior interface { + verbBehavior() +} + +// StandardBehavior indicates that the verb should be executed via the controller +type StandardBehavior struct{} + +func (StandardBehavior) verbBehavior() {} + +var _ VerbBehavior = StandardBehavior{} + +// DirectBehavior indicates that the verb should be executed by calling the function directly (for testing) +type DirectBehavior struct{} + +func (DirectBehavior) verbBehavior() {} + +var _ VerbBehavior = DirectBehavior{} + +// MockBehavior indicates the verb has a mock implementation +type MockBehavior struct { + Mock MockVerb +} + +func (MockBehavior) verbBehavior() {} + +var _ VerbBehavior = MockBehavior{} diff --git a/go-runtime/modulecontext/module_context_test.go b/go-runtime/modulecontext/module_context_test.go index 0d28919923..9818d418d1 100644 --- a/go-runtime/modulecontext/module_context_test.go +++ b/go-runtime/modulecontext/module_context_test.go @@ -1,102 +1,17 @@ package modulecontext import ( - "context" - "fmt" - "os" - "os/exec" //nolint:depguard - "path/filepath" + "context" //nolint:depguard "testing" "github.com/alecthomas/assert/v2" - "github.com/alecthomas/types/optional" - ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" - cf "github.com/TBD54566975/ftl/common/configuration" "github.com/TBD54566975/ftl/internal/log" ) -func TestConfigPriority(t *testing.T) { +func TestGettingAndSettingFromContext(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) - - moduleName := "test" - - cp := cf.InlineProvider[cf.Configuration]{ - Inline: true, - } - cr := cf.NewInMemoryResolver[cf.Configuration]() - cm, err := cf.New(ctx, cr, []cf.Provider[cf.Configuration]{cp}) - assert.NoError(t, err) - ctx = cf.ContextWithConfig(ctx, cm) - - sp := cf.InlineProvider[cf.Secrets]{ - Inline: true, - } - sr := cf.NewInMemoryResolver[cf.Secrets]() - sm, err := cf.New(ctx, sr, []cf.Provider[cf.Secrets]{sp}) - assert.NoError(t, err) - ctx = cf.ContextWithSecrets(ctx, sm) - - // Set 50 configs and 50 global configs - // It's hard to tell if module config beats global configs because we are dealing with unordered maps, or because the logic is correct - // Repeating it 50 times hopefully gives us a good chance of catching inconsistencies - for i := range 50 { - key := fmt.Sprintf("key%d", i) - - strValue := "HelloWorld" - globalStrValue := "GlobalHelloWorld" - assert.NoError(t, cm.Set(ctx, cf.Ref{Module: optional.Some(moduleName), Name: key}, strValue)) - assert.NoError(t, cm.Set(ctx, cf.Ref{Module: optional.None[string](), Name: key}, globalStrValue)) - } - - moduleContext := New(moduleName, cm, sm, NewDBProvider()) - - response, err := moduleContext.ToProto(ctx) - assert.NoError(t, err) - - for i := range 50 { - key := fmt.Sprintf("key%d", i) - assert.Equal(t, `"HelloWorld"`, string(response.Configs[key]), "module configs should beat global configs") - } -} - -func TestFromEnvironment(t *testing.T) { - // Setup a git repo with a ftl-project.toml file with known values. - dir := t.TempDir() - cmd := exec.Command("git", "init", dir) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err := cmd.Run() - assert.NoError(t, err) - - data, err := os.ReadFile("testdata/ftl-project.toml") - assert.NoError(t, err) - err = os.WriteFile(filepath.Join(dir, "ftl-project.toml"), data, 0600) - assert.NoError(t, err) - - t.Setenv("FTL_POSTGRES_DSN_ECHO_ECHO", "postgres://echo:echo@localhost:5432/echo") - - // Move into the temp git repo. - oldwd, err := os.Getwd() - assert.NoError(t, err) - - assert.NoError(t, os.Chdir(dir)) - t.Cleanup(func() { assert.NoError(t, os.Chdir(oldwd)) }) - - ctx := log.ContextWithNewDefaultLogger(context.Background()) - - moduleContext, err := FromEnvironment(ctx, "echo") - assert.NoError(t, err) - - response, err := moduleContext.ToProto(ctx) - assert.NoError(t, err) - - assert.Equal(t, &ftlv1.ModuleContextResponse{ - Module: "echo", - Configs: map[string][]uint8{"foo": []byte(`"bar"`)}, - Secrets: map[string][]uint8{"foo": []byte(`"bar"`)}, - Databases: []*ftlv1.ModuleContextResponse_DSN{ - {Name: "echo", Dsn: "postgres://echo:echo@localhost:5432/echo"}, - }, - }, response) + moduleCtx := New() + ctx = moduleCtx.ApplyToContext(ctx) + assert.Equal(t, moduleCtx, FromContext(ctx), "module context should be the same when read from context") } diff --git a/go-runtime/modulecontext/to_proto.go b/go-runtime/modulecontext/to_proto.go index 118b9618cf..347b0cc396 100644 --- a/go-runtime/modulecontext/to_proto.go +++ b/go-runtime/modulecontext/to_proto.go @@ -1,24 +1,13 @@ package modulecontext import ( - "context" - "fmt" - ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" ) // ToProto converts a ModuleContext to a proto response. -func (m *ModuleContext) ToProto(ctx context.Context) (*ftlv1.ModuleContextResponse, error) { - config, err := m.configManager.MapForModule(ctx, m.module) - if err != nil { - return nil, fmt.Errorf("failed to get config map: %w", err) - } - secrets, err := m.secretsManager.MapForModule(ctx, m.module) - if err != nil { - return nil, fmt.Errorf("failed to get secrets map: %w", err) - } - databases := make([]*ftlv1.ModuleContextResponse_DSN, 0, len(m.dbProvider.entries)) - for name, entry := range m.dbProvider.entries { +func (m *ModuleContext) ToProto(moduleName string) (*ftlv1.ModuleContextResponse, error) { + databases := make([]*ftlv1.ModuleContextResponse_DSN, 0, len(m.databases)) + for name, entry := range m.databases { databases = append(databases, &ftlv1.ModuleContextResponse_DSN{ Name: name, Type: ftlv1.ModuleContextResponse_DBType(entry.dbType), @@ -26,9 +15,9 @@ func (m *ModuleContext) ToProto(ctx context.Context) (*ftlv1.ModuleContextRespon }) } return &ftlv1.ModuleContextResponse{ - Module: m.module, - Configs: config, - Secrets: secrets, + Module: moduleName, + Configs: m.configs, + Secrets: m.secrets, Databases: databases, }, nil } diff --git a/go-runtime/server/server.go b/go-runtime/server/server.go index af5bcb0e6b..e682b93bf4 100644 --- a/go-runtime/server/server.go +++ b/go-runtime/server/server.go @@ -40,7 +40,7 @@ func NewUserVerbServer(moduleName string, handlers ...Handler) plugin.Constructo if err != nil { return nil, nil, fmt.Errorf("could not get config: %w", err) } - moduleCtx, err := modulecontext.FromProto(ctx, resp.Msg) + moduleCtx, err := modulecontext.FromProto(resp.Msg) if err != nil { return nil, nil, err }