Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: centralise config/secrets/dbs/mocks into modulecontext
Browse files Browse the repository at this point in the history
matt2e committed May 3, 2024

Verified

This commit was signed with the committer’s verified signature. The key has expired.
ripcurlx Christoph Atteneder
1 parent 61fdf00 commit 247e043
Showing 23 changed files with 451 additions and 524 deletions.
4 changes: 2 additions & 2 deletions backend/controller/controller.go
Original file line number Diff line number Diff line change
@@ -640,11 +640,11 @@ func (s *Service) GetModuleContext(ctx context.Context, req *connect.Request[ftl
if !ok {
return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("module %q not found", req.Msg.Module))
}
moduleContext, err := modulecontext.FromEnvironment(ctx, module.Name)
moduleContext, err := modulecontext.FromEnvironment(ctx, module.Name, false)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get module context: %w", err))
}
response, err := moduleContext.ToProto(ctx)
response, err := moduleContext.ToProto(module.Name)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not marshal module context: %w", err))
}
44 changes: 0 additions & 44 deletions common/configuration/in_memory_resolver.go

This file was deleted.

8 changes: 2 additions & 6 deletions common/configuration/manager.go
Original file line number Diff line number Diff line change
@@ -44,9 +44,7 @@ func configFromEnvironment() []string {
func NewDefaultSecretsManagerFromEnvironment(ctx context.Context) (*Manager[Secrets], error) {
var cr Resolver[Secrets] = ProjectConfigResolver[Secrets]{Config: configFromEnvironment()}
return DefaultSecretsMixin{
InlineProvider: InlineProvider[Secrets]{
Inline: true,
},
InlineProvider: InlineProvider[Secrets]{},
}.NewSecretsManager(ctx, cr)
}

@@ -55,9 +53,7 @@ func NewDefaultSecretsManagerFromEnvironment(ctx context.Context) (*Manager[Secr
func NewDefaultConfigurationManagerFromEnvironment(ctx context.Context) (*Manager[Configuration], error) {
cr := ProjectConfigResolver[Configuration]{Config: configFromEnvironment()}
return DefaultConfigMixin{
InlineProvider: InlineProvider[Configuration]{
Inline: true,
},
InlineProvider: InlineProvider[Configuration]{},
}.NewConfigurationManager(ctx, cr)
}

67 changes: 60 additions & 7 deletions common/configuration/manager_test.go
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ package configuration

import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
@@ -16,13 +17,7 @@ import (

func TestManager(t *testing.T) {
keyring.MockInit() // There's no way to undo this :\

config := filepath.Join(t.TempDir(), "ftl-project.toml")
existing, err := os.ReadFile("testdata/ftl-project.toml")
assert.NoError(t, err)
err = os.WriteFile(config, existing, 0600)
assert.NoError(t, err)

config := tempConfigPath(t, "testdata/ftl-project.toml", "manager")
ctx := log.ContextWithNewDefaultLogger(context.Background())

t.Run("Secrets", func(t *testing.T) {
@@ -60,6 +55,64 @@ func TestManager(t *testing.T) {
})
}

// TestMapPriority checks that module specific configs beat global configs when flattening for a module
func TestMapPriority(t *testing.T) {
ctx := log.ContextWithNewDefaultLogger(context.Background())
config := tempConfigPath(t, "", "map")
cm, err := New(ctx,
ProjectConfigResolver[Configuration]{Config: []string{config}},
[]Provider[Configuration]{
InlineProvider[Configuration]{
Inline: true,
},
})
assert.NoError(t, err)
moduleName := "test"

// Set 50 configs and 50 global configs
// It's hard to tell if module config beats global configs because we are dealing with unordered maps, or because the logic is correct
// Repeating it 50 times hopefully gives us a good chance of catching inconsistencies
for i := range 50 {
key := fmt.Sprintf("key%d", i)

strValue := "HelloWorld"
globalStrValue := "GlobalHelloWorld"
if i%2 == 0 {
// sometimes try setting the module config first
assert.NoError(t, cm.Set(ctx, Ref{Module: optional.Some(moduleName), Name: key}, strValue))
assert.NoError(t, cm.Set(ctx, Ref{Module: optional.None[string](), Name: key}, globalStrValue))
} else {
// other times try setting the global config first
assert.NoError(t, cm.Set(ctx, Ref{Module: optional.None[string](), Name: key}, globalStrValue))
assert.NoError(t, cm.Set(ctx, Ref{Module: optional.Some(moduleName), Name: key}, strValue))
}
}
result, err := cm.MapForModule(ctx, moduleName)
assert.NoError(t, err)

for i := range 50 {
key := fmt.Sprintf("key%d", i)
assert.Equal(t, `"HelloWorld"`, string(result[key]), "module configs should beat global configs")
}
}

func tempConfigPath(t *testing.T, existingPath string, prefix string) string {
t.Helper()

config := filepath.Join(t.TempDir(), fmt.Sprintf("%s-ftl-project.toml", prefix))
var existing []byte
var err error
if existingPath == "" {
existing = []byte{}
} else {
existing, err = os.ReadFile(existingPath)
assert.NoError(t, err)
}
err = os.WriteFile(config, existing, 0600)
assert.NoError(t, err)
return config
}

// nolint
func testManager[R Role](
t *testing.T,
54 changes: 31 additions & 23 deletions go-runtime/ftl/call.go
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ import (
ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
"github.com/TBD54566975/ftl/go-runtime/encoding"
"github.com/TBD54566975/ftl/go-runtime/modulecontext"
"github.com/TBD54566975/ftl/internal/rpc"
)

@@ -19,37 +20,44 @@ func call[Req, Resp any](ctx context.Context, callee Ref, req Req) (resp Resp, e
return resp, fmt.Errorf("%s: failed to marshal request: %w", callee, err)
}

if overrider, ok := CallOverriderFromContext(ctx); ok {
override, uncheckedResp, err := overrider.OverrideCall(ctx, callee, req)
behavior, err := modulecontext.FromContext(ctx).BehaviorForVerb(modulecontext.Ref(callee))
if err != nil {
return resp, fmt.Errorf("%s: %w", callee, err)
}
switch behavior := behavior.(type) {
case modulecontext.MockBehavior:
uncheckedResp, err := behavior.Mock(ctx, req)
if err != nil {
return resp, fmt.Errorf("%s: %w", callee, err)
}
if override {
if resp, ok = uncheckedResp.(Resp); ok {
return resp, nil
}
return resp, fmt.Errorf("%s: overridden verb had invalid response type %T, expected %v", callee, uncheckedResp, reflect.TypeFor[Resp]())
if r, ok := uncheckedResp.(Resp); ok {
return r, nil
}
}

client := rpc.ClientFromContext[ftlv1connect.VerbServiceClient](ctx)
cresp, err := client.Call(ctx, connect.NewRequest(&ftlv1.CallRequest{Verb: callee.ToProto(), Body: reqData}))
if err != nil {
return resp, fmt.Errorf("%s: failed to call Verb: %w", callee, err)
}
switch cresp := cresp.Msg.Response.(type) {
case *ftlv1.CallResponse_Error_:
return resp, fmt.Errorf("%s: %s", callee, cresp.Error.Message)

case *ftlv1.CallResponse_Body:
err = encoding.Unmarshal(cresp.Body, &resp)
return resp, fmt.Errorf("%s: overridden verb had invalid response type %T, expected %v", callee, uncheckedResp, reflect.TypeFor[Resp]())
case modulecontext.DirectBehavior:
panic("not implemented")
case modulecontext.StandardBehavior:
client := rpc.ClientFromContext[ftlv1connect.VerbServiceClient](ctx)
cresp, err := client.Call(ctx, connect.NewRequest(&ftlv1.CallRequest{Verb: callee.ToProto(), Body: reqData}))
if err != nil {
return resp, fmt.Errorf("%s: failed to decode response: %w", callee, err)
return resp, fmt.Errorf("%s: failed to call Verb: %w", callee, err)
}
return resp, nil
switch cresp := cresp.Msg.Response.(type) {
case *ftlv1.CallResponse_Error_:
return resp, fmt.Errorf("%s: %s", callee, cresp.Error.Message)

case *ftlv1.CallResponse_Body:
err = encoding.Unmarshal(cresp.Body, &resp)
if err != nil {
return resp, fmt.Errorf("%s: failed to decode response: %w", callee, err)
}
return resp, nil

default:
panic(fmt.Sprintf("%s: invalid response type %T", callee, cresp))
}
default:
panic(fmt.Sprintf("%s: invalid response type %T", callee, cresp))
panic(fmt.Sprintf("unknown behavior: %s", behavior))
}
}

21 changes: 0 additions & 21 deletions go-runtime/ftl/call_overrider.go

This file was deleted.

6 changes: 2 additions & 4 deletions go-runtime/ftl/config.go
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ import (
"runtime"
"strings"

"github.com/TBD54566975/ftl/common/configuration"
"github.com/TBD54566975/ftl/go-runtime/modulecontext"
)

// ConfigType is a type that can be used as a configuration value.
@@ -32,9 +32,7 @@ func (c ConfigValue[T]) GoString() string {

// Get returns the value of the configuration key from FTL.
func (c ConfigValue[T]) Get(ctx context.Context) (out T) {
cm := configuration.ConfigFromContext(ctx)
ref := configuration.NewRef(c.Module, c.Name)
err := cm.Get(ctx, ref, &out)
err := modulecontext.FromContext(ctx).GetConfig(c.Name, &out)
if err != nil {
panic(fmt.Errorf("failed to get %s: %w", c, err))
}
13 changes: 6 additions & 7 deletions go-runtime/ftl/config_test.go
Original file line number Diff line number Diff line change
@@ -6,22 +6,21 @@ import (

"github.com/alecthomas/assert/v2"

"github.com/TBD54566975/ftl/common/configuration"
"github.com/TBD54566975/ftl/common/projectconfig"
"github.com/TBD54566975/ftl/go-runtime/modulecontext"
"github.com/TBD54566975/ftl/internal/log"
)

func TestConfig(t *testing.T) {
ctx := log.ContextWithNewDefaultLogger(context.Background())
cr := configuration.ProjectConfigResolver[configuration.Configuration]{Config: []string{"testdata/ftl-project.toml"}}
assert.Equal(t, []string{"testdata/ftl-project.toml"}, projectconfig.ConfigPaths(cr.Config))
cm, err := configuration.NewConfigurationManager(ctx, cr)
assert.NoError(t, err)
ctx = configuration.ContextWithConfig(ctx, cm)

moduleCtx := modulecontext.New()
ctx = moduleCtx.ApplyToContext(ctx)

type C struct {
One string
Two string
}
config := Config[C]("test")
moduleCtx.SetConfig("test", C{"one", "two"})

Check failure on line 24 in go-runtime/ftl/config_test.go

GitHub Actions / Lint

Error return value of `moduleCtx.SetConfig` is not checked (errcheck)
assert.Equal(t, C{"one", "two"}, config.Get(ctx))
}
4 changes: 2 additions & 2 deletions go-runtime/ftl/database.go
Original file line number Diff line number Diff line change
@@ -27,8 +27,8 @@ func (d Database) String() string { return fmt.Sprintf("database %q", d.Name) }

// Get returns the sql db connection for the database.
func (d Database) Get(ctx context.Context) *sql.DB {
provider := modulecontext.DBProviderFromContext(ctx)
db, err := provider.Get(d.Name)
provider := modulecontext.FromContext(ctx)
db, err := provider.GetDatabase(d.Name, d.DBType)
if err != nil {
panic(err.Error())
}
26 changes: 5 additions & 21 deletions go-runtime/ftl/ftltest/ftltest.go
Original file line number Diff line number Diff line change
@@ -6,9 +6,6 @@ import (
"fmt"
"reflect"

"github.com/alecthomas/types/optional"

cf "github.com/TBD54566975/ftl/common/configuration"
"github.com/TBD54566975/ftl/go-runtime/ftl"
"github.com/TBD54566975/ftl/go-runtime/modulecontext"
"github.com/TBD54566975/ftl/internal/log"
@@ -17,15 +14,12 @@ import (
// Context suitable for use in testing FTL verbs with provided options
func Context(options ...func(context.Context) error) context.Context {
ctx := log.ContextWithNewDefaultLogger(context.Background())
context, err := modulecontext.FromEnvironment(ctx, ftl.Module())
context, err := modulecontext.FromEnvironment(ctx, ftl.Module(), true)
if err != nil {
panic(err)
}
ctx = context.ApplyToContext(ctx)

mockProvider := newMockVerbProvider()
ctx = ftl.ApplyCallOverriderToContext(ctx, mockProvider)

for _, option := range options {
err = option(ctx)
if err != nil {
@@ -49,8 +43,7 @@ func WithConfig[T ftl.ConfigType](config ftl.ConfigValue[T], value T) func(conte
if config.Module != ftl.Module() {
return fmt.Errorf("config %v does not match current module %s", config.Module, ftl.Module())
}
cm := cf.ConfigFromContext(ctx)
return cm.Set(ctx, cf.Ref{Module: optional.Some(config.Module), Name: config.Name}, value)
return modulecontext.FromContext(ctx).SetConfig(config.Name, value)
}
}

@@ -68,8 +61,7 @@ func WithSecret[T ftl.SecretType](secret ftl.SecretValue[T], value T) func(conte
if secret.Module != ftl.Module() {
return fmt.Errorf("secret %v does not match current module %s", secret.Module, ftl.Module())
}
sm := cf.SecretsFromContext(ctx)
return sm.Set(ctx, cf.Ref{Module: optional.Some(secret.Module), Name: secret.Name}, value)
return modulecontext.FromContext(ctx).SetSecret(secret.Name, value)
}
}

@@ -87,21 +79,13 @@ func WithSecret[T ftl.SecretType](secret ftl.SecretValue[T], value T) func(conte
func WhenVerb[Req any, Resp any](verb ftl.Verb[Req, Resp], fake func(ctx context.Context, req Req) (resp Resp, err error)) func(context.Context) error {
return func(ctx context.Context) error {
ref := ftl.FuncRef(verb)
overrider, ok := ftl.CallOverriderFromContext(ctx)
if !ok {
return fmt.Errorf("could not override %v with a fake, context not set up with call overrider", ref)
}
mockProvider, ok := overrider.(*mockVerbProvider)
if !ok {
return fmt.Errorf("could not override %v with a fake, call overrider is not a MockProvider", ref)
}
mockProvider.mocks[ref] = func(ctx context.Context, req any) (resp any, err error) {
modulecontext.FromContext(ctx).SetMockVerb(modulecontext.Ref(ref), func(ctx context.Context, req any) (resp any, err error) {
request, ok := req.(Req)
if !ok {
return nil, fmt.Errorf("invalid request type %T for %v, expected %v", req, ref, reflect.TypeFor[Req]())
}
return fake(ctx, request)
}
})
return nil
}
}
Loading

0 comments on commit 247e043

Please sign in to comment.