diff --git a/backend/controller/admin/cmd_client.go b/backend/controller/admin/cmd_client.go new file mode 100644 index 000000000..4b2d746ac --- /dev/null +++ b/backend/controller/admin/cmd_client.go @@ -0,0 +1,76 @@ +package admin + +import ( + "context" + "errors" + "net" + "net/url" + + "connectrpc.com/connect" + ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" + "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" +) + +type CmdClient interface { + Ping(ctx context.Context, req *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error) + + // List configuration. + ConfigList(ctx context.Context, req *connect.Request[ftlv1.ListConfigRequest]) (*connect.Response[ftlv1.ListConfigResponse], error) + + // Get a config value. + ConfigGet(ctx context.Context, req *connect.Request[ftlv1.GetConfigRequest]) (*connect.Response[ftlv1.GetConfigResponse], error) + + // Set a config value. + ConfigSet(ctx context.Context, req *connect.Request[ftlv1.SetConfigRequest]) (*connect.Response[ftlv1.SetConfigResponse], error) + + // Unset a config value. + ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetConfigRequest]) (*connect.Response[ftlv1.UnsetConfigResponse], error) + + // List secrets. + SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error) + + // Get a secret. + SecretGet(ctx context.Context, req *connect.Request[ftlv1.GetSecretRequest]) (*connect.Response[ftlv1.GetSecretResponse], error) + + // Set a secret. + SecretSet(ctx context.Context, req *connect.Request[ftlv1.SetSecretRequest]) (*connect.Response[ftlv1.SetSecretResponse], error) + + // Unset a secret. + SecretUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetSecretRequest]) (*connect.Response[ftlv1.UnsetSecretResponse], error) +} + +// NewCmdClient takes the service client and endpoint flag received by the cmd interface +// and returns an appropriate interface for the cmd library to use. +// +// If the controller is not present AND endpoint is local, then inject a purely-local +// implementation of the interface so that the user does not need to spin up a controller +// just to run the `ftl config/secret` commands. Otherwise, return back the gRPC client. +func NewCmdClient(ctx context.Context, adminClient ftlv1connect.AdminServiceClient, endpoint *url.URL) (CmdClient, error) { + _, err := adminClient.Ping(ctx, connect.NewRequest(&ftlv1.PingRequest{})) + if isConnectUnavailableError(err) && isEndpointLocal(endpoint) { + return newLocalCmdClient(ctx), nil + } + return adminClient, nil +} + +func isConnectUnavailableError(err error) bool { + var connectErr *connect.Error + if errors.As(err, &connectErr) { + return connectErr.Code() == connect.CodeUnavailable + } + return false +} + +func isEndpointLocal(endpoint *url.URL) bool { + h := endpoint.Hostname() + ips, err := net.LookupIP(h) + if err != nil { + panic(err.Error()) + } + for _, netip := range ips { + if netip.IsLoopback() { + return true + } + } + return false +} diff --git a/backend/controller/admin/cmd_client_test.go b/backend/controller/admin/cmd_client_test.go new file mode 100644 index 000000000..4250c5a95 --- /dev/null +++ b/backend/controller/admin/cmd_client_test.go @@ -0,0 +1,45 @@ +package admin + +import ( + "net/url" + "testing" + + "github.com/alecthomas/assert/v2" +) + +func TestIsEndpointLocal(t *testing.T) { + tests := []struct { + Name string + Endpoint string + Want bool + }{ + { + Name: "DefaultLocalhost", + Endpoint: "http://localhost:8892", + Want: true, + }, + { + Name: "NumericLocalhost", + Endpoint: "http://127.0.0.1:8892", + Want: true, + }, + { + Name: "TooLow", + Endpoint: "http://126.255.255.255:8892", + Want: false, + }, + { + Name: "TooHigh", + Endpoint: "http://128.0.0.1:8892", + Want: false, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + u, err := url.Parse(test.Endpoint) + assert.NoError(t, err) + assert.Equal(t, isEndpointLocal(u), test.Want) + }) + } +} diff --git a/backend/controller/admin/local_cmd_client.go b/backend/controller/admin/local_cmd_client.go new file mode 100644 index 000000000..7e3b747f2 --- /dev/null +++ b/backend/controller/admin/local_cmd_client.go @@ -0,0 +1,59 @@ +package admin + +import ( + "context" + + "connectrpc.com/connect" + ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" + "github.com/TBD54566975/ftl/common/configuration" +) + +// localCmdClient reads and writes to local projectconfig files without making any network +// calls. It allows us to interface with local ftl-project.toml files without needing to +// start a controller. +type localCmdClient struct { + as *AdminService +} + +func newLocalCmdClient(ctx context.Context) *localCmdClient { + cm := configuration.ConfigFromContext(ctx) + sm := configuration.SecretsFromContext(ctx) + return &localCmdClient{NewAdminService(cm, sm)} +} + +// Ping will always return a healthy response because localCmdClient is purely local. +func (l *localCmdClient) Ping(ctx context.Context, req *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error) { + return connect.NewResponse(&ftlv1.PingResponse{}), nil +} + +func (l *localCmdClient) ConfigList(ctx context.Context, req *connect.Request[ftlv1.ListConfigRequest]) (*connect.Response[ftlv1.ListConfigResponse], error) { + return l.as.ConfigList(ctx, req) +} + +func (l *localCmdClient) ConfigGet(ctx context.Context, req *connect.Request[ftlv1.GetConfigRequest]) (*connect.Response[ftlv1.GetConfigResponse], error) { + return l.as.ConfigGet(ctx, req) +} + +func (l *localCmdClient) ConfigSet(ctx context.Context, req *connect.Request[ftlv1.SetConfigRequest]) (*connect.Response[ftlv1.SetConfigResponse], error) { + return l.as.ConfigSet(ctx, req) +} + +func (l *localCmdClient) ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetConfigRequest]) (*connect.Response[ftlv1.UnsetConfigResponse], error) { + return l.as.ConfigUnset(ctx, req) +} + +func (l *localCmdClient) SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error) { + return l.as.SecretsList(ctx, req) +} + +func (l *localCmdClient) SecretGet(ctx context.Context, req *connect.Request[ftlv1.GetSecretRequest]) (*connect.Response[ftlv1.GetSecretResponse], error) { + return l.as.SecretGet(ctx, req) +} + +func (l *localCmdClient) SecretSet(ctx context.Context, req *connect.Request[ftlv1.SetSecretRequest]) (*connect.Response[ftlv1.SetSecretResponse], error) { + return l.as.SecretSet(ctx, req) +} + +func (l *localCmdClient) SecretUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetSecretRequest]) (*connect.Response[ftlv1.UnsetSecretResponse], error) { + return l.as.SecretUnset(ctx, req) +} diff --git a/cmd/ftl/cmd_config.go b/cmd/ftl/cmd_config.go index 460458901..e6db45629 100644 --- a/cmd/ftl/cmd_config.go +++ b/cmd/ftl/cmd_config.go @@ -9,8 +9,8 @@ import ( "connectrpc.com/connect" + "github.com/TBD54566975/ftl/backend/controller/admin" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" - "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" cf "github.com/TBD54566975/ftl/common/configuration" "github.com/alecthomas/types/optional" ) @@ -54,8 +54,8 @@ type configListCmd struct { Module string `optional:"" arg:"" placeholder:"MODULE" help:"List configuration only in this module."` } -func (s *configListCmd) Run(ctx context.Context, admin ftlv1connect.AdminServiceClient) error { - resp, err := admin.ConfigList(ctx, connect.NewRequest(&ftlv1.ListConfigRequest{ +func (s *configListCmd) Run(ctx context.Context, adminClient admin.CmdClient) error { + resp, err := adminClient.ConfigList(ctx, connect.NewRequest(&ftlv1.ListConfigRequest{ Module: &s.Module, IncludeValues: &s.Values, })) @@ -84,8 +84,8 @@ Returns a JSON-encoded configuration value. ` } -func (s *configGetCmd) Run(ctx context.Context, admin ftlv1connect.AdminServiceClient) error { - resp, err := admin.ConfigGet(ctx, connect.NewRequest(&ftlv1.GetConfigRequest{ +func (s *configGetCmd) Run(ctx context.Context, adminClient admin.CmdClient) error { + resp, err := adminClient.ConfigGet(ctx, connect.NewRequest(&ftlv1.GetConfigRequest{ Ref: configRefFromRef(s.Ref), })) if err != nil { @@ -107,7 +107,7 @@ type configSetCmd struct { Value *string `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""` } -func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, admin ftlv1connect.AdminServiceClient) error { +func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, adminClient admin.CmdClient) error { var err error var config []byte if s.Value != nil { @@ -135,7 +135,7 @@ func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, admin ftlv1conn if provider, ok := scmd.provider().Get(); ok { req.Provider = &provider } - _, err = admin.ConfigSet(ctx, connect.NewRequest(req)) + _, err = adminClient.ConfigSet(ctx, connect.NewRequest(req)) if err != nil { return err } @@ -146,14 +146,14 @@ type configUnsetCmd struct { Ref cf.Ref `arg:"" help:"Configuration reference in the form [.]."` } -func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, admin ftlv1connect.AdminServiceClient) error { +func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, adminClient admin.CmdClient) error { req := &ftlv1.UnsetConfigRequest{ Ref: configRefFromRef(s.Ref), } if provider, ok := scmd.provider().Get(); ok { req.Provider = &provider } - _, err := admin.ConfigUnset(ctx, connect.NewRequest(req)) + _, err := adminClient.ConfigUnset(ctx, connect.NewRequest(req)) if err != nil { return err } diff --git a/cmd/ftl/cmd_secret.go b/cmd/ftl/cmd_secret.go index ee4b52aab..5173f3025 100644 --- a/cmd/ftl/cmd_secret.go +++ b/cmd/ftl/cmd_secret.go @@ -13,8 +13,8 @@ import ( "github.com/mattn/go-isatty" "golang.org/x/term" + "github.com/TBD54566975/ftl/backend/controller/admin" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" - "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" cf "github.com/TBD54566975/ftl/common/configuration" ) @@ -58,8 +58,8 @@ type secretListCmd struct { Module string `optional:"" arg:"" placeholder:"MODULE" help:"List secrets only in this module."` } -func (s *secretListCmd) Run(ctx context.Context, admin ftlv1connect.AdminServiceClient) error { - resp, err := admin.SecretsList(ctx, connect.NewRequest(&ftlv1.ListSecretsRequest{ +func (s *secretListCmd) Run(ctx context.Context, adminClient admin.CmdClient) error { + resp, err := adminClient.SecretsList(ctx, connect.NewRequest(&ftlv1.ListSecretsRequest{ Module: &s.Module, IncludeValues: &s.Values, })) @@ -87,8 +87,8 @@ Returns a JSON-encoded secret value. ` } -func (s *secretGetCmd) Run(ctx context.Context, admin ftlv1connect.AdminServiceClient) error { - resp, err := admin.SecretGet(ctx, connect.NewRequest(&ftlv1.GetSecretRequest{ +func (s *secretGetCmd) Run(ctx context.Context, adminClient admin.CmdClient) error { + resp, err := adminClient.SecretGet(ctx, connect.NewRequest(&ftlv1.GetSecretRequest{ Ref: configRefFromRef(s.Ref), })) if err != nil { @@ -108,7 +108,7 @@ type secretSetCmd struct { Ref cf.Ref `arg:"" help:"Secret reference in the form [.]."` } -func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd, admin ftlv1connect.AdminServiceClient) error { +func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.CmdClient) error { // Prompt for a secret if stdin is a terminal, otherwise read from stdin. var err error var secret []byte @@ -142,7 +142,7 @@ func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd, admin ftlv1conn if provider, ok := scmd.provider().Get(); ok { req.Provider = &provider } - _, err = admin.SecretSet(ctx, connect.NewRequest(req)) + _, err = adminClient.SecretSet(ctx, connect.NewRequest(req)) if err != nil { return err } @@ -153,14 +153,14 @@ type secretUnsetCmd struct { Ref cf.Ref `arg:"" help:"Secret reference in the form [.]."` } -func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, admin ftlv1connect.AdminServiceClient) error { +func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.CmdClient) error { req := &ftlv1.UnsetSecretRequest{ Ref: configRefFromRef(s.Ref), } if provider, ok := scmd.provider().Get(); ok { req.Provider = &provider } - _, err := admin.SecretUnset(ctx, connect.NewRequest(req)) + _, err := adminClient.SecretUnset(ctx, connect.NewRequest(req)) if err != nil { return err } diff --git a/cmd/ftl/main.go b/cmd/ftl/main.go index 879b089e0..4fff90414 100644 --- a/cmd/ftl/main.go +++ b/cmd/ftl/main.go @@ -14,6 +14,7 @@ import ( kongtoml "github.com/alecthomas/kong-toml" "github.com/TBD54566975/ftl" + "github.com/TBD54566975/ftl/backend/controller/admin" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" cf "github.com/TBD54566975/ftl/common/configuration" "github.com/TBD54566975/ftl/common/projectconfig" @@ -121,7 +122,8 @@ func main() { adminServiceClient := rpc.Dial(ftlv1connect.NewAdminServiceClient, cli.Endpoint.String(), log.Error) ctx = rpc.ContextWithClient(ctx, adminServiceClient) - kctx.BindTo(adminServiceClient, (*ftlv1connect.AdminServiceClient)(nil)) + adminClient, err := admin.NewCmdClient(ctx, adminServiceClient, cli.Endpoint) + kctx.BindTo(adminClient, (*admin.CmdClient)(nil)) controllerServiceClient := rpc.Dial(ftlv1connect.NewControllerServiceClient, cli.Endpoint.String(), log.Error) ctx = rpc.ContextWithClient(ctx, controllerServiceClient) diff --git a/common/projectconfig/integration_test.go b/common/projectconfig/integration_test.go index 36bd17261..8414b7e26 100644 --- a/common/projectconfig/integration_test.go +++ b/common/projectconfig/integration_test.go @@ -43,3 +43,9 @@ func TestDefaultToRootWhenModuleDirsMissing(t *testing.T) { }), ) } + +func TestConfigCmdWithoutController(t *testing.T) { + in.RunWithoutController(t, "configs-ftl-project.toml", + in.ExecWithExpectedOutput("value\n", "ftl", "config", "get", "key"), + ) +} diff --git a/common/projectconfig/testdata/go/configs-ftl-project.toml b/common/projectconfig/testdata/go/configs-ftl-project.toml new file mode 100644 index 000000000..06f80b7af --- /dev/null +++ b/common/projectconfig/testdata/go/configs-ftl-project.toml @@ -0,0 +1,3 @@ +[global] + [global.configuration] + key = "inline://InZhbHVlIg" diff --git a/integration/actions.go b/integration/actions.go index d080a2d23..531156020 100644 --- a/integration/actions.go +++ b/integration/actions.go @@ -120,6 +120,17 @@ func Exec(cmd string, args ...string) Action { } } +// ExecWithExpectedOutput runs a command from the test working directory. +// The output is captured and is compared with the expected output. +func ExecWithExpectedOutput(want string, cmd string, args ...string) Action { + return func(t testing.TB, ic TestContext) { + Infof("Executing: %s %s", cmd, shellquote.Join(args...)) + output, err := ftlexec.Capture(ic, ic.workDir, cmd, args...) + assert.NoError(t, err) + assert.Equal(t, output, []byte(want)) + } +} + // ExecWithOutput runs a command from the test working directory. // The output is captured and is returned as part of the error. func ExecWithOutput(cmd string, args ...string) Action { diff --git a/integration/harness.go b/integration/harness.go index 43802f4d0..0b7e8b3a8 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -49,6 +49,21 @@ var buildOnce sync.Once // "database/ftl-project.toml" would set FTL_CONFIG to // "integration/testdata/go/database/ftl-project.toml"). func Run(t *testing.T, ftlConfigPath string, actions ...Action) { + run(t, ftlConfigPath, true, actions...) +} + +// RunWithoutController runs an integration test without starting the controller. +// ftlConfigPath: if FTL_CONFIG should be set for this test, then pass in the relative +// +// path based on ./testdata/go/ where "." denotes the directory containing the +// integration test (e.g. for "integration/harness_test.go" supplying +// "database/ftl-project.toml" would set FTL_CONFIG to +// "integration/testdata/go/database/ftl-project.toml"). +func RunWithoutController(t *testing.T, ftlConfigPath string, actions ...Action) { + run(t, ftlConfigPath, false, actions...) +} + +func run(t *testing.T, ftlConfigPath string, startController bool, actions ...Action) { tmpDir := t.TempDir() cwd, err := os.Getwd() @@ -76,27 +91,34 @@ func Run(t *testing.T, ftlConfigPath string, actions ...Action) { assert.NoError(t, err) }) - controller := rpc.Dial(ftlv1connect.NewControllerServiceClient, "http://localhost:8892", log.Debug) verbs := rpc.Dial(ftlv1connect.NewVerbServiceClient, "http://localhost:8892", log.Debug) - Infof("Starting ftl cluster") - ctx = startProcess(ctx, t, filepath.Join(binDir, "ftl"), "serve", "--recreate") + var controller ftlv1connect.ControllerServiceClient + if startController { + controller = rpc.Dial(ftlv1connect.NewControllerServiceClient, "http://localhost:8892", log.Debug) + + Infof("Starting ftl cluster") + ctx = startProcess(ctx, t, filepath.Join(binDir, "ftl"), "serve", "--recreate") + } ic := TestContext{ - Context: ctx, - rootDir: rootDir, - testData: filepath.Join(cwd, "testdata", "go"), - workDir: tmpDir, - binDir: binDir, - Controller: controller, - Verbs: verbs, + Context: ctx, + rootDir: rootDir, + testData: filepath.Join(cwd, "testdata", "go"), + workDir: tmpDir, + binDir: binDir, + Verbs: verbs, } - Infof("Waiting for controller to be ready") - ic.AssertWithRetry(t, func(t testing.TB, ic TestContext) { - _, err := ic.Controller.Status(ic, connect.NewRequest(&ftlv1.StatusRequest{})) - assert.NoError(t, err) - }) + if startController { + ic.Controller = controller + + Infof("Waiting for controller to be ready") + ic.AssertWithRetry(t, func(t testing.TB, ic TestContext) { + _, err := ic.Controller.Status(ic, connect.NewRequest(&ftlv1.StatusRequest{})) + assert.NoError(t, err) + }) + } Infof("Starting test")