fix: re-enable ftl config/secret working without a running controll…

…er (#1695)

Fixes #1677

Without starting a controller first, you can once again run `ftl
config/secret` commands:
```
$ ftl config list
key
echo.default
```

backend/controller/admin/client.go

@@ -0,0 +1,82 @@
+package admin
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/url"
+
+	"connectrpc.com/connect"
+	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
+	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
+)
+
+// Client standardizes an common interface between the AdminService as accessed via gRPC
+// and a purely-local variant that doesn't require a running controller to access.
+type Client interface {
+	Ping(ctx context.Context, req *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error)
+
+	// List configuration.
+	ConfigList(ctx context.Context, req *connect.Request[ftlv1.ListConfigRequest]) (*connect.Response[ftlv1.ListConfigResponse], error)
+
+	// Get a config value.
+	ConfigGet(ctx context.Context, req *connect.Request[ftlv1.GetConfigRequest]) (*connect.Response[ftlv1.GetConfigResponse], error)
+
+	// Set a config value.
+	ConfigSet(ctx context.Context, req *connect.Request[ftlv1.SetConfigRequest]) (*connect.Response[ftlv1.SetConfigResponse], error)
+
+	// Unset a config value.
+	ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetConfigRequest]) (*connect.Response[ftlv1.UnsetConfigResponse], error)
+
+	// List secrets.
+	SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error)
+
+	// Get a secret.
+	SecretGet(ctx context.Context, req *connect.Request[ftlv1.GetSecretRequest]) (*connect.Response[ftlv1.GetSecretResponse], error)
+
+	// Set a secret.
+	SecretSet(ctx context.Context, req *connect.Request[ftlv1.SetSecretRequest]) (*connect.Response[ftlv1.SetSecretResponse], error)
+
+	// Unset a secret.
+	SecretUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetSecretRequest]) (*connect.Response[ftlv1.UnsetSecretResponse], error)
+}
+
+// NewClient takes the service client and endpoint flag received by the cmd interface
+// and returns an appropriate interface for the cmd library to use.
+//
+// If the controller is not present AND endpoint is local, then inject a purely-local
+// implementation of the interface so that the user does not need to spin up a controller
+// just to run the `ftl config/secret` commands. Otherwise, return back the gRPC client.
+func NewClient(ctx context.Context, adminClient ftlv1connect.AdminServiceClient, endpoint *url.URL) (Client, error) {
+	isLocal, err := isEndpointLocal(endpoint)
+	if err != nil {
+		return adminClient, err
+	}
+	_, err = adminClient.Ping(ctx, connect.NewRequest(&ftlv1.PingRequest{}))
+	if isConnectUnavailableError(err) && isLocal {
+		return newLocalClient(ctx), nil
+	}
+	return adminClient, nil
+}
+
+func isConnectUnavailableError(err error) bool {
+	var connectErr *connect.Error
+	if errors.As(err, &connectErr) {
+		return connectErr.Code() == connect.CodeUnavailable
+	}
+	return false
+}
+
+func isEndpointLocal(endpoint *url.URL) (bool, error) {
+	h := endpoint.Hostname()
+	ips, err := net.LookupIP(h)
+	if err != nil {
+		return false, err
+	}
+	for _, netip := range ips {
+		if netip.IsLoopback() {
+			return true, nil
+		}
+	}
+	return false, nil
+}

backend/controller/admin/client_test.go

@@ -0,0 +1,47 @@
+package admin
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/alecthomas/assert/v2"
+)
+
+func TestIsEndpointLocal(t *testing.T) {
+	tests := []struct {
+		Name     string
+		Endpoint string
+		Want     bool
+	}{
+		{
+			Name:     "DefaultLocalhost",
+			Endpoint: "http://localhost:8892",
+			Want:     true,
+		},
+		{
+			Name:     "NumericLocalhost",
+			Endpoint: "http://127.0.0.1:8892",
+			Want:     true,
+		},
+		{
+			Name:     "TooLow",
+			Endpoint: "http://126.255.255.255:8892",
+			Want:     false,
+		},
+		{
+			Name:     "TooHigh",
+			Endpoint: "http://128.0.0.1:8892",
+			Want:     false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			u, err := url.Parse(test.Endpoint)
+			assert.NoError(t, err)
+			got, err := isEndpointLocal(u)
+			assert.NoError(t, err)
+			assert.Equal(t, got, test.Want)
+		})
+	}
+}

backend/controller/admin/local_client.go

@@ -0,0 +1,20 @@
+package admin
+
+import (
+	"context"
+
+	"github.com/TBD54566975/ftl/common/configuration"
+)
+
+// localClient reads and writes to local projectconfig files without making any network
+// calls. It allows us to interface with local ftl-project.toml files without needing to
+// start a controller.
+type localClient struct {
+	*AdminService
+}
+
+func newLocalClient(ctx context.Context) *localClient {
+	cm := configuration.ConfigFromContext(ctx)
+	sm := configuration.SecretsFromContext(ctx)
+	return &localClient{NewAdminService(cm, sm)}
+}

cmd/ftl/cmd_config.go

@@ -9,8 +9,8 @@ import (
 
 	"connectrpc.com/connect"
 
+	"github.com/TBD54566975/ftl/backend/controller/admin"
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
-	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	cf "github.com/TBD54566975/ftl/common/configuration"
 	"github.com/alecthomas/types/optional"
 )
@@ -54,8 +54,8 @@ type configListCmd struct {
 	Module string `optional:"" arg:"" placeholder:"MODULE" help:"List configuration only in this module."`
 }
 
-func (s *configListCmd) Run(ctx context.Context, admin ftlv1connect.AdminServiceClient) error {
-	resp, err := admin.ConfigList(ctx, connect.NewRequest(&ftlv1.ListConfigRequest{
+func (s *configListCmd) Run(ctx context.Context, adminClient admin.Client) error {
+	resp, err := adminClient.ConfigList(ctx, connect.NewRequest(&ftlv1.ListConfigRequest{
 		Module:        &s.Module,
 		IncludeValues: &s.Values,
 	}))
@@ -84,8 +84,8 @@ Returns a JSON-encoded configuration value.
 `
 }
 
-func (s *configGetCmd) Run(ctx context.Context, admin ftlv1connect.AdminServiceClient) error {
-	resp, err := admin.ConfigGet(ctx, connect.NewRequest(&ftlv1.GetConfigRequest{
+func (s *configGetCmd) Run(ctx context.Context, adminClient admin.Client) error {
+	resp, err := adminClient.ConfigGet(ctx, connect.NewRequest(&ftlv1.GetConfigRequest{
 		Ref: configRefFromRef(s.Ref),
 	}))
 	if err != nil {
@@ -107,7 +107,7 @@ type configSetCmd struct {
 	Value *string `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""`
 }
 
-func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, admin ftlv1connect.AdminServiceClient) error {
+func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, adminClient admin.Client) error {
 	var err error
 	var config []byte
 	if s.Value != nil {
@@ -135,7 +135,7 @@ func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd, admin ftlv1conn
 	if provider, ok := scmd.provider().Get(); ok {
 		req.Provider = &provider
 	}
-	_, err = admin.ConfigSet(ctx, connect.NewRequest(req))
+	_, err = adminClient.ConfigSet(ctx, connect.NewRequest(req))
 	if err != nil {
 		return err
 	}
@@ -146,14 +146,14 @@ type configUnsetCmd struct {
 	Ref cf.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
 }
 
-func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, admin ftlv1connect.AdminServiceClient) error {
+func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, adminClient admin.Client) error {
 	req := &ftlv1.UnsetConfigRequest{
 		Ref: configRefFromRef(s.Ref),
 	}
 	if provider, ok := scmd.provider().Get(); ok {
 		req.Provider = &provider
 	}
-	_, err := admin.ConfigUnset(ctx, connect.NewRequest(req))
+	_, err := adminClient.ConfigUnset(ctx, connect.NewRequest(req))
 	if err != nil {
 		return err
 	}

cmd/ftl/cmd_secret.go

@@ -13,8 +13,8 @@ import (
 	"github.com/mattn/go-isatty"
 	"golang.org/x/term"
 
+	"github.com/TBD54566975/ftl/backend/controller/admin"
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
-	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	cf "github.com/TBD54566975/ftl/common/configuration"
 )
 
@@ -58,8 +58,8 @@ type secretListCmd struct {
 	Module string `optional:"" arg:"" placeholder:"MODULE" help:"List secrets only in this module."`
 }
 
-func (s *secretListCmd) Run(ctx context.Context, admin ftlv1connect.AdminServiceClient) error {
-	resp, err := admin.SecretsList(ctx, connect.NewRequest(&ftlv1.ListSecretsRequest{
+func (s *secretListCmd) Run(ctx context.Context, adminClient admin.Client) error {
+	resp, err := adminClient.SecretsList(ctx, connect.NewRequest(&ftlv1.ListSecretsRequest{
 		Module:        &s.Module,
 		IncludeValues: &s.Values,
 	}))
@@ -87,8 +87,8 @@ Returns a JSON-encoded secret value.
 `
 }
 
-func (s *secretGetCmd) Run(ctx context.Context, admin ftlv1connect.AdminServiceClient) error {
-	resp, err := admin.SecretGet(ctx, connect.NewRequest(&ftlv1.GetSecretRequest{
+func (s *secretGetCmd) Run(ctx context.Context, adminClient admin.Client) error {
+	resp, err := adminClient.SecretGet(ctx, connect.NewRequest(&ftlv1.GetSecretRequest{
 		Ref: configRefFromRef(s.Ref),
 	}))
 	if err != nil {
@@ -108,7 +108,7 @@ type secretSetCmd struct {
 	Ref  cf.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
 }
 
-func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd, admin ftlv1connect.AdminServiceClient) error {
+func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.Client) error {
 	// Prompt for a secret if stdin is a terminal, otherwise read from stdin.
 	var err error
 	var secret []byte
@@ -142,7 +142,7 @@ func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd, admin ftlv1conn
 	if provider, ok := scmd.provider().Get(); ok {
 		req.Provider = &provider
 	}
-	_, err = admin.SecretSet(ctx, connect.NewRequest(req))
+	_, err = adminClient.SecretSet(ctx, connect.NewRequest(req))
 	if err != nil {
 		return err
 	}
@@ -153,14 +153,14 @@ type secretUnsetCmd struct {
 	Ref cf.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
 }
 
-func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, admin ftlv1connect.AdminServiceClient) error {
+func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.Client) error {
 	req := &ftlv1.UnsetSecretRequest{
 		Ref: configRefFromRef(s.Ref),
 	}
 	if provider, ok := scmd.provider().Get(); ok {
 		req.Provider = &provider
 	}
-	_, err := admin.SecretUnset(ctx, connect.NewRequest(req))
+	_, err := adminClient.SecretUnset(ctx, connect.NewRequest(req))
 	if err != nil {
 		return err
 	}

cmd/ftl/main.go

@@ -14,6 +14,7 @@ import (
 	kongtoml "github.com/alecthomas/kong-toml"
 
 	"github.com/TBD54566975/ftl"
+	"github.com/TBD54566975/ftl/backend/controller/admin"
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	cf "github.com/TBD54566975/ftl/common/configuration"
 	"github.com/TBD54566975/ftl/common/projectconfig"
@@ -121,7 +122,9 @@ func main() {
 
 	adminServiceClient := rpc.Dial(ftlv1connect.NewAdminServiceClient, cli.Endpoint.String(), log.Error)
 	ctx = rpc.ContextWithClient(ctx, adminServiceClient)
-	kctx.BindTo(adminServiceClient, (*ftlv1connect.AdminServiceClient)(nil))
+	adminClient, err := admin.NewClient(ctx, adminServiceClient, cli.Endpoint)
+	kctx.FatalIfErrorf(err)
+	kctx.BindTo(adminClient, (*admin.Client)(nil))
 
 	controllerServiceClient := rpc.Dial(ftlv1connect.NewControllerServiceClient, cli.Endpoint.String(), log.Error)
 	ctx = rpc.ContextWithClient(ctx, controllerServiceClient)

common/projectconfig/integration_test.go

@@ -43,3 +43,9 @@ func TestDefaultToRootWhenModuleDirsMissing(t *testing.T) {
 		}),
 	)
 }
+
+func TestConfigCmdWithoutController(t *testing.T) {
+	in.RunWithoutController(t, "configs-ftl-project.toml",
+		in.ExecWithExpectedOutput("value\n", "ftl", "config", "get", "key"),
+	)
+}

common/projectconfig/testdata/go/configs-ftl-project.toml

@@ -0,0 +1,3 @@
+[global]
+  [global.configuration]
+    key = "inline://InZhbHVlIg"

integration/actions.go

@@ -120,6 +120,17 @@ func Exec(cmd string, args ...string) Action {
 	}
 }
 
+// ExecWithExpectedOutput runs a command from the test working directory.
+// The output is captured and is compared with the expected output.
+func ExecWithExpectedOutput(want string, cmd string, args ...string) Action {
+	return func(t testing.TB, ic TestContext) {
+		Infof("Executing: %s %s", cmd, shellquote.Join(args...))
+		output, err := ftlexec.Capture(ic, ic.workDir, cmd, args...)
+		assert.NoError(t, err)
+		assert.Equal(t, output, []byte(want))
+	}
+}
+
 // ExecWithOutput runs a command from the test working directory.
 // The output is captured and is returned as part of the error.
 func ExecWithOutput(cmd string, args ...string) Action {