feat: allow importing and exporting of secrets

backend/controller/admin/admin.go

@@ -127,8 +127,12 @@ func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftl
 	}
 	secrets := []*ftlv1.ListSecretsResponse_Secret{}
 	for _, secret := range listing {
+		if req.Msg.Provider != nil && s.sm.ProviderKeyForAccessor(secret.Accessor) != secretProviderKey(req.Msg.Provider) {
+			// Skip secrets that don't match the provider in the request
+			continue
+		}
 		module, ok := secret.Module.Get()
-		if *req.Msg.Module != "" && module != *req.Msg.Module {
+		if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module {
 			continue
 		}
 		ref := secret.Name

backend/protos/xyz/block/ftl/v1/ftl.proto

@@ -463,6 +463,7 @@ enum SecretProvider {
 message ListSecretsRequest {
   optional string module = 1;
   optional bool include_values = 2;
+  optional SecretProvider provider = 3;
 }
 message ListSecretsResponse {
   message Secret {

cmd/ftl/cmd_secret.go

@@ -19,10 +19,12 @@ import (
 )
 
 type secretCmd struct {
-	List  secretListCmd  `cmd:"" help:"List secrets."`
-	Get   secretGetCmd   `cmd:"" help:"Get a secret."`
-	Set   secretSetCmd   `cmd:"" help:"Set a secret."`
-	Unset secretUnsetCmd `cmd:"" help:"Unset a secret."`
+	List   secretListCmd   `cmd:"" help:"List secrets."`
+	Get    secretGetCmd    `cmd:"" help:"Get a secret."`
+	Set    secretSetCmd    `cmd:"" help:"Set a secret."`
+	Unset  secretUnsetCmd  `cmd:"" help:"Unset a secret."`
+	Import secretImportCmd `cmd:"" help:"Import secrets."`
+	Export secretExportCmd `cmd:"" help:"Export secrets."`
 
 	Envar    bool `help:"Write configuration as environment variables." group:"Provider:" xor:"secretwriter"`
 	Inline   bool `help:"Write values inline in the configuration file." group:"Provider:" xor:"secretwriter"`
@@ -169,3 +171,91 @@ func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, adminClient a
 	}
 	return nil
 }
+
+type secretImportCmd struct {
+	Input *string `arg:"" placeholder:"JSON" help:"JSON to import as secrets (read from stdin if omitted). Format: [{\"ref\":\"<module>.<name>\",\"value\": <secret>}, ...]" optional:""`
+}
+
+func (s *secretImportCmd) Help() string {
+	return `
+Imports secrets from a JSON array.
+`
+}
+
+func (s *secretImportCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.Client) error {
+	var input []byte
+	var err error
+	if s.Input != nil {
+		input = []byte(*s.Input)
+	} else {
+		input, err = io.ReadAll(os.Stdin)
+		if err != nil {
+			return fmt.Errorf("failed to read config from stdin: %w", err)
+		}
+	}
+	var entries map[string]any
+	err = json.Unmarshal(input, &entries)
+	if err != nil {
+		return fmt.Errorf("could not parse JSON: %w", err)
+	}
+	for refPath, value := range entries {
+		ref, err := cf.ParseRef(refPath)
+		if err != nil {
+			return fmt.Errorf("could not parse ref %q: %w", refPath, err)
+		}
+		bytes, err := json.Marshal(value)
+		if err != nil {
+			return fmt.Errorf("could not marshal value for %q: %w", refPath, err)
+		}
+		req := &ftlv1.SetSecretRequest{
+			Ref:   configRefFromRef(ref),
+			Value: bytes,
+		}
+		if provider, ok := scmd.provider().Get(); ok {
+			req.Provider = &provider
+		}
+		_, err = adminClient.SecretSet(ctx, connect.NewRequest(req))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type secretExportCmd struct {
+}
+
+func (s *secretExportCmd) Help() string {
+	return `
+Outputs secrets in a JSON array. A provider can be used to filter which secrets are included.
+`
+}
+
+func (s *secretExportCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.Client) error {
+	req := &ftlv1.ListSecretsRequest{
+		IncludeValues: optional.Some(true).Ptr(),
+	}
+	if provider, ok := scmd.provider().Get(); ok {
+		req.Provider = &provider
+	}
+	listResponse, err := adminClient.SecretsList(ctx, connect.NewRequest(req))
+	if err != nil {
+		return fmt.Errorf("could not retreive secrets: %w", err)
+	}
+	entries := make(map[string]any, 0)
+	for _, secret := range listResponse.Msg.Secrets {
+		var value any
+		err = json.Unmarshal(secret.Value, &value)
+		if err != nil {
+			return fmt.Errorf("could not export %q: %w", secret.RefPath, err)
+		}
+		entries[secret.RefPath] = value
+	}
+
+	output, err := json.Marshal(entries)
+	if err != nil {
+		return fmt.Errorf("could not build output: %w", err)
+	}
+	fmt.Println(string(output))
+	return nil
+}

cmd/ftl/integration_test.go

@@ -31,3 +31,41 @@ func TestBox(t *testing.T) {
 		Exec("docker", "compose", "-f", "echo-compose.yml", "down", "--rmi", "local"),
 	)
 }
+
+func TestSecretImportExport(t *testing.T) {
+	firstProjFile := "ftl-project.toml"
+	secondProjFile := "ftl-project-2.toml"
+
+	// use a pointer to keep track of the exported json so that i can be modified from within actions
+	blank := ""
+	exported := &blank
+
+	RunWithoutController(t, "",
+		// duplicate project file
+		Exec("cp", firstProjFile, secondProjFile),
+		// import into first project file
+		Exec("ftl", "secret", "import", "--inline", "--config", firstProjFile, `
+		{
+			"test.one": 1,
+			"test.two": "a string",
+			"test2.three": {"key":"value"}
+		}
+		`),
+		// export from first project file
+		ExecWithOutput("ftl", []string{"secret", "export", "--config", firstProjFile}, func(output string) {
+			*exported = output
+			// make sure the exported json contains a secret (otherwise the test could pass with the first import doing nothing)
+			assert.Contains(t, output, "test.one")
+		}),
+		// import into second project file
+		// wrapped in a func to avoid capturing the initial valye of *exported
+		func(t testing.TB, ic TestContext) {
+			Exec("ftl", "secret", "import", *exported, "--inline", "--config", secondProjFile)(t, ic)
+		},
+		// export from second project file
+		ExecWithOutput("ftl", []string{"secret", "export", "--config", secondProjFile}, func(output string) {
+			// check that both exported the same json
+			assert.Equal(t, *exported, output)
+		}),
+	)
+}

common/configuration/manager.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
 
@@ -75,6 +76,10 @@ func New[R Role](ctx context.Context, router Router[R], providers []Provider[R])
 	return m, nil
 }
 
+func (m *Manager[R]) ProviderKeyForAccessor(accessor *url.URL) string {
+	return accessor.Scheme
+}
+
 // getData returns a data value for a configuration from the active providers.
 // The data can be unmarshalled from JSON.
 func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) {
@@ -89,7 +94,7 @@ func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) {
 	} else if err != nil {
 		return nil, err
 	}
-	provider, ok := m.providers[key.Scheme]
+	provider, ok := m.providers[m.ProviderKeyForAccessor(key)]
 	if !ok {
 		return nil, fmt.Errorf("no provider for scheme %q", key.Scheme)
 	}

go-runtime/compile/compile_integration_test.go

@@ -15,9 +15,8 @@ func TestNonExportedDecls(t *testing.T) {
 		in.CopyModule("echo"),
 		in.Deploy("echo"),
 		in.CopyModule("notexportedverb"),
-		in.ExpectError(
-			in.ExecWithOutput("ftl", "deploy", "notexportedverb"),
-			"call first argument must be a function but is an unresolved reference to echo.Echo, does it need to be exported?"),
+		in.ExecWithExpectedError("call first argument must be a function but is an unresolved reference to echo.Echo, does it need to be exported?",
+			"ftl", "deploy", "notexportedverb"),
 	)
 }
 
@@ -28,8 +27,7 @@ func TestUndefinedExportedDecls(t *testing.T) {
 		in.CopyModule("echo"),
 		in.Deploy("echo"),
 		in.CopyModule("undefinedverb"),
-		in.ExpectError(
-			in.ExecWithOutput("ftl", "deploy", "undefinedverb"),
-			"call first argument must be a function but is an unresolved reference to echo.Undefined"),
+		in.ExecWithExpectedError("call first argument must be a function but is an unresolved reference to echo.Undefined",
+			"ftl", "deploy", "undefinedverb"),
 	)
 }

integration/actions.go

@@ -141,13 +141,25 @@ func ExecWithExpectedOutput(want string, cmd string, args ...string) Action {
 	}
 }
 
-// ExecWithOutput runs a command from the test working directory.
-// The output is captured and is returned as part of the error.
-func ExecWithOutput(cmd string, args ...string) Action {
+// ExecWithExpectedError runs a command from the test working directory.
+// The expected string must be contained in the error output
+func ExecWithExpectedError(want string, cmd string, args ...string) Action {
 	return func(t testing.TB, ic TestContext) {
 		Infof("Executing: %s %s", cmd, shellquote.Join(args...))
 		output, err := ftlexec.Capture(ic, ic.workDir, cmd, args...)
 		assert.NoError(t, err, "%s", string(output))
+		assert.Error(t, err)
+		assert.Contains(t, string(output), want)
+	}
+}
+
+// ExecWithOutput runs a command from the test working directory and calls the capture func with the result.
+func ExecWithOutput(cmd string, args []string, capture func(output string)) Action {
+	return func(t testing.TB, ic TestContext) {
+		Infof("Executing: %s %s", cmd, shellquote.Join(args...))
+		output, err := ftlexec.Capture(ic, ic.workDir, cmd, args...)
+		assert.NoError(t, err)
+		capture(string(output))
 	}
 }