diff --git a/buildengine/testdata/projects/another/go.sum b/buildengine/testdata/projects/another/go.sum index ef44aba8ee..a13ddbc3e4 100644 --- a/buildengine/testdata/projects/another/go.sum +++ b/buildengine/testdata/projects/another/go.sum @@ -128,8 +128,8 @@ modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/sqlite v1.29.9 h1:9RhNMklxJs+1596GNuAX+O/6040bvOwacTxuFcRuQow= -modernc.org/sqlite v1.29.9/go.mod h1:ItX2a1OVGgNsFh6Dv60JQvGfJfTPHPVpV6DF59akYOA= +modernc.org/sqlite v1.29.10 h1:3u93dz83myFnMilBGCOLbr+HjklS6+5rJLx4q86RDAg= +modernc.org/sqlite v1.29.10/go.mod h1:ItX2a1OVGgNsFh6Dv60JQvGfJfTPHPVpV6DF59akYOA= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/buildengine/testdata/projects/other/go.sum b/buildengine/testdata/projects/other/go.sum index ef44aba8ee..a13ddbc3e4 100644 --- a/buildengine/testdata/projects/other/go.sum +++ b/buildengine/testdata/projects/other/go.sum @@ -128,8 +128,8 @@ modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/sqlite v1.29.9 h1:9RhNMklxJs+1596GNuAX+O/6040bvOwacTxuFcRuQow= -modernc.org/sqlite v1.29.9/go.mod h1:ItX2a1OVGgNsFh6Dv60JQvGfJfTPHPVpV6DF59akYOA= +modernc.org/sqlite v1.29.10 h1:3u93dz83myFnMilBGCOLbr+HjklS6+5rJLx4q86RDAg= +modernc.org/sqlite v1.29.10/go.mod h1:ItX2a1OVGgNsFh6Dv60JQvGfJfTPHPVpV6DF59akYOA= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/common/configuration/1password_provider.go b/common/configuration/1password_provider.go index 45eaacda82..f699d05f65 100644 --- a/common/configuration/1password_provider.go +++ b/common/configuration/1password_provider.go @@ -2,23 +2,23 @@ package configuration import ( "context" - "encoding/base64" "encoding/json" + "errors" "fmt" + "github.com/kballard/go-shellquote" "net/url" "regexp" "strings" - "github.com/alecthomas/types/optional" - "github.com/TBD54566975/ftl/internal/exec" + "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/slices" ) // OnePasswordProvider is a configuration provider that reads passwords from // 1Password vaults via the "op" command line tool. type OnePasswordProvider struct { - OnePassword bool `name:"op" help:"Write 1Password secret references - does not write to 1Password." group:"Provider:" xor:"configwriter"` + Vault string `name:"op" help:"Store a secret in this 1Password vault. The name of the 1Password item will be the and the secret will be stored in the password field." group:"Provider:" xor:"configwriter" placeholder:"VAULT"` } var _ MutableProvider[Secrets] = OnePasswordProvider{} @@ -27,172 +27,154 @@ func (OnePasswordProvider) Role() Secrets { return func (o OnePasswordProvider) Key() string { return "op" } func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil } -// Load returns either a single field if the op:// reference specifies a field, or all fields if not. -// -// A single value/password: -// op://Personal/With Spaces/username -// op --format json item get --vault Personal "With Spaces" --fields=username -// { id, value, ... } -// "value" -// -// All fields: -// op://Personal/With Spaces -// op --format json item get --vault Personal "With Spaces" -// { fields: [ { id, value, ... } ], ... } -// { id: value, ... } +// Load returns the secret stored in 1password. func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { - _, err := exec.LookPath("op") - if err != nil { - return nil, fmt.Errorf("1Password CLI tool \"op\" not found: %w", err) + if err := checkOpBinary(); err != nil { + return nil, err } - decoded, err := base64.RawURLEncoding.DecodeString(key.Host) + vault := key.Host + full, err := getItem(ctx, vault, ref) if err != nil { - return nil, fmt.Errorf("1Password secret reference must be a base64 encoded string: %w", err) + return nil, fmt.Errorf("get item failed: %w", err) + } + + secret, ok := full.password() + if !ok { + return nil, fmt.Errorf("password field not found in item %q", ref) } - parsedRef, err := decodeSecretRef(string(decoded)) + // Just to verify that it is JSON encoded. + var decoded interface{} + err = json.Unmarshal(secret, &decoded) if err != nil { - return nil, fmt.Errorf("1Password secret reference invalid: %w", err) + return nil, fmt.Errorf("secret is not JSON encoded: %w", err) } - args := []string{"--format", "json", "item", "get", "--vault", parsedRef.Vault, parsedRef.Item} - v, fieldSpecified := parsedRef.Field.Get() - if fieldSpecified { - args = append(args, "--fields", v) + return secret, nil +} + +var vaultRegex = regexp.MustCompile(`^[a-zA-Z0-9_\-.]+$`) + +// Store will save the given secret in 1Password via the `op` command. +// +// op does not support "create or update" as a single command. Neither does it support specifying an ID on create. +// Because of this, we need check if the item exists before creating it, and update it if it does. +func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { + if err := checkOpBinary(); err != nil { + return nil, err } - output, err := exec.Capture(ctx, ".", "op", args...) - if err != nil { - return nil, fmt.Errorf("run `op` with args %v: %w", args, err) + if !vaultRegex.MatchString(o.Vault) { + return nil, fmt.Errorf("vault name %q contains invalid characters. a-z A-Z 0-9 _ . - are valid", o.Vault) } - if fieldSpecified { - v, err := decodeSingle(output) + url := &url.URL{Scheme: "op", Host: o.Vault} + + _, err := getItem(ctx, o.Vault, ref) + if errors.As(err, new(itemNotFoundError)) { + err = createItem(ctx, o.Vault, ref, value) if err != nil { - return nil, err + return nil, fmt.Errorf("create item failed: %w", err) } + return url, nil - return json.Marshal(v.Value) + } else if err != nil { + return nil, fmt.Errorf("get item failed: %w", err) } - full, err := decodeFull(output) + err = editItem(ctx, o.Vault, ref, value) if err != nil { - return nil, err - } - - // Filter out anything without a value - filtered := slices.Filter(full, func(e entry) bool { - return e.Value != "" - }) - // Map to id: value - var mapped = make(map[string]string, len(filtered)) - for _, e := range filtered { - mapped[e.ID] = e.Value + return nil, fmt.Errorf("edit item failed: %w", err) } - return json.Marshal(mapped) + return url, nil } -func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { - var opref string - if err := json.Unmarshal(value, &opref); err != nil { - return nil, fmt.Errorf("1Password value must be a JSON string containing a 1Password secret refererence: %w", err) - } - if !strings.HasPrefix(opref, "op://") { - return nil, fmt.Errorf("1Password secret reference must start with \"op://\"") +func (o OnePasswordProvider) Writer() bool { return o.Vault != "" } + +func checkOpBinary() error { + _, err := exec.LookPath("op") + if err != nil { + return fmt.Errorf("1Password CLI tool \"op\" not found: %w", err) } - encoded := base64.RawURLEncoding.EncodeToString([]byte(opref)) - return &url.URL{Scheme: "op", Host: encoded}, nil + return nil } -func (o OnePasswordProvider) Writer() bool { return o.OnePassword } +type itemNotFoundError struct { + vault string + ref Ref +} -type entry struct { - ID string `json:"id"` - Value string `json:"value"` +func (e itemNotFoundError) Error() string { + return fmt.Sprintf("item %q not found in vault %q", e.ref, e.vault) } -type fullResponse struct { +// item is the JSON response from `op item get`. +type item struct { Fields []entry `json:"fields"` } -// Decode a full item response from op -func decodeFull(output []byte) ([]entry, error) { - var full fullResponse - if err := json.Unmarshal(output, &full); err != nil { - return nil, fmt.Errorf("error decoding op full response: %w", err) - } - return full.Fields, nil +type entry struct { + ID string `json:"id"` + Value string `json:"value"` } -// Decode a single field from op -func decodeSingle(output []byte) (*entry, error) { - var single entry - if err := json.Unmarshal(output, &single); err != nil { - return nil, fmt.Errorf("error decoding op single response: %w", err) - } - return &single, nil +func (i item) password() ([]byte, bool) { + secret, ok := slices.Find(i.Fields, func(item entry) bool { + return item.ID == "password" + }) + return []byte(secret.Value), ok } -// Custom parser for 1Password secret references because the format is not a standard URL, and we also need to -// allow users to omit the field name so that we can support secrets with multiple fields. -// -// Does not support "section-name". -// -// op:///[/] -// -// Secret references are case-insensitive and support the following characters: -// -// alphanumeric characters (a-z, A-Z, 0-9), -, _, . and the whitespace character -// -// If an item or field name includes a / or an unsupported character, use the item -// or field's unique identifier (ID) instead of its name. -// -// See https://developer.1password.com/docs/cli/secrets-reference-syntax/ -type secretRef struct { - Vault string - Item string - Field optional.Option[string] -} +// op --format json item get --vault Personal "With Spaces" +func getItem(ctx context.Context, vault string, ref Ref) (*item, error) { + logger := log.FromContext(ctx) -var validCharsRegex = regexp.MustCompile(`^[a-zA-Z0-9\-_\. ]+$`) + args := []string{"--format", "json", "item", "get", "--vault", vault, ref.String()} + output, err := exec.Capture(ctx, ".", "op", args...) + logger.Debugf("Getting item with args %s", shellquote.Join(args...)) + if err != nil { + // This is specifically not itemNotFoundError, to distinguish between vault not found and item not found. + if strings.Contains(string(output), "isn't a vault") { + return nil, fmt.Errorf("vault %q not found: %w", vault, err) + } -func decodeSecretRef(ref string) (*secretRef, error) { + // Item not found, seen two ways of reporting this: + if strings.Contains(string(output), "not found in vault") { + return nil, itemNotFoundError{vault, ref} + } + if strings.Contains(string(output), "isn't an item") { + return nil, itemNotFoundError{vault, ref} + } - // Take out and check the "op://" prefix - const prefix = "op://" - if !strings.HasPrefix(ref, prefix) { - return nil, fmt.Errorf("must start with \"op://\"") + return nil, fmt.Errorf("run `op` with args %s: %w", shellquote.Join(args...), err) } - ref = ref[len(prefix):] - - parts := strings.Split(ref, "/") - if len(parts) < 2 { - return nil, fmt.Errorf("must have at least 2 parts") - } - if len(parts) > 3 { - return nil, fmt.Errorf("must have at most 3 parts") + var full item + if err := json.Unmarshal(output, &full); err != nil { + return nil, fmt.Errorf("error decoding op full response: %w", err) } + return &full, nil +} - for _, part := range parts { - if part == "" { - return nil, fmt.Errorf("url parts must not be empty") - } - - if !validCharsRegex.MatchString(part) { - return nil, fmt.Errorf("url part %q contains unsupported characters. regex: %q", part, validCharsRegex) - } +// op item create --category Password --vault FTL --title mod.ule "password=val ue" +func createItem(ctx context.Context, vault string, ref Ref, secret []byte) error { + args := []string{"item", "create", "--category", "Password", "--vault", vault, "--title", ref.String(), "password=" + string(secret)} + _, err := exec.Capture(ctx, ".", "op", args...) + if err != nil { + return fmt.Errorf("create item failed in vault %q, ref %q: %w", vault, ref, err) } - secret := secretRef{ - Vault: parts[0], - Item: parts[1], - Field: optional.None[string](), - } - if len(parts) == 3 { - secret.Field = optional.Some(parts[2]) + return nil +} + +// op item edit --vault ftl test "password=with space" +func editItem(ctx context.Context, vault string, ref Ref, secret []byte) error { + args := []string{"item", "edit", "--vault", vault, ref.String(), "password=" + string(secret)} + _, err := exec.Capture(ctx, ".", "op", args...) + if err != nil { + return fmt.Errorf("edit item failed in vault %q, ref %q: %w", vault, ref, err) } - return &secret, nil + return nil } diff --git a/common/configuration/1password_provider_test.go b/common/configuration/1password_provider_test.go index 52185ae77f..51fd4976a9 100644 --- a/common/configuration/1password_provider_test.go +++ b/common/configuration/1password_provider_test.go @@ -1,82 +1,74 @@ +//go:build 1password + +// 1password needs to be running and will temporarily make a "ftl-test" vault. +// +// These tests will clean up before and after itself. + package configuration import ( - "github.com/alecthomas/types/optional" - "reflect" + "context" "testing" + + "github.com/alecthomas/assert/v2" + + "github.com/TBD54566975/ftl/internal/exec" + "github.com/TBD54566975/ftl/internal/log" ) -func TestDecodeSecretRef(t *testing.T) { - tests := []struct { - name string - ref string - want *secretRef - wantErr bool - }{ - { - name: "simple with field", - ref: "op://development/Access Keys/access_key_id", - want: &secretRef{ - Vault: "development", - Item: "Access Keys", - Field: optional.Some("access_key_id"), - }, - }, - { - name: "simple without field", - ref: "op://vault/item", - want: &secretRef{ - Vault: "vault", - Item: "item", - Field: optional.None[string](), - }, - }, - { - name: "lots of spaces", - ref: "op://My Awesome Vault/My Awesome Item/My Awesome Field", - want: &secretRef{ - Vault: "My Awesome Vault", - Item: "My Awesome Item", - Field: optional.Some("My Awesome Field"), - }, - }, - { - name: "missing op://", - ref: "development/Access Keys/access_key_id", - wantErr: true, - }, - { - name: "empty parts", - ref: "op://development//access_key_id", - wantErr: true, - }, - { - name: "invalid characters", - ref: "op://development/aws/acce$s", - wantErr: true, - }, - { - name: "too many parts", - ref: "op://development/Access Keys/access_key_id/extra", - wantErr: true, - }, - { - name: "too few parts", - ref: "op://development", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := decodeSecretRef(tt.ref) - if (err != nil) != tt.wantErr { - t.Errorf("decodeSecretRef() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("decodeSecretRef() got = %v, want %v", got, tt.want) - } - }) - } +const vault = "ftl-test" +const module = "test.module" + +func createVault(ctx context.Context) error { + args := []string{"vault", "create", vault} + _, err := exec.Capture(ctx, ".", "op", args...) + return err +} + +func clean(ctx context.Context) bool { + args := []string{"vault", "delete", vault} + _, err := exec.Capture(ctx, ".", "op", args...) + return err == nil +} + +func Test1PasswordProvider(t *testing.T) { + ctx := log.ContextWithNewDefaultLogger(context.Background()) + + // OK to fail the initial clean. + _ = clean(ctx) + t.Cleanup(func() { + if !clean(ctx) { + t.Fatal("clean failed") + } + }) + + err := createVault(ctx) + assert.NoError(t, err) + + _, err = getItem(ctx, vault, Ref{Name: module}) + assert.Error(t, err) + + var pw1 = []byte("hunter1") + var pw2 = []byte(`{ + "user": "root", + "password": "hunter🪤" + }`) + + err = createItem(ctx, vault, Ref{Name: module}, pw1) + assert.NoError(t, err) + + value, err := getItem(ctx, vault, Ref{Name: module}) + assert.NoError(t, err) + secret, ok := value.password() + assert.True(t, ok) + assert.Equal(t, pw1, secret) + + err = editItem(ctx, vault, Ref{Name: module}, pw2) + assert.NoError(t, err) + + value, err = getItem(ctx, vault, Ref{Name: module}) + assert.NoError(t, err) + secret, ok = value.password() + assert.True(t, ok) + assert.Equal(t, pw2, secret) }