From 5cfb3257e74ead4b98ed8b0e340e57dd39f9189c Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 31 Mar 2020 10:45:16 -0700 Subject: [PATCH] Use docker-credential-helper to manage secrets (WIP) Store service credentials securely in the stores supported by docker: - https://github.com/docker/docker-credential-helpers#available-programs Introduces a top-level config property, "secretStore" and additional command line arguments to manage the stored secrets. The value of secretStore is used to find a helper command, `docker-credential-`. The docker project currently provides 4 store helpers: - "osxkeychain" (OS X only) - "secretservice" (Linux only) - "wincred" (Windows only) - "pass" (any OS supporting pass, which uses gpg2) Docker-for-desktop installs the credential helpers above, as well as "desktop" (docker-credential-desktop). Generic installation instructions for the helpers: - https://github.com/docker/docker-credential-helpers#installation Users could provide additional helpers, the only requirement is that the helper implements the credential store protocol: - https://github.com/docker/docker-credential-helpers#development The credential protocol is open, and new credential stores can be implemented by any CLI satisfying the protocol: - https://github.com/docker/docker-credential-helpers#development The modifications to existing modules is not tested due to lack of API keys, but demonstrates the unobtrusive changes required to use the secret store. --- cfg/secrets.go | 164 +++++++++++++++++++++++++++++++ flags/flags.go | 63 ++++++++++++ go.mod | 1 + go.sum | 2 + modules/azuredevops/settings.go | 8 ++ modules/bamboohr/settings.go | 8 ++ modules/buildkite/settings.go | 11 ++- modules/circleci/settings.go | 8 ++ modules/datadog/settings.go | 16 +++ modules/digitalocean/settings.go | 8 ++ modules/football/settings.go | 9 ++ modules/gerrit/settings.go | 8 ++ modules/github/settings.go | 9 ++ modules/gitlab/settings.go | 8 ++ modules/gitlabtodo/settings.go | 8 ++ 15 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 cfg/secrets.go diff --git a/cfg/secrets.go b/cfg/secrets.go new file mode 100644 index 0000000000..acfc447f4a --- /dev/null +++ b/cfg/secrets.go @@ -0,0 +1,164 @@ +package cfg + +import ( + "errors" + "fmt" + "runtime" + + "github.com/docker/docker-credential-helpers/client" + "github.com/docker/docker-credential-helpers/credentials" + "github.com/olebedev/config" + "github.com/wtfutil/wtf/logger" +) + +type Secret struct { + Service string + Secret string + Username string + Store string +} + +// Configure the secret for a service. +// +// Does not overwrite explicitly configured values, so is safe to call +// if username and secret were explicitly set in module config. +// +// Input: +// * service: URL or identifier for service if configured by user. Not all +// modules support or need this. Optional, defatuls to serviceDefault. +// * serviceDefault: Default URL or identifier for service. Must be unique, +// using the API URL is customary, but using the module name is reasonable. +// Required, secrets cannot be stored unless associated with a service. +// +// Output: +// * username: If a user/subdomain/identifier specific to the service is +// configurable, it can be saved as a "username". Optional. +// * secret: The secret for service. Optional. +func ConfigureSecret( + globalConfig *config.Config, + service string, + serviceDefault string, + username *string, + secret *string, // unfortunate order dependency... +) { + notWanted := func(out *string) bool { + return out == nil && *out != "" + } + + // Don't try to fetch from cred store if nothing is wanted. + if notWanted(secret) && notWanted(username) { + return + } + + if service == "" { + service = serviceDefault + } + + if service == "" { + return + } + + cred, err := FetchSecret(globalConfig, service) + + if err != nil { + logger.Log(fmt.Sprintf("Loading secret failed: %s", err.Error())) + return + } + + if cred == nil { + // No secret store configued. + return + } + + if username != nil && *username == "" { + *username = cred.Username + } + + if secret != nil && *secret == "" { + *secret = cred.Secret + } +} + +// Fetch secret for `service`. Service is customarily a URL, but can be any +// identifier uniquely used by wtf to identify the service, such as the name +// of the module. nil is returned if the secretStore global property is not +// present or the secret is not found in that store. +func FetchSecret(globalConfig *config.Config, service string) (*Secret, error) { + prog := newProgram(globalConfig) + + if prog == nil { + // No secret store configured. + return nil, nil + } + + cred, err := client.Get(prog.runner, service) + + if err != nil { + return nil, fmt.Errorf("get %v from %v: %w", service, prog.store, err) + } + + return &Secret{ + Service: cred.ServerURL, + Secret: cred.Secret, + Username: cred.Username, + Store: prog.store, + }, nil +} + +func StoreSecret(globalConfig *config.Config, secret *Secret) error { + prog := newProgram(globalConfig) + + if prog == nil { + return errors.New("Cannot store secrets: wtf.secretStore is not configured") + } + + cred := &credentials.Credentials{ + ServerURL: secret.Service, + Username: secret.Username, + Secret: secret.Secret, + } + + // docker-credential requires a username, but it isn't necessary for + // all services. Use a default if a username was not set. + if cred.Username == "" { + cred.Username = "default" + } + + err := client.Store(prog.runner, cred) + + if err != nil { + return fmt.Errorf("store %v: %w", prog.store, err) + } + + return nil +} + +type program struct { + store string + runner client.ProgramFunc +} + +func newProgram(globalConfig *config.Config) *program { + secretStore := globalConfig.UString("wtf.secretStore", "(none)") + + if secretStore == "(none)" { + return nil + } + + if secretStore == "" { + switch runtime.GOOS { + case "windows": + secretStore = "winrt" + case "darwin": + secretStore = "osxkeychain" + default: + secretStore = "secretservice" + } + + } + + return &program{ + secretStore, + client.NewShellProgramFunc("docker-credential-" + secretStore), + } +} diff --git a/flags/flags.go b/flags/flags.go index bc6fcbdb5c..bca315f501 100644 --- a/flags/flags.go +++ b/flags/flags.go @@ -17,10 +17,28 @@ type Flags struct { Module string `short:"m" long:"module" optional:"yes" description:"Display info about a specific module, i.e.: 'wtfutil -m=todo'"` Profile bool `short:"p" long:"profile" optional:"yes" description:"Profile application memory usage"` Version bool `short:"v" long:"version" description:"Show version info"` + // Work-around go-flags misfeatures. If any sub-command is defined + // then `wtf` (no sub-commands, the common usage), is warned about. + Opt struct { + Cmd string `positional-arg-name:"command"` + Args []string `positional-arg-name:"args"` + } `positional-args:"yes"` hasCustom bool } +var EXTRA = ` +Commands: + save-secret [username] + service Service URL or name for secret. + secret Secret to be saved for the service. + username Username to associate with the service. + Save a secret into the secret store. Requires wtf.secretStore + to be configured. See individual modules for information on what + service, secret, and username means for their configuration. Not + all modules use secrets, and not all secrets require a username. +` + // NewFlags creates an instance of Flags func NewFlags() *Flags { flags := Flags{} @@ -46,6 +64,50 @@ func (flags *Flags) RenderIf(version, date string, config *config.Config) { fmt.Println(fmt.Sprintf("%s (%s)", version, date)) os.Exit(0) } + + if flags.Opt.Cmd == "" { + return + } + + switch cmd := flags.Opt.Cmd; cmd { + case "save-secret": + var service, secret, username string + args := flags.Opt.Args + + if len(flags.Opt.Args) < 2 || args[0] == "" { + fmt.Fprintf(os.Stderr, "save-secret: service required, see `%s --help`\n", os.Args[0]) + os.Exit(1) + } + + if len(flags.Opt.Args) < 2 || args[1] == "" { + fmt.Fprintf(os.Stderr, "save-secret: secret required, see `%s --help`\n", os.Args[0]) + os.Exit(1) + } + + service = args[0] + secret = args[1] + + if len(args) > 2 { + username = args[2] + } + + err := cfg.StoreSecret(config, &cfg.Secret{ + Service: service, + Secret: secret, + Username: username, + }) + + if err != nil { + fmt.Fprintf(os.Stderr, "Saving secret for service %v: %s\n", service, err.Error()) + os.Exit(1) + } + + fmt.Printf("Saved secret for service %v\n", service) + os.Exit(0) + default: + fmt.Fprintf(os.Stderr, "Command `%s` is not supported, try `%s --help`\n", cmd, os.Args[0]) + os.Exit(1) + } } // HasCustomConfig returns TRUE if a config path was passed in, FALSE if one was not @@ -68,6 +130,7 @@ func (flags *Flags) Parse() { parser := goFlags.NewParser(flags, goFlags.Default) if _, err := parser.Parse(); err != nil { if flagsErr, ok := err.(*goFlags.Error); ok && flagsErr.Type == goFlags.ErrHelp { + fmt.Println(EXTRA) os.Exit(0) } } diff --git a/go.mod b/go.mod index 533c9bfb70..75b1a217d2 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/digitalocean/godo v1.33.1 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 + github.com/docker/docker-credential-helpers v0.6.3 github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 diff --git a/go.sum b/go.sum index b2adfd68c3..3ada72645b 100644 --- a/go.sum +++ b/go.sum @@ -143,6 +143,8 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= diff --git a/modules/azuredevops/settings.go b/modules/azuredevops/settings.go index 5fdca9c709..2d7c679aa1 100644 --- a/modules/azuredevops/settings.go +++ b/modules/azuredevops/settings.go @@ -35,5 +35,13 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co projectName: ymlConfig.UString("projectName", os.Getenv("WTF_AZURE_DEVOPS_PROJECT_NAME")), } + cfg.ConfigureSecret( + globalConfig, + settings.orgURL, + "", + &settings.projectName, + &settings.apiToken, + ) + return &settings } diff --git a/modules/bamboohr/settings.go b/modules/bamboohr/settings.go index 112db588b2..ef8e11dc37 100644 --- a/modules/bamboohr/settings.go +++ b/modules/bamboohr/settings.go @@ -27,5 +27,13 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co subdomain: ymlConfig.UString("subdomain", os.Getenv("WTF_BAMBOO_HR_SUBDOMAIN")), } + cfg.ConfigureSecret( + globalConfig, + "", + name, + &settings.subdomain, + &settings.apiKey, + ) + return &settings } diff --git a/modules/buildkite/settings.go b/modules/buildkite/settings.go index 6a0c67ad25..b1459b5e2a 100644 --- a/modules/buildkite/settings.go +++ b/modules/buildkite/settings.go @@ -1,10 +1,11 @@ package buildkite import ( + "os" + "github.com/olebedev/config" "github.com/wtfutil/wtf/cfg" "github.com/wtfutil/wtf/utils" - "os" ) const ( @@ -35,6 +36,14 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co pipelines: buildPipelineSettings(ymlConfig), } + cfg.ConfigureSecret( + globalConfig, + "", + name, + &settings.orgSlug, + &settings.apiKey, + ) + return &settings } diff --git a/modules/circleci/settings.go b/modules/circleci/settings.go index b16ef7f9b8..6ab870ed0c 100644 --- a/modules/circleci/settings.go +++ b/modules/circleci/settings.go @@ -26,5 +26,13 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co apiKey: ymlConfig.UString("apiKey", ymlConfig.UString("apikey", os.Getenv("WTF_CIRCLE_API_KEY"))), } + cfg.ConfigureSecret( + globalConfig, + "", + name, + nil, + &settings.apiKey, + ) + return &settings } diff --git a/modules/datadog/settings.go b/modules/datadog/settings.go index a8d1c4a72a..502cbacbfd 100644 --- a/modules/datadog/settings.go +++ b/modules/datadog/settings.go @@ -30,5 +30,21 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co tags: ymlConfig.UList("monitors.tags"), } + cfg.ConfigureSecret( + globalConfig, + "", + "datadog-api", + nil, + &settings.apiKey, + ) + + cfg.ConfigureSecret( + globalConfig, + "", + "datadog-app", + nil, + &settings.applicationKey, + ) + return &settings } diff --git a/modules/digitalocean/settings.go b/modules/digitalocean/settings.go index 6d2e1c696d..00a3f81647 100644 --- a/modules/digitalocean/settings.go +++ b/modules/digitalocean/settings.go @@ -31,5 +31,13 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co dateFormat: ymlConfig.UString("dateFormat", wtf.DateFormat), } + cfg.ConfigureSecret( + globalConfig, + "", + name, + nil, + &settings.apiKey, + ) + return &settings } diff --git a/modules/football/settings.go b/modules/football/settings.go index dda7575919..3348830813 100644 --- a/modules/football/settings.go +++ b/modules/football/settings.go @@ -33,5 +33,14 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co matchesTo: ymlConfig.UInt("matchesTo", 5), standingCount: ymlConfig.UInt("standingCount", 5), } + + cfg.ConfigureSecret( + globalConfig, + "", + name, + nil, + &settings.apiKey, + ) + return &settings } diff --git a/modules/gerrit/settings.go b/modules/gerrit/settings.go index 0d78550ef9..4631832539 100644 --- a/modules/gerrit/settings.go +++ b/modules/gerrit/settings.go @@ -41,6 +41,14 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co verifyServerCertificate: ymlConfig.UBool("verifyServerCertificate", true), } + cfg.ConfigureSecret( + globalConfig, + settings.domain, + name, + nil, // Seems like it should be mandatory, but its optional above. + &settings.password, + ) + settings.colors.rows.even = ymlConfig.UString("colors.rows.even", "white") settings.colors.rows.odd = ymlConfig.UString("colors.rows.odd", "blue") diff --git a/modules/github/settings.go b/modules/github/settings.go index 0aac22a0ee..e9cb496c87 100644 --- a/modules/github/settings.go +++ b/modules/github/settings.go @@ -17,6 +17,7 @@ type Settings struct { common *cfg.Common apiKey string `help:"Your GitHub API token."` + apiSecret string `help:"Secret store for your GitHub API token."` baseURL string `help:"Your GitHub Enterprise API URL." optional:"true"` customQueries []customQuery `help:"Custom queries allow you to filter pull requests and issues however you like. Give the query a title and a filter. Filters can be copied directly from GitHub’s UI." optional:"true"` enableStatus bool `help:"Display pull request mergeability status (‘dirty’, ‘clean’, ‘unstable’, ‘blocked’)." optional:"true"` @@ -45,6 +46,14 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co settings.repositories = cfg.ParseAsMapOrList(ymlConfig, "repositories") settings.customQueries = parseCustomQueries(ymlConfig) + cfg.ConfigureSecret( + globalConfig, + settings.baseURL, + name, + &settings.username, + &settings.apiKey, + ) + return &settings } diff --git a/modules/gitlab/settings.go b/modules/gitlab/settings.go index 0c7db373c3..bbd6b42642 100644 --- a/modules/gitlab/settings.go +++ b/modules/gitlab/settings.go @@ -32,6 +32,14 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co username: ymlConfig.UString("username"), } + cfg.ConfigureSecret( + globalConfig, + settings.domain, + name, + &settings.username, + &settings.apiKey, + ) + settings.projects = cfg.ParseAsMapOrList(ymlConfig, "projects") return &settings diff --git a/modules/gitlabtodo/settings.go b/modules/gitlabtodo/settings.go index b1f3d72d02..ee47972385 100644 --- a/modules/gitlabtodo/settings.go +++ b/modules/gitlabtodo/settings.go @@ -32,5 +32,13 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co showProject: ymlConfig.UBool("showProject", true), } + cfg.ConfigureSecret( + globalConfig, + settings.domain, + name, + nil, + &settings.apiKey, + ) + return &settings }