From 8d56e6354aa91bead3f4c3fc18a30be4d9ddb220 Mon Sep 17 00:00:00 2001 From: Meijer Date: Thu, 24 Sep 2020 21:07:19 +0200 Subject: [PATCH] Support Vault backend KVv2. Connected to #1331. Signed-off-by: Michael Meijer --- CHANGELOG.md | 7 ++ internal/providers/vault/README.md | 98 +++++++++++++++++++++ internal/providers/vault/provider.go | 97 +++++++++++++------- test/providers/vault/start | 8 +- test/providers/vault/vault_provider_test.go | 32 ++++++- test/providers/vault/vault_summon_test.go | 6 ++ 6 files changed, 214 insertions(+), 34 deletions(-) create mode 100644 internal/providers/vault/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 511b7d5e3..22a2c343f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Added + +- The `vault` provider now supports loading secrets from the KV Version 2 secret + engine. Reference a secret in Vault using the right path and a field + navigation in the Secretless configuration. + [#1331](https://github.com/cyberark/secretless-broker/issues/1331) + ## [1.7.0] - 2020-09-11 ### Added diff --git a/internal/providers/vault/README.md b/internal/providers/vault/README.md new file mode 100644 index 000000000..90b2e3eae --- /dev/null +++ b/internal/providers/vault/README.md @@ -0,0 +1,98 @@ +# Vault Provider + +The Vault provider for Secretless can fetch secrets from configured secret +engines in [HashiCorp Vault](https://www.vaultproject.io). The provider is based +on the [Vault API client](https://pkg.go.dev/github.com/hashicorp/vault/api) in +Go. It reads the secret object from the configured path and returns the value +navigated to by the configured fields (or default field otherwise). + +## Usage Documentation + +The Vault provider is configured in the `secretless.yml` using: + +```yaml +from: vault +get: /path/to/secret/in/vault +``` + +Or with explicit fields navigating to the value in the secret returned at path: + +```yaml +from: vault +get: /path/to/secret/in/vault#navigate.to.this.field +``` + +The provider will read a secret (object) at a given path and returns the value +of field `value` (by default). By appending `#data.fieldName` to the path, the +provider will instead read the value at the field `fieldName` in the object +`data` in the secret (object) instead. + +Below are some examples showing how to configure the provider for secrets. + +### Example: API key from KV backends (v1 and v2) + +Below is an excerpt of an example configuration for a fictional "Example +Service" that requires an API key, e.g. used in a request header. It gets the +API key from Vault's KV version 1 backend at path `kv/example-service` under the +secret's `value` field. + +```yaml +version: 2 +services: + my_example_service: + connector: generic_http + listenOn: tcp://0.0.0.0:8080 + credentials: + apikey: + from: vault + get: kv/example-service + # gets path to API key in Vault, field 'value' holds the API key + ... +``` + +A slightly different configuration explicitly sets the field `api-key` (instead +of the default `value`) to hold the API key. + +```yaml +version: 2 +services: + my_example_service: + connector: generic_http + listenOn: tcp://0.0.0.0:8080 + credentials: + apikey: + from: vault + get: kv/example-service#api-key + # gets path to API key in Vault, field 'api-key' holds the API key + ... +``` + +If the secret is stored in a KV v2 backend (mounted at `secret` by default), the +configuration must use the use the `data` segment in the path and the +`#data.api-key` suffix. This is behavior specific to KV v2 in Vault, see Vault +API docs. + +```yaml +version: 2 +services: + my_example_service: + connector: generic_http + listenOn: tcp://0.0.0.0:8080 + credentials: + apikey: + from: vault + get: secret/data/example-service#data.api-key + # gets path to API key in Vault stored in the KV v2 secret engine + ... +``` + +## Limitations + +- Only token-based login to Vault supported at the moment. +- Only secrets that are "read" in Vault are supported at the moment. Backends + that require "writes" to obtain the secret (e.g. PKI, dynamic database + credentials) are not supported at the moment. +- Backends that have multiple values change simultaneously (e.g. client id and + secret, database username and password) are not supported at the moment. +- Limited support for KV v2 secret engine, only latest version of a secret can + be retrieved. diff --git a/internal/providers/vault/provider.go b/internal/providers/vault/provider.go index dafca906c..788a76b7a 100644 --- a/internal/providers/vault/provider.go +++ b/internal/providers/vault/provider.go @@ -16,7 +16,10 @@ type Provider struct { } // ProviderFactory constructs a Provider. The API client is configured from -// environment variables. +// environment variables. Underlying Vault API client by default uses: +// - VAULT_ADDR: endpoint of Vault, e.g. http://vault:8200/ +// - VAULT_TOKEN: token to login to Vault +// See Vault API docs at https://godoc.org/github.com/hashicorp/vault/api func ProviderFactory(options plugin_v1.ProviderOptions) (plugin_v1.Provider, error) { config := vault.DefaultConfig() @@ -39,53 +42,87 @@ func (p *Provider) GetName() string { return p.Name } -// VaultDefaultField is the default value returned by the provider from the -// hash returned by the Vault. -const VaultDefaultField = "value" +// DefaultField is the default field name the provider expects to find the secret value. +const DefaultField = "value" -// parseVaultID returns the secret id and field name. +// parseVaultID returns the path to the secret (object) and (normalized) field/property path to the secret value. func parseVaultID(id string) (string, string) { tokens := strings.SplitN(id, "#", 2) switch len(tokens) { case 1: - return tokens[0], VaultDefaultField + return tokens[0], DefaultField default: return tokens[0], tokens[1] } } -// GetValue obtains a value by id. Any secret which is stored in the vault is recognized. -// The datatype returned by Vault is map[string]interface{}. Therefore this provider needs -// to know which field to return from the map. By default, it returns the 'value'. -// An alternative field can be obtained by appending '#fieldName' to the id argument. -func (p *Provider) GetValue(id string) (value []byte, err error) { - id, fieldName := parseVaultID(id) +// valueOf returns the value when navigating the obj along the fields. +// Suppose obj = { "foo": { "bar": "qux" } } and fields = "foo.bar", then value returned is "qux". +func valueOf(obj map[string]interface{}, fields string) (interface{}, bool) { + // Split fields to navigate by ".", e.g. if fields = [ "foo.bar" ] then it becomes a slice of [ "foo", "bar" ] + nav := strings.Split(fields, ".") - var secret *vault.Secret - if secret, err = p.Client.Logical().Read(id); err != nil { - return - } - // secret can be nil if it's not found - if secret == nil { - err = fmt.Errorf("HashiCorp Vault provider could not find a secret called '%s'", id) - return + // Traverse, starting at given obj, moving deeper into the object structure + for _, field := range nav[:len(nav)-1] { + // Get value of field in object + value, ok := obj[field] + if !ok { + return nil, false + } + + // Value should be a (nested) object, hence update obj with value for next iteration + obj, ok = value.(map[string]interface{}) + if !ok { + return nil, false + } } - var ok bool - var valueObj interface{} - valueObj, ok = secret.Data[fieldName] + // Last field in navigation holds the actual value + field := nav[len(nav)-1] + return obj[field], true +} + +// parseSecret returns value navigated by given fields on secret object. +// Note that a secret returned from Vault is effectively a JSON object. +func parseSecret(secret *vault.Secret, path string, fields string) ([]byte, error) { + value, ok := valueOf(secret.Data, fields) if !ok { - err = fmt.Errorf("HashiCorp Vault provider expects the secret '%s' to contain field '%s'", id, fieldName) - return + err := fmt.Errorf("HashiCorp Vault provider expects secret in '%s' at '%s'", fields, path) + return nil, err } - switch v := valueObj.(type) { + // Secret value must be either string or bytes + switch v := value.(type) { case string: - value = []byte(v) + return []byte(v), nil case []byte: - value = v + return v, nil default: - err = fmt.Errorf("HashiCorp Vault provider expects the secret to be a string or byte[], got %T", v) + err := fmt.Errorf("HashiCorp Vault provider expects the secret to be a string or byte[], got %T", v) + return nil, err } - return +} + +// GetValue obtains a value by id. The id should contain the path in Vault to the secret. It may be appended with a +// hash following the object property path to the secret value; defaults to DefaultField. +// For example: +// - `kv/database/password` returns the value of field `value` in the secret object at given path. +// - `kv/database#password` returns the value of field `password` in the secret object at path `kv/database`. +// - `secret/data/database#data.value` returns the value of field `value` wrapped in object `data` in secret object +// at path `secret/data/database`. +// Secrets in Vault are stored as (JSON) objects in the shape of map[string]interface{}. Both path to the secret and +// fields to the value in the secret must follow Vault API client conventions. Please see documentation of Vault for +// details. +func (p *Provider) GetValue(id string) ([]byte, error) { + path, fields := parseVaultID(id) + secret, err := p.Client.Logical().Read(path) + if err != nil { + return nil, err + } + if secret == nil { + err = fmt.Errorf("HashiCorp Vault provider could not find secret '%s'", path) + return nil, err + } + + return parseSecret(secret, path, fields) } diff --git a/test/providers/vault/start b/test/providers/vault/start index 74017dde1..2424d7a90 100755 --- a/test/providers/vault/start +++ b/test/providers/vault/start @@ -49,7 +49,13 @@ VAULT_ADDR=http://localhost:$vault_port VAULT_TOKEN=$root_token ENV -vault_cmd secrets enable kv +# cubbyhole enabled by default, mounted at /cubbyhole +vault_cmd write cubbyhole/first-secret 'some-key=one' +vault_cmd write cubbyhole/second-secret 'value=two' + +# KV v2 enabled by default, mounted at /secret +vault_cmd secrets enable -version=1 kv vault_cmd kv put kv/db/password 'password=db-secret' vault_cmd kv put kv/frontend/admin-password 'password=frontend-secret' vault_cmd kv put kv/web/password 'value=web-secret' +vault_cmd kv put secret/service 'api-key=service-api-key' diff --git a/test/providers/vault/vault_provider_test.go b/test/providers/vault/vault_provider_test.go index 9dbc985b1..25db08528 100644 --- a/test/providers/vault/vault_provider_test.go +++ b/test/providers/vault/vault_provider_test.go @@ -31,19 +31,45 @@ func TestVault_Provider(t *testing.T) { Convey("Reports when the secret is not found", t, func() { value, err := provider.GetValue("foobar") So(err, ShouldNotBeNil) - So(err.Error(), ShouldEqual, "HashiCorp Vault provider could not find a secret called 'foobar'") + So(err.Error(), ShouldEqual, "HashiCorp Vault provider could not find secret 'foobar'") So(value, ShouldBeNil) }) - Convey("Can provide a secret", t, func() { + Convey("Reports when a field in the secret is not found", t, func() { + value, err := provider.GetValue("cubbyhole/first-secret#foo.bar") + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "HashiCorp Vault provider expects secret in 'foo.bar' at 'cubbyhole/first-secret'") + So(value, ShouldBeNil) + }) + + Convey("Can provide a cubbyhole secret", t, func() { + value, err := provider.GetValue("cubbyhole/first-secret#some-key") + So(err, ShouldBeNil) + So(string(value), ShouldEqual, "one") + }) + + Convey("Can provide a cubbyhole secret with default field name", t, func() { + value, err := provider.GetValue("cubbyhole/second-secret") + So(err, ShouldBeNil) + So(string(value), ShouldEqual, "two") + }) + + Convey("Can provide a KV v1 secret", t, func() { value, err := provider.GetValue("kv/db/password#password") So(err, ShouldBeNil) So(string(value), ShouldEqual, "db-secret") }) - Convey("Can provide a secret with default field name", t, func() { + Convey("Can provide a KV v1 secret with default field name", t, func() { value, err := provider.GetValue("kv/web/password") So(err, ShouldBeNil) So(string(value), ShouldEqual, "web-secret") }) + + // note the "data" in path and in the fields to navigate, which is required in KV v2 + Convey("Can provide latest KV v2 secret", t, func() { + value, err := provider.GetValue("secret/data/service#data.api-key") + So(err, ShouldBeNil) + So(string(value), ShouldEqual, "service-api-key") + }) } diff --git a/test/providers/vault/vault_summon_test.go b/test/providers/vault/vault_summon_test.go index 2f51eeac5..4ed09b3ad 100644 --- a/test/providers/vault/vault_summon_test.go +++ b/test/providers/vault/vault_summon_test.go @@ -14,8 +14,11 @@ import ( func TestVault_Summon(t *testing.T) { secretsDescriptor := ` +FIRST_SECRET: !var cubbyhole/first-secret#some-key +SECOND_SECRET: !var cubbyhole/second-secret DB_PASSWORD: !var kv/db/password#password WEB_PASSWORD: !var kv/web/password +SVC_API_KEY: !var secret/data/service#data.api-key ` defaultArgs := []string{"summon2", "-p", "vault", "--yaml", secretsDescriptor, "env"} @@ -36,7 +39,10 @@ WEB_PASSWORD: !var kv/web/password lines, err := runCommand(defaultArgs) So(err, ShouldBeNil) + So(lines, ShouldContain, "FIRST_SECRET=one") + So(lines, ShouldContain, "SECOND_SECRET=two") So(lines, ShouldContain, "DB_PASSWORD=db-secret") So(lines, ShouldContain, "WEB_PASSWORD=web-secret") + So(lines, ShouldContain, "SVC_API_KEY=service-api-key") }) }