Support Vault backend KVv2. Connected to cyberark#1331.

Signed-off-by: Michael Meijer <[email protected]>

CHANGELOG.md

@@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added
+
+- The `vault` provider now supports loading secrets from the KV Version 2 secret
+  engine. Reference a secret in Vault using the right path and a field
+  navigation in the Secretless configuration.
+  [#1331](https://github.com/cyberark/secretless-broker/issues/1331)
+
 ## [1.7.0] - 2020-09-11
 
 ### Added

internal/providers/vault/README.md

@@ -0,0 +1,98 @@
+# Vault Provider
+
+The Vault provider for Secretless can fetch secrets from configured secret
+engines in [HashiCorp Vault](https://www.vaultproject.io). The provider is based
+on the [Vault API client](https://pkg.go.dev/github.com/hashicorp/vault/api) in
+Go. It reads the secret object from the configured path and returns the value
+navigated to by the configured fields (or default field otherwise).
+
+## Usage Documentation
+
+The Vault provider is configured in the `secretless.yml` using:
+
+```yaml
+from: vault
+get: /path/to/secret/in/vault
+```
+
+Or with explicit fields navigating to the value in the secret returned at path:
+
+```yaml
+from: vault
+get: /path/to/secret/in/vault#navigate.to.this.field
+```
+
+The provider will read a secret (object) at a given path and returns the value
+of field `value` (by default). By appending `#data.fieldName` to the path, the
+provider will instead read the value at the field `fieldName` in the object
+`data` in the secret (object) instead.
+
+Below are some examples showing how to configure the provider for secrets.
+
+### Example: API key from KV backends (v1 and v2)
+
+Below is an excerpt of an example configuration for a fictional "Example
+Service" that requires an API key, e.g. used in a request header. It gets the
+API key from Vault's KV version 1 backend at path `kv/example-service` under the
+secret's `value` field.
+
+```yaml
+version: 2
+services:
+  my_example_service:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8080
+    credentials:
+      apikey:
+        from: vault
+        get: kv/example-service
+        # gets path to API key in Vault, field 'value' holds the API key
+    ...
+```
+
+A slightly different configuration explicitly sets the field `api-key` (instead
+of the default `value`) to hold the API key.
+
+```yaml
+version: 2
+services:
+  my_example_service:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8080
+    credentials:
+      apikey:
+        from: vault
+        get: kv/example-service#api-key
+        # gets path to API key in Vault, field 'api-key' holds the API key
+    ...
+```
+
+If the secret is stored in a KV v2 backend (mounted at `secret` by default), the
+configuration must use the use the `data` segment in the path and the
+`#data.api-key` suffix. This is behavior specific to KV v2 in Vault, see Vault
+API docs.
+
+```yaml
+version: 2
+services:
+  my_example_service:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8080
+    credentials:
+      apikey:
+        from: vault
+        get: secret/data/example-service#data.api-key
+        # gets path to API key in Vault stored in the KV v2 secret engine
+    ...
+```
+
+## Limitations
+
+- Only token-based login to Vault supported at the moment.
+- Only secrets that are "read" in Vault are supported at the moment. Backends
+  that require "writes" to obtain the secret (e.g. PKI, dynamic database
+  credentials) are not supported at the moment.
+- Backends that have multiple values change simultaneously (e.g. client id and
+  secret, database username and password) are not supported at the moment.
+- Limited support for KV v2 secret engine, only latest version of a secret can
+  be retrieved.

internal/providers/vault/provider.go

@@ -16,7 +16,10 @@ type Provider struct {
 }
 
 // ProviderFactory constructs a Provider. The API client is configured from
-// environment variables.
+// environment variables. Underlying Vault API client by default uses:
+// - VAULT_ADDR: endpoint of Vault, e.g. http://vault:8200/
+// - VAULT_TOKEN: token to login to Vault
+// See Vault API docs at https://godoc.org/github.com/hashicorp/vault/api
 func ProviderFactory(options plugin_v1.ProviderOptions) (plugin_v1.Provider, error) {
 	config := vault.DefaultConfig()
 
@@ -39,53 +42,87 @@ func (p *Provider) GetName() string {
 	return p.Name
 }
 
-// VaultDefaultField is the default value returned by the provider from the
-// hash returned by the Vault.
-const VaultDefaultField = "value"
+// DefaultField is the default field name the provider expects to find the secret value.
+const DefaultField = "value"
 
-// parseVaultID returns the secret id and field name.
+// parseVaultID returns the path to the secret (object) and (normalized) field/property path to the secret value.
 func parseVaultID(id string) (string, string) {
 	tokens := strings.SplitN(id, "#", 2)
 	switch len(tokens) {
 	case 1:
-		return tokens[0], VaultDefaultField
+		return tokens[0], DefaultField
 	default:
 		return tokens[0], tokens[1]
 	}
 }
 
-// GetValue obtains a value by id. Any secret which is stored in the vault is recognized.
-// The datatype returned by Vault is map[string]interface{}. Therefore this provider needs
-// to know which field to return from the map. By default, it returns the 'value'.
-// An alternative field can be obtained by appending '#fieldName' to the id argument.
-func (p *Provider) GetValue(id string) (value []byte, err error) {
-	id, fieldName := parseVaultID(id)
+// valueOf returns the value when navigating the obj along the fields.
+// Suppose obj = { "foo": { "bar": "qux" } } and fields = "foo.bar", then value returned is "qux".
+func valueOf(obj map[string]interface{}, fields string) (interface{}, bool) {
+	// Split fields to navigate by ".", e.g. if fields = [ "foo.bar" ] then it becomes a slice of [ "foo", "bar" ]
+	nav := strings.Split(fields, ".")
 
-	var secret *vault.Secret
-	if secret, err = p.Client.Logical().Read(id); err != nil {
-		return
-	}
-	// secret can be nil if it's not found
-	if secret == nil {
-		err = fmt.Errorf("HashiCorp Vault provider could not find a secret called '%s'", id)
-		return
+	// Traverse, starting at given obj, moving deeper into the object structure
+	for _, field := range nav[:len(nav)-1] {
+		// Get value of field in object
+		value, ok := obj[field]
+		if !ok {
+			return nil, false
+		}
+
+		// Value should be a (nested) object, hence update obj with value for next iteration
+		obj, ok = value.(map[string]interface{})
+		if !ok {
+			return nil, false
+		}
 	}
 
-	var ok bool
-	var valueObj interface{}
-	valueObj, ok = secret.Data[fieldName]
+	// Last field in navigation holds the actual value
+	field := nav[len(nav)-1]
+	return obj[field], true
+}
+
+// parseSecret returns value navigated by given fields on secret object.
+// Note that a secret returned from Vault is effectively a JSON object.
+func parseSecret(secret *vault.Secret, path string, fields string) ([]byte, error) {
+	value, ok := valueOf(secret.Data, fields)
 	if !ok {
-		err = fmt.Errorf("HashiCorp Vault provider expects the secret '%s' to contain field '%s'", id, fieldName)
-		return
+		err := fmt.Errorf("HashiCorp Vault provider expects secret in '%s' at '%s'", fields, path)
+		return nil, err
 	}
 
-	switch v := valueObj.(type) {
+	// Secret value must be either string or bytes
+	switch v := value.(type) {
 	case string:
-		value = []byte(v)
+		return []byte(v), nil
 	case []byte:
-		value = v
+		return v, nil
 	default:
-		err = fmt.Errorf("HashiCorp Vault provider expects the secret to be a string or byte[], got %T", v)
+		err := fmt.Errorf("HashiCorp Vault provider expects the secret to be a string or byte[], got %T", v)
+		return nil, err
 	}
-	return
+}
+
+// GetValue obtains a value by id. The id should contain the path in Vault to the secret. It may be appended with a
+// hash following the object property path to the secret value; defaults to DefaultField.
+// For example:
+//   - `kv/database/password` returns the value of field `value` in the secret object at given path.
+//   - `kv/database#password` returns the value of field `password` in the secret object at path `kv/database`.
+//   - `secret/data/database#data.value` returns the value of field `value` wrapped in object `data` in secret object
+//  	at path `secret/data/database`.
+// Secrets in Vault are stored as (JSON) objects in the shape of map[string]interface{}. Both path to the secret and
+// fields to the value in the secret must follow Vault API client conventions. Please see documentation of Vault for
+// details.
+func (p *Provider) GetValue(id string) ([]byte, error) {
+	path, fields := parseVaultID(id)
+	secret, err := p.Client.Logical().Read(path)
+	if err != nil {
+		return nil, err
+	}
+	if secret == nil {
+		err = fmt.Errorf("HashiCorp Vault provider could not find secret '%s'", path)
+		return nil, err
+	}
+
+	return parseSecret(secret, path, fields)
 }

test/providers/vault/start

@@ -49,7 +49,13 @@ VAULT_ADDR=http://localhost:$vault_port
 VAULT_TOKEN=$root_token
 ENV
 
-vault_cmd secrets enable kv
+# cubbyhole enabled by default, mounted at /cubbyhole
+vault_cmd write cubbyhole/first-secret 'some-key=one'
+vault_cmd write cubbyhole/second-secret 'value=two'
+
+# KV v2 enabled by default, mounted at /secret
+vault_cmd secrets enable -version=1 kv
 vault_cmd kv put kv/db/password 'password=db-secret'
 vault_cmd kv put kv/frontend/admin-password 'password=frontend-secret'
 vault_cmd kv put kv/web/password 'value=web-secret'
+vault_cmd kv put secret/service 'api-key=service-api-key'

test/providers/vault/vault_provider_test.go

@@ -31,19 +31,45 @@ func TestVault_Provider(t *testing.T) {
 	Convey("Reports when the secret is not found", t, func() {
 		value, err := provider.GetValue("foobar")
 		So(err, ShouldNotBeNil)
-		So(err.Error(), ShouldEqual, "HashiCorp Vault provider could not find a secret called 'foobar'")
+		So(err.Error(), ShouldEqual, "HashiCorp Vault provider could not find secret 'foobar'")
 		So(value, ShouldBeNil)
 	})
 
-	Convey("Can provide a secret", t, func() {
+	Convey("Reports when a field in the secret is not found", t, func() {
+		value, err := provider.GetValue("cubbyhole/first-secret#foo.bar")
+		So(err, ShouldNotBeNil)
+		So(err.Error(), ShouldEqual, "HashiCorp Vault provider expects secret in 'foo.bar' at 'cubbyhole/first-secret'")
+		So(value, ShouldBeNil)
+	})
+
+	Convey("Can provide a cubbyhole secret", t, func() {
+		value, err := provider.GetValue("cubbyhole/first-secret#some-key")
+		So(err, ShouldBeNil)
+		So(string(value), ShouldEqual, "one")
+	})
+
+	Convey("Can provide a cubbyhole secret with default field name", t, func() {
+		value, err := provider.GetValue("cubbyhole/second-secret")
+		So(err, ShouldBeNil)
+		So(string(value), ShouldEqual, "two")
+	})
+
+	Convey("Can provide a KV v1 secret", t, func() {
 		value, err := provider.GetValue("kv/db/password#password")
 		So(err, ShouldBeNil)
 		So(string(value), ShouldEqual, "db-secret")
 	})
 
-	Convey("Can provide a secret with default field name", t, func() {
+	Convey("Can provide a KV v1 secret with default field name", t, func() {
 		value, err := provider.GetValue("kv/web/password")
 		So(err, ShouldBeNil)
 		So(string(value), ShouldEqual, "web-secret")
 	})
+
+	// note the "data" in path and in the fields to navigate, which is required in KV v2
+	Convey("Can provide latest KV v2 secret", t, func() {
+		value, err := provider.GetValue("secret/data/service#data.api-key")
+		So(err, ShouldBeNil)
+		So(string(value), ShouldEqual, "service-api-key")
+	})
 }

test/providers/vault/vault_summon_test.go

@@ -14,8 +14,11 @@ import (
 
 func TestVault_Summon(t *testing.T) {
 	secretsDescriptor := `
+FIRST_SECRET: !var cubbyhole/first-secret#some-key
+SECOND_SECRET: !var cubbyhole/second-secret
 DB_PASSWORD: !var kv/db/password#password
 WEB_PASSWORD: !var kv/web/password
+SVC_API_KEY: !var secret/data/service#data.api-key
 `
 	defaultArgs := []string{"summon2", "-p", "vault", "--yaml", secretsDescriptor, "env"}
 
@@ -36,7 +39,10 @@ WEB_PASSWORD: !var kv/web/password
 		lines, err := runCommand(defaultArgs)
 
 		So(err, ShouldBeNil)
+		So(lines, ShouldContain, "FIRST_SECRET=one")
+		So(lines, ShouldContain, "SECOND_SECRET=two")
 		So(lines, ShouldContain, "DB_PASSWORD=db-secret")
 		So(lines, ShouldContain, "WEB_PASSWORD=web-secret")
+		So(lines, ShouldContain, "SVC_API_KEY=service-api-key")
 	})
 }