Support Vault backend KVv2. Connected to cyberark#1331.

Signed-off-by: Michael Meijer <[email protected]>

CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+- The `vault` provider now supports loading secrets from the KV Secrets Engine - 
+  Version 2. To reference secrets using this version, preface the secret path with 
+  `kv-v2:` in the Secretless configuration. [#1331](https://github.com/cyberark/secretless-broker/issues/1331)
 
 ## [1.7.0] - 2020-09-11
 

internal/providers/vault/README.md

@@ -0,0 +1,94 @@
+# Vault Provider
+
+The Vault provider for Secretless can fetch secrets from configured backends 
+in [HashiCorp Vault](https://www.vaultproject.io). The provider is based on 
+the [Vault API client](https://pkg.go.dev/github.com/hashicorp/vault/api) in Go. 
+It reads values from the configured paths.
+
+## Usage Documentation
+
+The Vault provider is configured in the `secretless.yml` using:
+```yaml
+from: vault
+get: /path/to/secret/in/vault
+```
+for the credentials in the 
+service connector configuration. Note that Vault supports various backends 
+that store different kinds of secrets. The Vault provider supports: 
+cubbyhole, KV v1 and KV v2. It may support other secret engines of Vault, specifically 
+those whose reading of secret at given path yields value at given field, i.e. similar 
+to the cubbyhole and KV secret engines.
+
+The provider will read a secret at a given path and returns the value of the field 
+`value` (by default) from the secret. By appending `#fieldName` to the path, the 
+provider will instead read the value at the field `fieldName`. For example: 
+`/path/to/secret#fieldName`. By prepending the backend name `kv-v2:` to the path, 
+the provider will change the way it handles the secret returned by the Vault client. 
+This is due to the KV v2 backend behaving differently in Vault. For example: 
+`kv-v2/path/to/secret`. This variant supports explicit field name too, e.g. 
+`kv-v2/path/to/secret#fieldName`. Please keep in mind that the path to the secret 
+in KV v2 has a `data` segment in the path in Vault, e.g. `secret/data/to/my/secret`. 
+This is specific to the KV v2 backend.
+
+Below are some examples showing how to configure the provider for secrets.
+
+### Example: API key from KV backends (v1 and v2)
+
+Here's an excerpt of an example configuration for a fictional "Example Service"
+that requires an API key, e.g. used in a request header. It gets the API key 
+from Vault's KV version 1 backend at path `kv/example-service` 
+under the secret's `value` field.
+
+```yaml
+version: 2
+services:
+  my_example_service:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8080
+    credentials:
+      apikey:
+        from: vault
+        get: kv/example-service  # path to API key in Vault, field 'value' holds the API key
+    ...
+```
+
+A slightly different configuration explicitly sets the field `api-key` 
+(instead of the default `value`) to hold the API key.
+
+```yaml
+version: 2
+services:
+  my_example_service:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8080
+    credentials:
+      apikey:
+        from: vault
+        get: kv/example-service#api-key  # path to API key in Vault, field 'api-key' holds the API key 
+    ...
+```
+If the secret is stored in a KV v2 backend (mounted at `secret` by default), 
+the configuration must use the `kv-v2:` prefix to tell the provider to 
+behave accordingly when reading the secret.
+
+```yaml
+version: 2
+services:
+  my_example_service:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8080
+    credentials:
+      apikey:
+        from: vault
+        get: kv-v2:secret/data/example-service#api-key  # path to API key in Vault, field 'api-key' holds the API key 
+    ...
+```
+
+## Limitations
+
+- Only token-based login to Vault supported at the moment.
+- Due to differences in e.g. the Vault KV version 1 and 2 backends, a prefix 
+  to the path is required to instruct Secretless to handle retrieval 
+  appropriately.
+- Backends that have multiple values change simultaneously (e.g. client 
+  id and secret, or database username and password) are not supported at the moment.

internal/providers/vault/provider.go

@@ -16,7 +16,10 @@ type Provider struct {
 }
 
 // ProviderFactory constructs a Provider. The API client is configured from
-// environment variables.
+// environment variables. Underlying Vault API client by default uses:
+// - VAULT_ADDR: endpoint of Vault, e.g. http://vault:8200/
+// - VAULT_TOKEN: token to login to Vault
+// Please see Vault API docs at https://godoc.org/github.com/hashicorp/vault/api
 func ProviderFactory(options plugin_v1.ProviderOptions) (plugin_v1.Provider, error) {
 	config := vault.DefaultConfig()
 
@@ -43,6 +46,20 @@ func (p *Provider) GetName() string {
 // hash returned by the Vault.
 const VaultDefaultField = "value"
 
+// VaultBackendKVv2 is the Vault backend KV version 2.
+const VaultBackendKVv2 = "kv-v2"
+
+// parseVaultBackend returns the (optional) backend of the backend from the id.
+func parseVaultBackend(id string) (string, string) {
+	tokens := strings.SplitN(id, ":", 2)
+	switch len(tokens) {
+	case 1:
+		return "", id
+	default:
+		return tokens[0], tokens[1]
+	}
+}
+
 // parseVaultID returns the secret id and field name.
 func parseVaultID(id string) (string, string) {
 	tokens := strings.SplitN(id, "#", 2)
@@ -54,31 +71,40 @@ func parseVaultID(id string) (string, string) {
 	}
 }
 
-// GetValue obtains a value by id. Any secret which is stored in the vault is recognized.
-// The datatype returned by Vault is map[string]interface{}. Therefore this provider needs
-// to know which field to return from the map. By default, it returns the 'value'.
-// An alternative field can be obtained by appending '#fieldName' to the id argument.
-func (p *Provider) GetValue(id string) (value []byte, err error) {
-	id, fieldName := parseVaultID(id)
-
-	var secret *vault.Secret
-	if secret, err = p.Client.Logical().Read(id); err != nil {
+// parseVaultGenericSecret returns secret as value of given field at given path.
+// This supports e.g. the KV v1 backend, database backend of Vault.
+func parseVaultGenericSecret(secret *vault.Secret, path, fieldName string) (value []byte, err error) {
+	valueObj, ok := secret.Data[fieldName]
+	if !ok {
+		err = fmt.Errorf("HashiCorp Vault provider expects the secret '%s' to contain field '%s'", path, fieldName)
 		return
 	}
-	// secret can be nil if it's not found
-	if secret == nil {
-		err = fmt.Errorf("HashiCorp Vault provider could not find a secret called '%s'", id)
+
+	return parseVaultSecretValue(valueObj)
+}
+
+// parseVaultKVv2Secret extracts secret as value of given field at given path,
+// unwrapping the additional "data" object in KV v2 backend.
+func parseVaultKVv2Secret(secret *vault.Secret, path, fieldName string) (value []byte, err error) {
+	// KV v2 backend wraps secret in a "data" object, unlike the KV v1 backend, e.g.
+	// see "Read Secret Version" in https://www.vaultproject.io/api/secret/kv/kv-v2.html
+	wrappedObj, ok := secret.Data["data"].(map[string]interface{})
+	if !ok {
+		err = fmt.Errorf("HashiCorp Vault provider expects the KV v2 secret '%s' wrapper 'data'", path)
 		return
 	}
 
-	var ok bool
-	var valueObj interface{}
-	valueObj, ok = secret.Data[fieldName]
+	valueObj, ok := wrappedObj[fieldName]
 	if !ok {
-		err = fmt.Errorf("HashiCorp Vault provider expects the secret '%s' to contain field '%s'", id, fieldName)
+		err = fmt.Errorf("HashiCorp Vault provider expects the KV v2 secret '%s' to contain field '%s'", path, fieldName)
 		return
 	}
 
+	return parseVaultSecretValue(valueObj)
+}
+
+// parseVaultSecretValue as either string or bytes, error otherwise.
+func parseVaultSecretValue(valueObj interface{}) (value []byte, err error) {
 	switch v := valueObj.(type) {
 	case string:
 		value = []byte(v)
@@ -89,3 +115,41 @@ func (p *Provider) GetValue(id string) (value []byte, err error) {
 	}
 	return
 }
+
+// GetValue obtains a value by id. The id should contain the path in Vault to the secret.
+// Prepend the path with `kv-v2:` when reading a secret from the KV v2 backend.
+// Append the path with `#fieldName` to fetch the value at given field; defaults to VaultDefaultField.
+// Example paths:
+//   - `kv/database/password` returns the value of field `value` in the secret object at given path.
+//   - `kv/database#password` returns the value of field `password` in the secret object at path `kv/database`.
+//   - `kv-v2:secret/data/database` returns the value of field `value` in secret object at path
+//     `secret/data/database`, unwrapping the secret data as required for KV v2 backend.
+//   - `kv-v2:secret/data/database#password` returns the value of field `password` in secret object at path
+//     `secret/data/database`, unwrapping the secret data as required for KV v2 backend.
+// Secrets in Vault are stored as objects in the shape of map[string]interface{}. In case of the KV v2 backend it is
+// wrapped in another object map[string]interface{} with a key/field `data` referencing the secret.
+// Note that in the Vault API the path to a KV v2 secret includes the `data` segment, while this is not required in
+// Vault CLI. Due to this provider using the Vault API client, the given id to GetValue must follow the conventions
+// of the Vault API path. Please see documentation of Vault for details.
+func (p *Provider) GetValue(id string) (value []byte, err error) {
+	backend, rest := parseVaultBackend(id)
+	path, fieldName := parseVaultID(rest)
+
+	var secret *vault.Secret
+	if secret, err = p.Client.Logical().Read(path); err != nil {
+		return
+	}
+	// secret can be nil if it's not found
+	if secret == nil {
+		err = fmt.Errorf("HashiCorp Vault provider could not find a secret called '%s'", id)
+		return
+	}
+
+	// change secret parsing behavior depending on backend
+	switch backend {
+	case VaultBackendKVv2:
+		return parseVaultKVv2Secret(secret, path, fieldName)
+	default:
+		return parseVaultGenericSecret(secret, path, fieldName)
+	}
+}

test/providers/vault/start

@@ -49,7 +49,14 @@ VAULT_ADDR=http://localhost:$vault_port
 VAULT_TOKEN=$root_token
 ENV
 
-vault_cmd secrets enable kv
+# cubbyhole enabled by default, mounted at /cubbyhole
+vault_cmd write cubbyhole/first-secret 'some-key=one'
+vault_cmd write cubbyhole/second-secret 'value=two'
+
+# KV v2 enabled by default, mounted at /secret
+vault_cmd secrets enable -version=1 kv
 vault_cmd kv put kv/db/password 'password=db-secret'
 vault_cmd kv put kv/frontend/admin-password 'password=frontend-secret'
 vault_cmd kv put kv/web/password 'value=web-secret'
+vault_cmd kv put secret/service 'api-key=service-api-key'
+vault_cmd kv put secret/message 'value=exposed'

test/providers/vault/vault_provider_test.go

@@ -35,15 +35,41 @@ func TestVault_Provider(t *testing.T) {
 		So(value, ShouldBeNil)
 	})
 
-	Convey("Can provide a secret", t, func() {
+	Convey("Can provide a cubbyhole secret", t, func() {
+		value, err := provider.GetValue("cubbyhole/first-secret#some-key")
+		So(err, ShouldBeNil)
+		So(string(value), ShouldEqual, "one")
+	})
+
+	Convey("Can provide a cubbyhole secret with default field name", t, func() {
+		value, err := provider.GetValue("cubbyhole/second-secret")
+		So(err, ShouldBeNil)
+		So(string(value), ShouldEqual, "two")
+	})
+
+	Convey("Can provide a KV v1 secret", t, func() {
 		value, err := provider.GetValue("kv/db/password#password")
 		So(err, ShouldBeNil)
 		So(string(value), ShouldEqual, "db-secret")
 	})
 
-	Convey("Can provide a secret with default field name", t, func() {
+	Convey("Can provide a KV v1 secret with default field name", t, func() {
 		value, err := provider.GetValue("kv/web/password")
 		So(err, ShouldBeNil)
 		So(string(value), ShouldEqual, "web-secret")
 	})
+
+	// note the "data" in path, which is required in KV v2
+	Convey("Can provide latest KV v2 secret", t, func() {
+		value, err := provider.GetValue("kv-v2:secret/data/service#api-key")
+		So(err, ShouldBeNil)
+		So(string(value), ShouldEqual, "service-api-key")
+	})
+
+	// note the "data" in path, which is required in KV v2
+	Convey("Can provide latest KV v2 secret with default field name", t, func() {
+		value, err := provider.GetValue("kv-v2:secret/data/message")
+		So(err, ShouldBeNil)
+		So(string(value), ShouldEqual, "exposed")
+	})
 }

test/providers/vault/vault_summon_test.go

@@ -14,8 +14,12 @@ import (
 
 func TestVault_Summon(t *testing.T) {
 	secretsDescriptor := `
+FIRST_SECRET: !var cubbyhole/first-secret#some-key
+SECOND_SECRET: !var cubbyhole/second-secret
 DB_PASSWORD: !var kv/db/password#password
 WEB_PASSWORD: !var kv/web/password
+SVC_API_KEY: !var kv-v2:secret/data/service#api-key
+MSG: !var kv-v2:secret/data/message
 `
 	defaultArgs := []string{"summon2", "-p", "vault", "--yaml", secretsDescriptor, "env"}
 
@@ -36,7 +40,11 @@ WEB_PASSWORD: !var kv/web/password
 		lines, err := runCommand(defaultArgs)
 
 		So(err, ShouldBeNil)
+		So(lines, ShouldContain, "FIRST_SECRET=one")
+		So(lines, ShouldContain, "SECOND_SECRET=two")
 		So(lines, ShouldContain, "DB_PASSWORD=db-secret")
 		So(lines, ShouldContain, "WEB_PASSWORD=web-secret")
+		So(lines, ShouldContain, "SVC_API_KEY=service-api-key")
+		So(lines, ShouldContain, "MSG=exposed")
 	})
 }