Merge remote-tracking branch 'oss/master' into cache-exceptions

* oss/master:
  changelog++
  Support MongoDB session-wide write concern (#3646)
  Clarify api_addr related errors on VaultPluginTLSProvider (#3620)
  allowed/disallowed_policies as TypeCommaStringSlice (#3641)
  Update example payload and response for pem_keys field which needs \n after header and before footer in order to be accepted as a valid RSA or ECDSA public key (#3632)
  Docs: Update /sys/policies/ re: beta refs to address #3624 (#3629)
  Update secrets page
  Remove beta notice
  Expanding on the quick start guide with how to set up an intermediate authority (#3622)
  Docs: mlock() notes, fixes #3605 (#3614)
  Fix spelling (#3609)
  Add command to example to register plugin (#3601)
  update relatedtools, add Goldfish UI. (#3597)
  Fix docs for Transit API (#3588)
  Update cassandra docs with consistency value.
  Remove Trailing White space in Kubernetes Doc (#3360)
  Missing  command for vault PUT operation (#3355)
  Update some rekey docs

CHANGELOG.md

@@ -1,11 +1,25 @@
 ## 0.9.1 (Unreleased)
 
+DEPRECATIONS/CHANGES:
+
+ * Token Auth Backend Roles parameter types: For `allowed_policies` and
+   `disallowed_policies` in role definitions in the token auth backend, input
+   can now be a comma-separated string or an array of strings. Reading a role
+   will now return arrays for these parameters.
+
 IMPROVEMENTS:
 
+ * auth/token: `allowed_policies` and `disallowed_policies` can now be specified
+   as a comma-separated string or an array of strings [GH-3641]
+ * database/mongodb: Add optional `write_concern` parameter, which can be set
+   during database configuration. This establishes a session-wide [write
+   concern](https://docs.mongodb.com/manual/reference/write-concern/) for the
+   lifecycle of the mount [GH-3646]
+
 BUG FIXES:
 
- * database/mysql: Allow the creation statement to use commands that are not
-   yet supported by the prepare statement protocol [GH-3619]
+ * database/mysql: Allow the creation statement to use commands that are not yet
+   supported by the prepare statement protocol [GH-3619]
  * core: Fix potential panic that could occur using plugins when a node
    transitioned from active to standby [GH-3638]
 

helper/pluginutil/tls.go

@@ -146,19 +146,19 @@ func VaultPluginTLSProvider(apiTLSConfig *api.TLSConfig) func() (*tls.Config, er
 
 		addrRaw := wt.Claims().Get("addr")
 		if addrRaw == nil {
-			return nil, errors.New("decoded token does not contain primary cluster address")
+			return nil, errors.New("decoded token does not contain the active node's api_addr")
 		}
 		vaultAddr, ok := addrRaw.(string)
 		if !ok {
-			return nil, errors.New("decoded token's address not valid")
+			return nil, errors.New("decoded token's api_addr not valid")
 		}
 		if vaultAddr == "" {
-			return nil, errors.New(`no address for the vault found`)
+			return nil, errors.New(`no vault api_addr found`)
 		}
 
 		// Sanity check the value
 		if _, err := url.Parse(vaultAddr); err != nil {
-			return nil, fmt.Errorf("error parsing the vault address: %s", err)
+			return nil, fmt.Errorf("error parsing the vault api_addr: %s", err)
 		}
 
 		// Unwrap the token

plugins/database/mongodb/connection_producer.go

@@ -2,6 +2,8 @@ package mongodb
 
 import (
 	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net"
@@ -21,10 +23,12 @@ import (
 // interface for databases to make connections.
 type mongoDBConnectionProducer struct {
 	ConnectionURL string `json:"connection_url" structs:"connection_url" mapstructure:"connection_url"`
+	WriteConcern  string `json:"write_concern" structs:"write_concern" mapstructure:"write_concern"`
 
 	Initialized bool
 	Type        string
 	session     *mgo.Session
+	safe        *mgo.Safe
 	sync.Mutex
 }
 
@@ -42,6 +46,30 @@ func (c *mongoDBConnectionProducer) Initialize(conf map[string]interface{}, veri
 		return fmt.Errorf("connection_url cannot be empty")
 	}
 
+	if c.WriteConcern != "" {
+		input := c.WriteConcern
+
+		// Try to base64 decode the input. If successful, consider the decoded
+		// value as input.
+		inputBytes, err := base64.StdEncoding.DecodeString(input)
+		if err == nil {
+			input = string(inputBytes)
+		}
+
+		concern := &mgo.Safe{}
+		err = json.Unmarshal([]byte(input), concern)
+		if err != nil {
+			return fmt.Errorf("error mashalling write_concern: %s", err)
+		}
+
+		// Guard against empty, non-nil mgo.Safe object; we don't want to pass that
+		// into mgo.SetSafe in Connection().
+		if (mgo.Safe{} == *concern) {
+			return fmt.Errorf("provided write_concern values did not map to any mgo.Safe fields")
+		}
+		c.safe = concern
+	}
+
 	// Set initialized to true at this point since all fields are set,
 	// and the connection can be established at a later time.
 	c.Initialized = true
@@ -78,6 +106,11 @@ func (c *mongoDBConnectionProducer) Connection() (interface{}, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	if c.safe != nil {
+		c.session.SetSafe(c.safe)
+	}
+
 	c.session.SetSyncTimeout(1 * time.Minute)
 	c.session.SetSocketTimeout(1 * time.Minute)
 

plugins/database/mongodb/mongodb_test.go

@@ -16,6 +16,8 @@ import (
 
 const testMongoDBRole = `{ "db": "admin", "roles": [ { "role": "readWrite" } ] }`
 
+const testMongoDBWriteConcern = `{ "wmode": "majority", "wtimeout": 5000 }`
+
 func prepareMongoDBTestContainer(t *testing.T) (cleanup func(), retURL string) {
 	if os.Getenv("MONGODB_URL") != "" {
 		return func() {}, os.Getenv("MONGODB_URL")
@@ -129,6 +131,44 @@ func TestMongoDB_CreateUser(t *testing.T) {
 	}
 }
 
+func TestMongoDB_CreateUser_writeConcern(t *testing.T) {
+	cleanup, connURL := prepareMongoDBTestContainer(t)
+	defer cleanup()
+
+	connectionDetails := map[string]interface{}{
+		"connection_url": connURL,
+		"write_concern":  testMongoDBWriteConcern,
+	}
+
+	dbRaw, err := New()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	db := dbRaw.(*MongoDB)
+	err = db.Initialize(connectionDetails, true)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	statements := dbplugin.Statements{
+		CreationStatements: testMongoDBRole,
+	}
+
+	usernameConfig := dbplugin.UsernameConfig{
+		DisplayName: "test",
+		RoleName:    "test",
+	}
+
+	username, password, err := db.CreateUser(statements, usernameConfig, time.Now().Add(time.Minute))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if err := testCredsExist(t, connURL, username, password); err != nil {
+		t.Fatalf("Could not connect with new credentials: %s", err)
+	}
+}
+
 func TestMongoDB_RevokeUser(t *testing.T) {
 	cleanup, connURL := prepareMongoDBTestContainer(t)
 	defer cleanup()

vault/token_store.go

@@ -178,14 +178,12 @@ func NewTokenStore(c *Core, config *logical.BackendConfig) (*TokenStore, error)
 					},
 
 					"allowed_policies": &framework.FieldSchema{
-						Type:        framework.TypeString,
-						Default:     "",
+						Type:        framework.TypeCommaStringSlice,
 						Description: tokenAllowedPoliciesHelp,
 					},
 
 					"disallowed_policies": &framework.FieldSchema{
-						Type:        framework.TypeString,
-						Default:     "",
+						Type:        framework.TypeCommaStringSlice,
 						Description: tokenDisallowedPoliciesHelp,
 					},
 
@@ -2465,18 +2463,18 @@ func (ts *TokenStore) tokenStoreRoleCreateUpdate(
 		return logical.ErrorResponse(fmt.Sprintf("error registering path suffix: %s", consts.ErrPathContainsParentReferences)), nil
 	}
 
-	allowedPoliciesStr, ok := data.GetOk("allowed_policies")
+	allowedPoliciesRaw, ok := data.GetOk("allowed_policies")
 	if ok {
-		entry.AllowedPolicies = policyutil.SanitizePolicies(strings.Split(allowedPoliciesStr.(string), ","), policyutil.DoNotAddDefaultPolicy)
+		entry.AllowedPolicies = policyutil.SanitizePolicies(allowedPoliciesRaw.([]string), policyutil.DoNotAddDefaultPolicy)
 	} else if req.Operation == logical.CreateOperation {
-		entry.AllowedPolicies = policyutil.SanitizePolicies(strings.Split(data.Get("allowed_policies").(string), ","), policyutil.DoNotAddDefaultPolicy)
+		entry.AllowedPolicies = policyutil.SanitizePolicies(data.Get("allowed_policies").([]string), policyutil.DoNotAddDefaultPolicy)
 	}
 
-	disallowedPoliciesStr, ok := data.GetOk("disallowed_policies")
+	disallowedPoliciesRaw, ok := data.GetOk("disallowed_policies")
 	if ok {
-		entry.DisallowedPolicies = strutil.ParseDedupLowercaseAndSortStrings(disallowedPoliciesStr.(string), ",")
+		entry.DisallowedPolicies = strutil.RemoveDuplicates(disallowedPoliciesRaw.([]string), true)
 	} else if req.Operation == logical.CreateOperation {
-		entry.DisallowedPolicies = strutil.ParseDedupLowercaseAndSortStrings(data.Get("disallowed_policies").(string), ",")
+		entry.DisallowedPolicies = strutil.RemoveDuplicates(data.Get("disallowed_policies").([]string), true)
 	}
 
 	// Store it

website/source/api/auth/kubernetes/index.html.md

@@ -46,7 +46,7 @@ access the Kubernetes API.
 {
   "kubernetes_host": "https://192.168.99.100:8443",
   "kubernetes_ca_cert": "-----BEGIN CERTIFICATE-----.....-----END CERTIFICATE-----",
-  "pem_keys": "-----BEGIN CERTIFICATE-----.....-----END CERTIFICATE-----"
+  "pem_keys": "-----BEGIN CERTIFICATE-----\n.....\n-----END CERTIFICATE-----"
 }
 ```
 
@@ -83,7 +83,9 @@ $ curl \
   "data":{
       "kubernetes_host": "https://192.168.99.100:8443",
       "kubernetes_ca_cert": "-----BEGIN CERTIFICATE-----.....-----END CERTIFICATE-----",
-      "pem_keys": "-----BEGIN CERTIFICATE-----.....-----END CERTIFICATE-----"
+      "pem_keys": "-----BEGIN CERTIFICATE-----
+      .....
+      -----END CERTIFICATE-----"
   },
   ...
 }

website/source/api/relatedtools.html.md

@@ -24,5 +24,6 @@ The following list of tools is maintained by the community of Vault users; Hashi
 * [vault-exec](https://github.com/kmanning/vault_exec) - a shell wrapper to execute arbitrary scripts using temporary AWS credentials managed by Vault
 * [pouch](https://github.com/tuenti/pouch) - A set of tools to manage provisioning of secrets on hosts based on the AppRole authentication method of Vault
 * [vault-aws-creds](https://github.com/jantman/vault-aws-creds) - Python helper to export Vault-provided temporary AWS creds into the environment
+* [goldfish](https://github.com/Caiyeon/goldfish) - A Vault UI panel written with VueJS and Vault native Go API.
 
 Want to add your own project, or one that you use? Additions are welcome via [pull requests](https://github.com/hashicorp/vault/blob/master/website/source/api/relatedtools.html.md).

website/source/api/secret/cassandra/index.html.md

@@ -62,6 +62,11 @@ Cassandra.
 
 - `connect_timeout` `(string: "5s")` – Specifies the connection timeout to use.
 
+- `consistency` `(string: "")` – Specifies the consistency option to use.  See
+  the [gocql
+  definition](https://github.com/gocql/gocql/blob/master/frame.go#L203) for
+  valid options.
+
 TLS works as follows:
 
 - If `tls` is set to true, the connection will use TLS; this happens

website/source/api/secret/databases/mongodb.html.md

@@ -20,18 +20,26 @@ has a number of parameters to further configure a connection.
 
 | Method   | Path                         | Produces               |
 | :------- | :--------------------------- | :--------------------- |
-| `POST`   | `/database/config/:name`     | `204 (empty body)` |
+| `POST`   | `/database/config/:name`     | `204 (empty body)`     |
 
 ### Parameters
-- `connection_url` `(string: <required>)` – Specifies the MongoDB standard connection string (URI).
+
+- `connection_url` `(string: <required>)` – Specifies the MongoDB standard
+  connection string (URI).
+- `write_concern` `(string: "")` - Specifies the MongoDB [write
+  concern][mongodb-write-concern]. This is set for the entirety of the session,
+  maintained for the lifecycle of the plugin process. Must be a serialized JSON
+  object, or a base64-encoded serialized JSON object. The JSON payload values
+  map to the values in the [Safe][mgo-safe] struct from the mgo driver.
 
 ### Sample Payload
 
 ```json
 {
   "plugin_name": "mongodb-database-plugin",
   "allowed_roles": "readonly",
-  "connection_url": "mongodb://admin:[email protected]:27017/admin?ssl=true"
+  "connection_url": "mongodb://admin:[email protected]:27017/admin?ssl=true",
+  "write_concern": "{ \"wmode\": \"majority\", \"wtimeout\": 5000 }"
 }
 ```
 
@@ -68,7 +76,7 @@ list the plugin does not support that statement type.
   [MongoDB's documentation](https://docs.mongodb.com/manual/reference/method/db.createUser/).
 
 - `revocation_statements` `(string: "")` – Specifies the database statements to
-  be executed to revoke a user. Must be a serialized JSON object, or a base64-encoded 
+  be executed to revoke a user. Must be a serialized JSON object, or a base64-encoded
   serialized JSON object. The object can optionally contain a "db" string. If no
   "db" value is provided, it defaults to the "admin" database.
 
@@ -84,4 +92,7 @@ list the plugin does not support that statement type.
     }
   ]
 }
-```
+```
+
+[mongodb-write-concern]: https://docs.mongodb.com/manual/reference/write-concern/
+[mgo-safe]: https://godoc.org/gopkg.in/mgo.v2#Safe

website/source/api/secret/ssh/index.html.md

@@ -143,7 +143,7 @@ This endpoint creates or updates a named role.
   credentials can be created for any domain. See also `allow_bare_domains` and
   `allow_subdomains`.
 
-- `key_option_specs` `(string: "")` – Specifies a aomma separated option
+- `key_option_specs` `(string: "")` – Specifies a comma separated option
   specification which will be prefixed to RSA keys in the remote host's
   authorized_keys file. N.B.: Vault does not check this string for validity.
 

website/source/api/secret/transit/index.html.md

@@ -790,7 +790,7 @@ supports signing.
    Required if key derivation is enabled; currently only available with ed25519
    keys.
 
- - `prehashed` `(bool: false)` - Set to `true` when the input is already
+- `prehashed` `(bool: false)` - Set to `true` when the input is already
    hashed. If the key type is `rsa-2048` or `rsa-4096`, then the algorithm used
    to hash the input should be indicated by the `algorithm` parameter.
 
@@ -855,11 +855,11 @@ data.
   `/transit/hmac` function. Either this must be supplied or `signature` must be
   supplied.
 
- - `context` `(string: "")` - Base64 encoded context for key derivation.
+- `context` `(string: "")` - Base64 encoded context for key derivation.
    Required if key derivation is enabled; currently only available with ed25519
    keys.
 
- - `prehashed` `(bool: false)` - Set to `true` when the input is already
+- `prehashed` `(bool: false)` - Set to `true` when the input is already
    hashed. If the key type is `rsa-2048` or `rsa-4096`, then the algorithm used
    to hash the input should be indicated by the `algorithm` parameter.
 

website/source/api/system/policies.html.md

@@ -6,18 +6,12 @@ description: |-
   The `/sys/policies/` endpoints are used to manage ACL, RGP, and EGP policies in Vault.
 ---
 
-# NOTE: Only in 0.9 Betas
-
-Please note that this endpoint is only available in 0.9 beta releases of Vault
-at this time.
-
 # `/sys/policies/`
 
 The `/sys/policies` endpoints are used to manage ACL, RGP, and EGP policies in Vault.
 
-Note: RGPs and EGPs are Vault Enterprise Premium features, and the associated endpoints are not available in Vault Open Source or Vault Enterprise Pro.
 
-In addition, `/sys/policies/acl` will be available in an upcoming Vault Open Source/Vault Enterprise Pro release, but not until Sentinel exits beta.
+~> **NOTE**: This endpoint is only available in Vault version 0.9+. Please also note that RGPs and EGPs are Vault Enterprise Premium features and the associated endpoints are not available in Vault Open Source or Vault Enterprise Pro.
 
 ## List ACL Policies
 

website/source/docs/auth/kubernetes.html.md

@@ -111,7 +111,7 @@ login it first must be configured in a role.
 
 ```
 vault write auth/kubernetes/role/demo \
-    bound_service_account_names=vault-auth \ 
+    bound_service_account_names=vault-auth \
     bound_service_account_namespaces=default \
     policies=default \
     ttl=1h

website/source/docs/configuration/index.html.md

@@ -88,6 +88,8 @@ to specify where the configuration is.
     sudo setcap cap_ipc_lock=+ep $(readlink -f $(which vault))
     ```
 
+    If you use a Linux distribution with systemd, you can also add the above `setcap` command as an [ExecStartPre](https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStartPre=) additional command in your Vault unit file to ensure that `mlock()` capability is added to the `vault` binary before executing.
+
 - `plugin_directory` `(string: "")` – A directory from which plugins are
   allowed to be loaded. Vault must have permission to read files in this
   directory to successfully load plugins.
@@ -119,9 +121,10 @@ to specify where the configuration is.
 
 The following parameters are used on backends that support [high availability][high-availability].
 
-- `api_addr` `(string: "")` - Specifies the address (full URL) to
-  advertise to other Vault servers in the cluster for client redirection. This
-  can also be provided via the environment variable `VAULT_API_ADDR`.
+- `api_addr` `(string: "")` - Specifies the address (full URL) to advertise to
+  other Vault servers in the cluster for client redirection. This value is also
+  used for [plugin backends][plugins]. This can also be provided via the
+  environment variable `VAULT_API_ADDR`.
 
 - `cluster_addr` `(string: "")` -  – Specifies the address to advertise to other
   Vault servers in the cluster for request forwarding. This can also be provided
@@ -139,3 +142,4 @@ The following parameters are used on backends that support [high availability][h
 [sealwrap]: /docs/enterprise/sealwrap/index.html
 [telemetry]: /docs/configuration/telemetry.html
 [high-availability]: /docs/concepts/ha.html
+[plugins]: /docs/plugin/index.html