Merge remote-tracking branch 'origin/master' into auth_aws_bind_list

.travis.yml

@@ -7,7 +7,7 @@ services:
   - docker
 
 go:
-  - 1.9.1
+  - "1.10"
 
 matrix:
   allow_failures:

CHANGELOG.md

@@ -1,10 +1,54 @@
-## 0.9.4 (Unreleased)
+## 0.9.4 (February 20th, 2018)
+
+SECURITY:
+
+ * Role Tags used with the EC2 style of AWS auth were being improperly parsed;
+   as a result they were not being used to properly restrict values.
+   Implementations following our suggestion of using these as defense-in-depth
+   rather than the only source of restriction should not have significant
+   impact.
+
+FEATURES:
+
+ * **ChaCha20-Poly1305 support in `transit`**: You can now encrypt and decrypt
+   with ChaCha20-Poly1305 in `transit`. Key derivation and convergent
+   encryption is also supported.
+ * **Okta Push support in Okta Auth Backend**: If a user account has MFA
+   required within Okta, an Okta Push MFA flow can be used to successfully
+   finish authentication.
+ * **PKI Improvements**: Custom OID subject alternate names can now be set,
+   subject to allow restrictions that support globbing. Additionally, Country,
+   Locality, Province, Street Address, and Postal Code can now be set in
+   certificate subjects.
+ * **Manta Storage**: Joyent Triton Manta can now be used for Vault storage
+ * **Google Cloud Spanner Storage**: Google Cloud Spanner can now be used for
+   Vault storage
 
 IMPROVEMENTS:
 
  * auth/centrify: Add CLI helper
  * audit: Always log failure metrics, even if zero, to ensure the values appear
    on dashboards [GH-3937]
+ * cli: Disable color when output is not a TTY [GH-3897]
+ * cli: Add `-format` flag to all subcommands [GH-3897]
+ * cli: Do not display deprecation warnings when the format is not table
+   [GH-3897]
+ * core: If over a predefined lease count (256k), log a warning not more than
+   once a minute. Too many leases can be problematic for many of the storage
+   backends and often this number of leases is indicative of a need for
+   workflow improvements. [GH-3957]
+ * secret/nomad: Have generated ACL tokens cap out at 64 characters [GH-4009]
+ * secret/pki: Country, Locality, Province, Street Address, and Postal Code can
+   now be set on certificates [GH-3992]
+ * secret/pki: UTF-8 Other Names can now be set in Subject Alternate Names in
+   issued certs; allowed values can be set per role and support globbing
+   [GH-3889]
+ * secret/pki: Add a flag to make the common name optional on certs [GH-3940]
+ * secret/pki: Ensure only DNS-compatible names go into DNS SANs; additionally,
+   properly handle IDNA transformations for these DNS names [GH-3953]
+ * secret/ssh: Add `valid-principles` flag to CLI for CA mode [GH-3922]
+ * storage/manta: Add Manta storage [GH-3270]
+ * ui (Enterprise): Support for ChaCha20-Poly1305 keys in the transit engine.
 
 BUG FIXES:
 
@@ -14,17 +58,39 @@ BUG FIXES:
  * auth/approle: Cleanup of secret ID accessors during tidy and removal of
    dangling accessor entries [GH-3924]
  * auth/aws-ec2: Avoid masking of role tag response [GH-3941]
+ * auth/cert: Verify DNS SANs in the authenticating certificate [GH-3982]
  * auth/okta: Return configured durations as seconds, not nanoseconds [GH-3871]
  * auth/token: Token creation via the CLI no longer forces periodic token
    creation. Passing an explicit zero value for the period no longer create
    periodic tokens. [GH-3880]
+ * command: Fix interpreted formatting directives when printing raw fields
+   [GH-4005]
+ * command: Correctly format output when using -field and -format flags at the
+   same time [GH-3987]
+ * command/rekey: Re-add lost `stored-shares` parameter [GH-3974]
  * command/ssh: Create and reuse the api client [GH-3909]
+ * command/status: Fix panic when status returns 500 from leadership lookup
+   [GH-3998]
+ * identity: Fix race when creating entities [GH-3932]
+ * plugin/gRPC: Fixed an issue with list requests and raw responses coming from 
+   plugins using gRPC transport [GH-3881]
+ * plugin/gRPC: Fix panic when special paths are not set [GH-3946]
+ * secret/pki: Verify a name is a valid hostname before adding to DNS SANs
+   [GH-3918]
+ * secret/transit: Fix auditing when reading a key after it has been backed up
+   or restored [GH-3919]
+ * secret/transit: Fix storage/memory consistency when persistence fails
+   [GH-3959]
+ * storage/consul: Validate that service names are RFC 1123 compliant [GH-3960]
  * storage/etcd3: Fix memory ballooning with standby instances [GH-3798]
  * storage/etcd3: Fix large lists (like token loading at startup) not being
    handled [GH-3772]
+ * storage/postgresql: Fix compatibility with versions using custom string
+   version tags [GH-3949]
  * storage/zookeeper: Update vendoring to fix freezing issues [GH-3896]
- * plugin/gRPC: Fixed an issue with list requests and raw responses coming from 
-   plugins using gRPC transport [GH-3881]
+ * ui (Enterprise): Decoding the replication token should no longer error and
+   prevent enabling of a secondary replication cluster via the ui.
+ * plugin/gRPC: Add connection info to the request object [GH-3997]
 
 ## 0.9.3 (January 28th, 2018)
 
@@ -330,10 +396,12 @@ IMPROVEMENTS:
  * api: Add ability to set custom headers on each call [GH-3394]
  * command/server: Add config option to disable requesting client certificates
    [GH-3373]
+ * auth/aws: Max retries can now be customized for the AWS client [GH-3965]
  * core: Disallow mounting underneath an existing path, not just over [GH-2919]
  * physical/file: Use `700` as permissions when creating directories. The files
    themselves were `600` and are all encrypted, but this doesn't hurt.
  * secret/aws: Add ability to use custom IAM/STS endpoints [GH-3416]
+ * secret/aws: Max retries can now be customized for the AWS client [GH-3965]
  * secret/cassandra: Work around Cassandra ignoring consistency levels for a
    user listing query [GH-3469]
  * secret/pki: Private keys can now be marshalled as PKCS#8 [GH-3518]

Makefile

@@ -11,7 +11,7 @@ EXTERNAL_TOOLS=\
 BUILD_TAGS?=vault
 GOFMT_FILES?=$$(find . -name '*.go' | grep -v vendor)
 
-GO_VERSION_MIN=1.9.1
+GO_VERSION_MIN=1.10
 
 default: dev
 

README.md

@@ -59,7 +59,7 @@ Developing Vault
 --------------------
 
 If you wish to work on Vault itself or any of its built-in systems, you'll
-first need [Go](https://www.golang.org) installed on your machine (version 1.9+
+first need [Go](https://www.golang.org) installed on your machine (version 1.10+
 is *required*).
 
 For local dev first make sure Go is properly installed, including setting up a

api/sys_generate_root.go

@@ -99,11 +99,11 @@ func (c *Sys) generateRootUpdateCommon(path, shard, nonce string) (*GenerateRoot
 }
 
 type GenerateRootStatusResponse struct {
-	Nonce            string
-	Started          bool
-	Progress         int
-	Required         int
-	Complete         bool
+	Nonce            string `json:"nonce"`
+	Started          bool   `json:"started"`
+	Progress         int    `json:"progress"`
+	Required         int    `json:"required"`
+	Complete         bool   `json:"complete"`
 	EncodedToken     string `json:"encoded_token"`
 	EncodedRootToken string `json:"encoded_root_token"`
 	PGPFingerprint   string `json:"pgp_fingerprint"`

api/sys_rekey.go

@@ -177,27 +177,27 @@ type RekeyInitRequest struct {
 }
 
 type RekeyStatusResponse struct {
-	Nonce           string
-	Started         bool
-	T               int
-	N               int
-	Progress        int
-	Required        int
+	Nonce           string   `json:"nonce"`
+	Started         bool     `json:"started"`
+	T               int      `json:"t"`
+	N               int      `json:"n"`
+	Progress        int      `json:"progress"`
+	Required        int      `json:"required"`
 	PGPFingerprints []string `json:"pgp_fingerprints"`
-	Backup          bool
+	Backup          bool     `json:"backup"`
 }
 
 type RekeyUpdateResponse struct {
-	Nonce           string
-	Complete        bool
-	Keys            []string
+	Nonce           string   `json:"nonce"`
+	Complete        bool     `json:"complete"`
+	Keys            []string `json:"keys"`
 	KeysB64         []string `json:"keys_base64"`
 	PGPFingerprints []string `json:"pgp_fingerprints"`
-	Backup          bool
+	Backup          bool     `json:"backup"`
 }
 
 type RekeyRetrieveResponse struct {
-	Nonce   string
-	Keys    map[string][]string
+	Nonce   string              `json:"nonce"`
+	Keys    map[string][]string `json:"keys"`
 	KeysB64 map[string][]string `json:"keys_base64"`
 }

builtin/credential/aws/client.go

@@ -34,6 +34,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg
 	}
 
 	endpoint := aws.String("")
+	var maxRetries int = aws.UseServiceDefaultRetries
 	if config != nil {
 		// Override the default endpoint with the configured endpoint.
 		switch {
@@ -47,6 +48,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg
 
 		credsConfig.AccessKey = config.AccessKey
 		credsConfig.SecretKey = config.SecretKey
+		maxRetries = config.MaxRetries
 	}
 
 	credsConfig.HTTPClient = cleanhttp.DefaultClient()
@@ -65,6 +67,7 @@ func (b *backend) getRawClientConfig(ctx context.Context, s logical.Storage, reg
 		Region:      aws.String(region),
 		HTTPClient:  cleanhttp.DefaultClient(),
 		Endpoint:    endpoint,
+		MaxRetries:  aws.Int(maxRetries),
 	}, nil
 }
 

builtin/credential/aws/path_config_client.go

@@ -3,6 +3,7 @@ package awsauth
 import (
 	"context"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/fatih/structs"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/logical/framework"
@@ -47,6 +48,11 @@ func pathConfigClient(b *backend) *framework.Path {
 				Default:     "",
 				Description: "Value to require in the X-Vault-AWS-IAM-Server-ID request header",
 			},
+			"max_retries": &framework.FieldSchema{
+				Type:        framework.TypeInt,
+				Default:     aws.UseServiceDefaultRetries,
+				Description: "Maximum number of retries for recoverable exceptions of AWS APIs",
+			},
 		},
 
 		ExistenceCheck: b.pathConfigClientExistenceCheck,
@@ -220,6 +226,13 @@ func (b *backend) pathConfigClientCreateUpdate(ctx context.Context, req *logical
 		configEntry.IAMServerIdHeaderValue = data.Get("iam_server_id_header_value").(string)
 	}
 
+	maxRetriesInt, ok := data.GetOk("max_retries")
+	if ok {
+		configEntry.MaxRetries = maxRetriesInt.(int)
+	} else if req.Operation == logical.CreateOperation {
+		configEntry.MaxRetries = data.Get("max_retries").(int)
+	}
+
 	// Since this endpoint supports both create operation and update operation,
 	// the error checks for access_key and secret_key not being set are not present.
 	// This allows calling this endpoint multiple times to provide the values.
@@ -254,6 +267,7 @@ type clientConfig struct {
 	IAMEndpoint            string `json:"iam_endpoint" structs:"iam_endpoint" mapstructure:"iam_endpoint"`
 	STSEndpoint            string `json:"sts_endpoint" structs:"sts_endpoint" mapstructure:"sts_endpoint"`
 	IAMServerIdHeaderValue string `json:"iam_server_id_header_value" structs:"iam_server_id_header_value" mapstructure:"iam_server_id_header_value"`
+	MaxRetries             int    `json:"max_retries"`
 }
 
 const pathConfigClientHelpSyn = `

builtin/credential/aws/path_login.go

@@ -1534,6 +1534,9 @@ func submitCallerIdentityRequest(method, endpoint string, parsedUrl *url.URL, bo
 	// the endpoint to talk to alternate web addresses
 	request := buildHttpRequest(method, endpoint, parsedUrl, body, headers)
 	client := cleanhttp.DefaultClient()
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	}
 	response, err := client.Do(request)
 	if err != nil {
 		return nil, fmt.Errorf("error making request: %v", err)