Add AWS Secret Engine Root Credential Rotation

builtin/logical/aws/backend.go

@@ -36,6 +36,7 @@ func Backend() *backend {
 
 		Paths: []*framework.Path{
 			pathConfigRoot(&b),
+			pathConfigRotateRoot(&b),
 			pathConfigLease(&b),
 			pathRoles(&b),
 			pathListRoles(&b),
@@ -60,7 +61,7 @@ type backend struct {
 	// Mutex to protect access to reading and writing policies
 	roleMutex sync.RWMutex
 
-	// Mutex to protect access to iam/sts clients
+	// Mutex to protect access to iam/sts clients and client configs
 	clientMutex sync.RWMutex
 
 	// iamClient and stsClient hold configured iam and sts clients for reuse, and
@@ -99,7 +100,7 @@ func (b *backend) clientIAM(ctx context.Context, s logical.Storage) (iamiface.IA
 		return b.iamClient, nil
 	}
 
-	iamClient, err := clientIAM(ctx, s)
+	iamClient, err := nonCachedClientIAM(ctx, s)
 	if err != nil {
 		return nil, err
 	}
@@ -126,7 +127,7 @@ func (b *backend) clientSTS(ctx context.Context, s logical.Storage) (stsiface.ST
 		return b.stsClient, nil
 	}
 
-	stsClient, err := clientSTS(ctx, s)
+	stsClient, err := nonCachedClientSTS(ctx, s)
 	if err != nil {
 		return nil, err
 	}

builtin/logical/aws/backend_test.go

@@ -67,6 +67,7 @@ func TestBackend_basicSTS(t *testing.T) {
 		Backend: getBackend(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepConfigWithCreds(t, accessKey),
+			testAccStepRotateRoot(accessKey),
 			testAccStepWritePolicy(t, "test", testDynamoPolicy),
 			testAccStepRead(t, "sts", "test", []credentialTestFunc{listDynamoTablesTest}),
 			testAccStepWriteArnPolicyRef(t, "test", ec2PolicyArn),
@@ -433,6 +434,44 @@ func testAccStepConfigWithCreds(t *testing.T, accessKey *awsAccessKey) logicalte
 	}
 }
 
+func testAccStepRotateRoot(oldAccessKey *awsAccessKey) logicaltest.TestStep {
+	return logicaltest.TestStep{
+		Operation: logical.UpdateOperation,
+		Path:      "config/rotate-root",
+		Check: func(resp *logical.Response) error {
+			if resp == nil {
+				return fmt.Errorf("received nil response from config/rotate-root")
+			}
+			newAccessKeyID := resp.Data["access_key"].(string)
+			if newAccessKeyID == oldAccessKey.AccessKeyID {
+				return fmt.Errorf("rotate-root didn't rotate access key")
+			}
+			awsConfig := &aws.Config{
+				Region:      aws.String("us-east-1"),
+				HTTPClient:  cleanhttp.DefaultClient(),
+				Credentials: credentials.NewStaticCredentials(oldAccessKey.AccessKeyID, oldAccessKey.SecretAccessKey, ""),
+			}
+			// sigh....
+			oldAccessKey.AccessKeyID = newAccessKeyID
+			log.Println("[WARN] Sleeping for 10 seconds waiting for AWS...")
+			time.Sleep(10 * time.Second)
+			svc := sts.New(session.New(awsConfig))
+			params := &sts.GetCallerIdentityInput{}
+			_, err := svc.GetCallerIdentity(params)
+			if err == nil {
+				return fmt.Errorf("bad: old credentials succeeded after rotate")
+			}
+			if aerr, ok := err.(awserr.Error); ok {
+				if aerr.Code() != "InvalidClientTokenId" {
+					return fmt.Errorf("Unknown error returned from AWS: %#v", aerr)
+				}
+				return nil
+			}
+			return err
+		},
+	}
+}
+
 func testAccStepRead(t *testing.T, path, name string, credentialTests []credentialTestFunc) logicaltest.TestStep {
 	return logicaltest.TestStep{
 		Operation: logical.ReadOperation,

builtin/logical/aws/client.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/vault/logical"
 )
 
+// NOTE: The caller is required to ensure that b.clientMutex is at least read locked
 func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*aws.Config, error) {
 	credsConfig := &awsutil.CredentialsConfig{}
 	var endpoint string
@@ -68,7 +69,7 @@ func getRootConfig(ctx context.Context, s logical.Storage, clientType string) (*
 	}, nil
 }
 
-func clientIAM(ctx context.Context, s logical.Storage) (*iam.IAM, error) {
+func nonCachedClientIAM(ctx context.Context, s logical.Storage) (*iam.IAM, error) {
 	awsConfig, err := getRootConfig(ctx, s, "iam")
 	if err != nil {
 		return nil, err
@@ -82,7 +83,7 @@ func clientIAM(ctx context.Context, s logical.Storage) (*iam.IAM, error) {
 	return client, nil
 }
 
-func clientSTS(ctx context.Context, s logical.Storage) (*sts.STS, error) {
+func nonCachedClientSTS(ctx context.Context, s logical.Storage) (*sts.STS, error) {
 	awsConfig, err := getRootConfig(ctx, s, "sts")
 	if err != nil {
 		return nil, err

builtin/logical/aws/path_config_root.go

@@ -56,6 +56,9 @@ func (b *backend) pathConfigRootWrite(ctx context.Context, req *logical.Request,
 	stsendpoint := data.Get("sts_endpoint").(string)
 	maxretries := data.Get("max_retries").(int)
 
+	b.clientMutex.Lock()
+	defer b.clientMutex.Unlock()
+
 	entry, err := logical.StorageEntryJSON("config/root", rootConfig{
 		AccessKey:   data.Get("access_key").(string),
 		SecretKey:   data.Get("secret_key").(string),
@@ -74,8 +77,6 @@ func (b *backend) pathConfigRootWrite(ctx context.Context, req *logical.Request,
 
 	// clear possible cached IAM / STS clients after successfully updating
 	// config/root
-	b.clientMutex.Lock()
-	defer b.clientMutex.Unlock()
 	b.iamClient = nil
 	b.stsClient = nil
 

builtin/logical/aws/path_config_rotate_root.go

@@ -0,0 +1,124 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/vault/logical"
+	"github.com/hashicorp/vault/logical/framework"
+)
+
+func pathConfigRotateRoot(b *backend) *framework.Path {
+	return &framework.Path{
+		Pattern: "config/rotate-root",
+		Callbacks: map[logical.Operation]framework.OperationFunc{
+			logical.UpdateOperation: b.pathConfigRotateRootUpdate,
+		},
+
+		HelpSynopsis:    pathConfigRotateRootHelpSyn,
+		HelpDescription: pathConfigRotateRootHelpDesc,
+	}
+}
+
+func (b *backend) pathConfigRotateRootUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	// have to get the client config first because that takes out a read lock
+	client, err := b.clientIAM(ctx, req.Storage)
+	if err != nil {
+		return nil, err
+	}
+	if client == nil {
+		return nil, fmt.Errorf("nil IAM client")
+	}
+
+	b.clientMutex.Lock()
+	defer b.clientMutex.Unlock()
+
+	rawRootConfig, err := req.Storage.Get(ctx, "config/root")
+	if err != nil {
+		return nil, err
+	}
+	if rawRootConfig == nil {
+		return nil, fmt.Errorf("no configuration found for config/root")
+	}
+	var config rootConfig
+	if err := rawRootConfig.DecodeJSON(&config); err != nil {
+		return nil, errwrap.Wrapf("error reading root configuration: {{err}}", err)
+	}
+
+	if config.AccessKey == "" || config.SecretKey == "" {
+		return logical.ErrorResponse("Cannot call config/rotate-root when either access_key or secret_key is empty"), nil
+	}
+
+	var getUserInput iam.GetUserInput // empty input means get current user
+	getUserRes, err := client.GetUser(&getUserInput)
+	if err != nil {
+		return nil, errwrap.Wrapf("error calling GetUser: {{err}}", err)
+	}
+	if getUserRes == nil {
+		return nil, fmt.Errorf("nil response from GetUser")
+	}
+	if getUserRes.User == nil {
+		return nil, fmt.Errorf("nil user returned from GetUser")
+	}
+	if getUserRes.User.UserName == nil {
+		return nil, fmt.Errorf("nil UserName returned from GetUser")
+	}
+
+	createAccessKeyInput := iam.CreateAccessKeyInput{
+		UserName: getUserRes.User.UserName,
+	}
+	createAccessKeyRes, err := client.CreateAccessKey(&createAccessKeyInput)
+	if err != nil {
+		return nil, errwrap.Wrapf("error calling CreateAccessKey: {{err}}", err)
+	}
+	if createAccessKeyRes.AccessKey == nil {
+		return nil, fmt.Errorf("nil response from CreateAccessKey")
+	}
+	if createAccessKeyRes.AccessKey.AccessKeyId == nil || createAccessKeyRes.AccessKey.SecretAccessKey == nil {
+		return nil, fmt.Errorf("nil AccessKeyId or SecretAccessKey returned from CreateAccessKey")
+	}
+
+	oldAccessKey := config.AccessKey
+
+	config.AccessKey = *createAccessKeyRes.AccessKey.AccessKeyId
+	config.SecretKey = *createAccessKeyRes.AccessKey.SecretAccessKey
+
+	newEntry, err := logical.StorageEntryJSON("config/root", config)
+	if err != nil {
+		return nil, errwrap.Wrapf("error generating new config/root JSON: {{err}}", err)
+	}
+	if err := req.Storage.Put(ctx, newEntry); err != nil {
+		return nil, errwrap.Wrapf("error saving new config/root: {{err}}", err)
+	}
+
+	b.iamClient = nil
+	b.stsClient = nil
+
+	deleteAccessKeyInput := iam.DeleteAccessKeyInput{
+		AccessKeyId: aws.String(oldAccessKey),
+		UserName:    getUserRes.User.UserName,
+	}
+	_, err = client.DeleteAccessKey(&deleteAccessKeyInput)
+	if err != nil {
+		return nil, errwrap.Wrapf("error deleting old access key: {{err}}", err)
+	}
+
+	return &logical.Response{
+		Data: map[string]interface{}{
+			"access_key": config.AccessKey,
+		},
+	}, nil
+}
+
+const pathConfigRotateRootHelpSyn = `
+Request to rotate the AWS credentials used by Vault
+`
+
+const pathConfigRotateRootHelpDesc = `
+This path attempts to rotate the AWS credentials used by Vault for this mount.
+It is only valid if Vault has been configured to use AWS IAM credentials via the
+config/root endpoint.
+`

builtin/logical/aws/path_user.go

@@ -111,15 +111,15 @@ func (b *backend) pathCredsRead(ctx context.Context, req *logical.Request, d *fr
 	}
 }
 
-func pathUserRollback(ctx context.Context, req *logical.Request, _kind string, data interface{}) error {
+func (b *backend) pathUserRollback(ctx context.Context, req *logical.Request, _kind string, data interface{}) error {
 	var entry walUser
 	if err := mapstructure.Decode(data, &entry); err != nil {
 		return err
 	}
 	username := entry.UserName
 
 	// Get the client
-	client, err := clientIAM(ctx, req.Storage)
+	client, err := b.clientIAM(ctx, req.Storage)
 	if err != nil {
 		return err
 	}

builtin/logical/aws/rollback.go

@@ -9,11 +9,11 @@ import (
 	"github.com/hashicorp/vault/logical/framework"
 )
 
-var walRollbackMap = map[string]framework.WALRollbackFunc{
-	"user": pathUserRollback,
-}
-
 func (b *backend) walRollback(ctx context.Context, req *logical.Request, kind string, data interface{}) error {
+	walRollbackMap := map[string]framework.WALRollbackFunc{
+		"user": b.pathUserRollback,
+	}
+
 	if !b.System().LocalMount() && b.System().ReplicationState().HasState(consts.ReplicationPerformancePrimary) {
 		return nil
 	}

builtin/logical/aws/secret_access_keys.go

@@ -68,7 +68,6 @@ func genUsername(displayName, policyName, userType string) (ret string, warning
 func (b *backend) secretTokenCreate(ctx context.Context, s logical.Storage,
 	displayName, policyName, policy string,
 	lifeTimeInSeconds int64) (*logical.Response, error) {
-
 	stsClient, err := b.clientSTS(ctx, s)
 	if err != nil {
 		return logical.ErrorResponse(err.Error()), nil
@@ -114,7 +113,6 @@ func (b *backend) secretTokenCreate(ctx context.Context, s logical.Storage,
 func (b *backend) assumeRole(ctx context.Context, s logical.Storage,
 	displayName, roleName, roleArn, policy string,
 	lifeTimeInSeconds int64) (*logical.Response, error) {
-
 	stsClient, err := b.clientSTS(ctx, s)
 	if err != nil {
 		return logical.ErrorResponse(err.Error()), nil
@@ -164,7 +162,6 @@ func (b *backend) secretAccessKeysCreate(
 	ctx context.Context,
 	s logical.Storage,
 	displayName, policyName string, role *awsRoleEntry) (*logical.Response, error) {
-
 	iamClient, err := b.clientIAM(ctx, s)
 	if err != nil {
 		return logical.ErrorResponse(err.Error()), nil
@@ -313,7 +310,7 @@ func (b *backend) secretAccessKeysRevoke(ctx context.Context, req *logical.Reque
 	}
 
 	// Use the user rollback mechanism to delete this user
-	err := pathUserRollback(ctx, req, "user", map[string]interface{}{
+	err := b.pathUserRollback(ctx, req, "user", map[string]interface{}{
 		"username": username,
 	})
 	if err != nil {

website/source/api/secret/aws/index.html.md

@@ -80,6 +80,47 @@ $ curl \
     http://127.0.0.1:8200/v1/aws/config/root
 ```
 
+## Rotate Root IAM Credentials
+
+When you have configured Vault with static credentials, you can use this
+endpoint to have Vault rotate the access key it used. Note that, due to AWS
+eventual consistency, after calling this endpoint, subsequent calls from Vault
+to AWS may fail for a few seconds until AWS becomes consistent again.
+
+
+In order to call this endpoint, Vault's AWS access key MUST be the only access
+key on the IAM user; otherwise, generation of a new access key will fail. Once
+this method is called, Vault will now be the only entity that knows the AWS
+secret key is used to access AWS.
+
+| Method   | Path                         | Produces               |
+| :------- | :--------------------------- | :--------------------- |
+| `POST`   | `/aws/config/rotate-root`    | `200 application/json` |
+
+### Parameters
+
+There are no parameters to this operation.
+
+### Sample Request
+
+```$ curl \
+    --header "X-Vault-Token: ..." \
+    --request POST \
+    http://127.0.0.1:8211/v1/aws/config/rotate-root
+```
+
+### Sample Response
+
+```json
+{
+  "data": {
+    "access_key": "AKIA..."
+  }
+}
+```
+
+The new access key Vault uses is returned by this operation.
+
 ## Configure Lease
 
 This endpoint configures lease settings for the AWS secrets engine. It is