Skip to content

Commit

Permalink
Improve Vault error handling
Browse files Browse the repository at this point in the history
  • Loading branch information
dadgar committed Apr 3, 2018
1 parent b4bb89d commit 16ec448
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 13 deletions.
23 changes: 20 additions & 3 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -1939,10 +1939,27 @@ func (c *Client) deriveToken(alloc *structs.Allocation, taskNames []string, vcli
// Unwrap the vault token
unwrapResp, err := vclient.Logical().Unwrap(wrappedToken)
if err != nil {
return nil, fmt.Errorf("failed to unwrap the token for task %q: %v", taskName, err)
if structs.VaultUnrecoverableError.MatchString(err.Error()) {
return nil, err
}

// The error is recoverable
return nil, structs.NewRecoverableError(
fmt.Errorf("failed to unwrap the token for task %q: %v", taskName, err), true)
}

// Validate the response
var validationErr error
if unwrapResp == nil {
validationErr = fmt.Errorf("Vault returned nil secret when unwrapping")
} else if unwrapResp.Auth == nil {
validationErr = fmt.Errorf("Vault returned unwrap secret with nil Auth. Secret warnings: %v", unwrapResp.Warnings)
} else if unwrapResp.Auth.ClientToken == "" {
validationErr = fmt.Errorf("Vault returned unwrap secret with empty Auth.ClientToken. Secret warnings: %v", unwrapResp.Warnings)
}
if unwrapResp == nil || unwrapResp.Auth == nil || unwrapResp.Auth.ClientToken == "" {
return nil, fmt.Errorf("failed to unwrap the token for task %q", taskName)
if validationErr != nil {
c.logger.Printf("[WARN] client.vault: failed to unwrap token: %v", err)
return nil, structs.NewRecoverableError(validationErr, true)
}

// Append the unwrapped token to the return value
Expand Down
4 changes: 0 additions & 4 deletions nomad/node_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -1398,10 +1398,6 @@ func (n *Node) DeriveVaultToken(args *structs.DeriveVaultTokenRequest,
tokens := make(map[string]string, len(results))
for task, secret := range results {
w := secret.WrapInfo
if w == nil {
return fmt.Errorf("Vault returned Secret without WrapInfo")
}

tokens[task] = w.Token
accessor := &structs.VaultAccessor{
Accessor: w.WrappedAccessor,
Expand Down
6 changes: 6 additions & 0 deletions nomad/structs/structs.go
Original file line number Diff line number Diff line change
Expand Up @@ -5136,6 +5136,12 @@ func (d *EphemeralDisk) Copy() *EphemeralDisk {
return ld
}

var (
// VaultUnrecoverableError matches unrecoverable errors returned by a Vault
// server
VaultUnrecoverableError = regexp.MustCompile(`Code:\s+40(0|3|4)`)
)

const (
// VaultChangeModeNoop takes no action when a new token is retrieved.
VaultChangeModeNoop = "noop"
Expand Down
23 changes: 17 additions & 6 deletions nomad/vault.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"log"
"math/rand"
"regexp"
"sync"
"sync/atomic"
"time"
Expand Down Expand Up @@ -70,9 +69,6 @@ const (
)

var (
// vaultUnrecoverableError matches unrecoverable errors
vaultUnrecoverableError = regexp.MustCompile(`Code:\s+40(0|3|4)`)

// vaultCapabilitiesCapability is the expected capability of Nomad's Vault
// token on the the path. The token must have at least one of the
// capabilities.
Expand Down Expand Up @@ -695,7 +691,7 @@ func (v *vaultClient) validateCapabilities(role string, root bool) error {
_, _, err := v.hasCapability(vaultCapabilitiesLookupPath, vaultCapabilitiesCapability)
if err != nil {
// Check if there is a permission denied
if vaultUnrecoverableError.MatchString(err.Error()) {
if structs.VaultUnrecoverableError.MatchString(err.Error()) {
// Since we can't read permissions, we just log a warning that we
// can't tell if the Vault token will work
msg := fmt.Sprintf("Can not lookup token capabilities. "+
Expand Down Expand Up @@ -894,14 +890,29 @@ func (v *vaultClient) CreateToken(ctx context.Context, a *structs.Allocation, ta

// Determine whether it is unrecoverable
if err != nil {
if vaultUnrecoverableError.MatchString(err.Error()) {
if structs.VaultUnrecoverableError.MatchString(err.Error()) {
return secret, err
}

// The error is recoverable
return nil, structs.NewRecoverableError(err, true)
}

// Validate the response
var validationErr error
if secret == nil {
validationErr = fmt.Errorf("Vault returned nil Secret")
} else if secret.WrapInfo == nil {
validationErr = fmt.Errorf("Vault returned Secret with nil WrapInfo. Secret warnings: %v", secret.Warnings)
} else if secret.WrapInfo.WrappedAccessor == "" {
validationErr = fmt.Errorf("Vault returned WrapInfo without WrappedAccessor. Secret warnings: %v", secret.Warnings)
}
if validationErr != nil {
v.logger.Printf("[WARN] vault: failed to CreateToken: %v", err)
return nil, structs.NewRecoverableError(validationErr, true)
}

// Got a valid response
return secret, nil
}

Expand Down

0 comments on commit 16ec448

Please sign in to comment.