Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

backend/remote: also retry on server errors #20588

Merged
merged 2 commits into from
Mar 8, 2019
Merged

backend/remote: also retry on server errors #20588

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9517a00
gomod: update `go-tfe`
Mar 6, 2019
0232d84
backend/remote: also retry on server errors
Mar 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 48 additions & 4 deletions backend/remote/backend.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"sort"
"strings"
"sync"
"time"

tfe "github.com/hashicorp/go-tfe"
version "github.com/hashicorp/go-version"
Expand Down Expand Up @@ -56,6 +57,9 @@ type Remote struct {
// client is the remote backend API client.
client *tfe.Client

// lastRetry is set to the last time a request was retried.
lastRetry time.Time

// hostname of the remote backend server.
hostname string

Expand Down Expand Up @@ -279,10 +283,11 @@ func (b *Remote) Configure(obj cty.Value) tfdiags.Diagnostics {
}

cfg := &tfe.Config{
Address: service.String(),
BasePath: service.Path,
Token: token,
Headers: make(http.Header),
Address: service.String(),
BasePath: service.Path,
Token: token,
Headers: make(http.Header),
RetryLogHook: b.retryLogHook,
}

// Set the version header to the current version.
Expand Down Expand Up @@ -324,6 +329,9 @@ func (b *Remote) Configure(obj cty.Value) tfdiags.Diagnostics {
b.local = backendLocal.NewWithBackend(b)
b.forceLocal = b.forceLocal || !entitlements.Operations

// Enable retries for server errors as the backend is now fully configured.
b.client.RetryServerErrors(true)

return diags
}

Expand Down Expand Up @@ -470,6 +478,31 @@ func (b *Remote) token() (string, error) {
return "", nil
}

// retryLogHook is invoked each time a request is retried allowing the
// backend to log any connection issues to prevent data loss.
func (b *Remote) retryLogHook(attemptNum int, resp *http.Response) {
if b.CLI != nil {
// Ignore the first retry to make sure any delayed output will
// be written to the console before we start logging retries.
//
// The retry logic in the TFE client will retry both rate limited
// requests and server errors, but in the remote backend we only
// care about server errors so we ignore rate limit (429) errors.
if attemptNum == 0 || resp.StatusCode == 429 {
// Reset the last retry time.
b.lastRetry = time.Now()
return
}

if attemptNum == 1 {
b.CLI.Output(b.Colorize().Color(strings.TrimSpace(initialRetryError)))
} else {
b.CLI.Output(b.Colorize().Color(strings.TrimSpace(
fmt.Sprintf(repeatedRetryError, time.Since(b.lastRetry).Round(time.Second)))))
}
}
}

// Workspaces implements backend.Enhanced.
func (b *Remote) Workspaces() ([]string, error) {
if b.prefix == "" {
Expand Down Expand Up @@ -858,6 +891,17 @@ func checkConstraintsWarning(err error) tfdiags.Diagnostic {
)
}

// The newline in this error is to make it look good in the CLI!
const initialRetryError = `
[reset][yellow]There was an error connecting to the remote backend. Please do not exit
Terraform to prevent data loss! Trying to restore the connection...
[reset]
`

const repeatedRetryError = `
[reset][yellow]Still trying to restore the connection... (%s elapsed)[reset]
`

const operationCanceled = `
[reset][red]The remote operation was successfully cancelled.[reset]
`
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ require (
github.com/hashicorp/go-retryablehttp v0.5.1
github.com/hashicorp/go-rootcerts v1.0.0
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 // indirect
github.com/hashicorp/go-tfe v0.3.10
github.com/hashicorp/go-tfe v0.3.11
github.com/hashicorp/go-uuid v1.0.0
github.com/hashicorp/go-version v1.1.0
github.com/hashicorp/golang-lru v0.5.0 // indirect
Expand Down
6 changes: 2 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,8 @@ github.com/hashicorp/go-slug v0.2.0 h1:MVdZAkTmDsUi1AT+3NQDsn8n3ssnVSIHwiM6RcUHv
github.com/hashicorp/go-slug v0.2.0/go.mod h1:+zDycQOzGqOqMW7Kn2fp9vz/NtqpMLQlgb9JUF+0km4=
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-tfe v0.3.10 h1:6uPnPHNPxXDe3k/Vt6fovygYTaWJ8f/7zdHc++f7NJU=
github.com/hashicorp/go-tfe v0.3.10/go.mod h1:LHLchj07PCYgQqcyE5Sz+g4zrMNW+nALKbiSNTZedEs=
github.com/hashicorp/go-tfe v0.3.11 h1:PHw0f1XeriVkqBikhXgZm/t65GR/fEH1iUl/d/9qKbU=
github.com/hashicorp/go-tfe v0.3.11/go.mod h1:LHLchj07PCYgQqcyE5Sz+g4zrMNW+nALKbiSNTZedEs=
github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
Expand Down Expand Up @@ -245,8 +245,6 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3v
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.0.1-0.20181016162627-9eb73efc1fcc h1:9GUJohDyEsZO3cDfQuSxTf38xvk+gRWe+fDv3L9oWHA=
github.com/lib/pq v1.0.1-0.20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lusis/go-artifactory v0.0.0-20160115162124-7e4ce345df82 h1:wnfcqULT+N2seWf6y4yHzmi7GD2kNx4Ute0qArktD48=
github.com/lusis/go-artifactory v0.0.0-20160115162124-7e4ce345df82/go.mod h1:y54tfGmO3NKssKveTEFFzH8C/akrSOy/iW9qEAUDV84=
github.com/marstr/guid v1.1.0 h1:/M4H/1G4avsieL6BbUwCOBzulmoeKVP5ux/3mQNnbyI=
Expand Down
85 changes: 60 additions & 25 deletions vendor/github.com/hashicorp/go-tfe/tfe.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion vendor/modules.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ github.com/hashicorp/go-rootcerts
github.com/hashicorp/go-safetemp
# github.com/hashicorp/go-slug v0.2.0
github.com/hashicorp/go-slug
# github.com/hashicorp/go-tfe v0.3.10
# github.com/hashicorp/go-tfe v0.3.11
github.com/hashicorp/go-tfe
# github.com/hashicorp/go-uuid v1.0.0
github.com/hashicorp/go-uuid
Expand Down