Skip to content

Commit

Permalink
Cleanup AWS EC2 eventual consistency warnings
Browse files Browse the repository at this point in the history
  • Loading branch information
Ciprian Hacman committed Jul 27, 2020
1 parent d0a2bae commit 0876165
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 13 deletions.
22 changes: 21 additions & 1 deletion upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"sort"
"strconv"
"strings"
"time"

"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
Expand Down Expand Up @@ -298,7 +299,26 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos
}

// @step: attempt to create the autoscaling group for us
if _, err := t.Cloud.Autoscaling().CreateAutoScalingGroup(request); err != nil {
err := func() error {
var err error
for attempt := 0; attempt < 10; attempt++ {
if _, err = t.Cloud.Autoscaling().CreateAutoScalingGroup(request); err == nil {
return nil
}

code := awsup.AWSErrorCode(err)
message := awsup.AWSErrorMessage(err)
if code == "ValidationError" && strings.Contains(message, "Invalid IAM Instance Profile name") {
klog.V(4).Infof("got an error indicating that the IAM instance profile is not ready: %q", message)
time.Sleep(5 * time.Second)
continue
}
klog.V(4).Infof("ErrorCode=%q, Message=%q", code, message)
}

return err
}()
if err != nil {
return fmt.Errorf("error creating AutoscalingGroup: %v", err)
}

Expand Down
20 changes: 8 additions & 12 deletions upup/pkg/fi/cloudup/awstasks/launchtemplate_target_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,24 +143,20 @@ func (t *LaunchTemplate) RenderAWS(c *awsup.AWSAPITarget, a, ep, changes *Launch
}
// @step: attempt to create the launch template
err = func() error {
var err error
for attempt := 0; attempt < 10; attempt++ {
if _, err = c.Cloud.EC2().CreateLaunchTemplate(input); err == nil {
return nil
}

if awsup.AWSErrorCode(err) == "ValidationError" {
message := awsup.AWSErrorMessage(err)
if strings.Contains(message, "not authorized") || strings.Contains(message, "Invalid IamInstance") {
if attempt > 10 {
return fmt.Errorf("IAM instance profile not yet created/propagated (original error: %v)", message)
}
klog.V(4).Infof("got an error indicating that the IAM instance profile %q is not ready: %q", fi.StringValue(ep.IAMInstanceProfile.Name), message)

time.Sleep(5 * time.Second)
continue
}
klog.V(4).Infof("ErrorCode=%q, Message=%q", awsup.AWSErrorCode(err), awsup.AWSErrorMessage(err))
code := awsup.AWSErrorCode(err)
message := awsup.AWSErrorMessage(err)
if code == "ValidationError" && (strings.Contains(message, "not authorized") || strings.Contains(message, "Invalid IamInstance")) {
klog.V(4).Infof("got an error indicating that the IAM instance profile %q is not ready: %q", fi.StringValue(ep.IAMInstanceProfile.Name), message)
time.Sleep(5 * time.Second)
continue
}
klog.V(4).Infof("ErrorCode=%q, Message=%q", awsup.AWSErrorCode(err), awsup.AWSErrorMessage(err))
}

return err
Expand Down

0 comments on commit 0876165

Please sign in to comment.