diff --git a/CHANGELOG/CHANGELOG-1.5.md b/CHANGELOG/CHANGELOG-1.5.md index 9623c0264..adc175e47 100644 --- a/CHANGELOG/CHANGELOG-1.5.md +++ b/CHANGELOG/CHANGELOG-1.5.md @@ -3,7 +3,7 @@
-## [v1.5.6](https://github.com/aws/aws-k8s-tester/releases/tag/v1.5.6) (2020-12) +## [v1.5.6](https://github.com/aws/aws-k8s-tester/releases/tag/v1.5.6) (2021-01) See [code changes](https://github.com/aws/aws-k8s-tester/compare/v1.5.5...v1.5.6). @@ -12,6 +12,11 @@ See [code changes](https://github.com/aws/aws-k8s-tester/compare/v1.5.5...v1.5.6 - Expose [configuration of sonobuoy worker/systemd-logs container](https://github.com/aws/aws-k8s-tester/pull/190). - Warn [file open errors in configuration validator, rather than error out](https://github.com/aws/aws-k8s-tester/pull/191). +### `eks` + +- Fix [`eks/mng` delete retries](https://github.com/aws/aws-k8s-tester/pull/196). + - See https://github.com/aws/aws-k8s-tester/issues/195. + ### Dependency - Upgrade [`github.com/aws/aws-sdk-go`](https://github.com/aws/aws-sdk-go/releases) from [`v1.35.27`](https://github.com/aws/aws-sdk-go/releases/tag/v1.35.27) to [`v1.35.30`](https://github.com/aws/aws-sdk-go/releases/tag/v1.35.30). diff --git a/eks/mng/mng.go b/eks/mng/mng.go index 9087a9b02..950ec2f01 100644 --- a/eks/mng/mng.go +++ b/eks/mng/mng.go @@ -208,36 +208,24 @@ func (ts *tester) Delete() error { errs = append(errs, err.Error()) } } - err = nil failedMNGs := make(map[string]struct{}) for name := range ts.cfg.EKSConfig.AddOnManagedNodeGroups.MNGs { - var derr error for i := 0; i < 5; i++ { // retry, leakly ENI may take awhile to be deleted - derr = ts.deleteASG(name) - if derr != nil { - failedMNGs[name] = struct{}{} - ts.cfg.Logger.Warn("failed to delete mng; retrying", zap.String("name", name), zap.Error(derr)) - select { - case <-ts.cfg.Stopc: - ts.cfg.Logger.Warn("aborted") - return nil - case <-time.After(time.Minute): - } + derr := ts.deleteASG(name) + if derr == nil { + ts.cfg.Logger.Info("successfully deleted mng", zap.String("name", name)) + delete(failedMNGs, name) + break } - break - } - if derr != nil { - if err == nil { - err = derr - } else { - err = fmt.Errorf("%v; %v", err, derr) + ts.cfg.Logger.Warn("failed to delete mng; retrying", zap.String("name", name), zap.Error(derr)) + failedMNGs[name] = struct{}{} + select { + case <-ts.cfg.Stopc: + ts.cfg.Logger.Warn("aborted") + return nil + case <-time.After(time.Minute): } - continue } - delete(failedMNGs, name) - } - if err != nil { - errs = append(errs, err.Error()) } waitDur := time.Minute @@ -250,21 +238,25 @@ func (ts *tester) Delete() error { time.Sleep(10 * time.Second) } } + err = nil for name := range failedMNGs { + ts.cfg.Logger.Warn("retrying mng delete after failure", zap.String("name", name)) var derr error for i := 0; i < 5; i++ { // retry, leakly ENI may take awhile to be deleted derr = ts.deleteASG(name) - if derr != nil { - ts.cfg.Logger.Warn("failed to retry-delete mng; retrying", zap.String("name", name), zap.Error(derr)) - select { - case <-ts.cfg.Stopc: - ts.cfg.Logger.Warn("aborted") - return nil - case <-time.After(time.Minute): - } + if derr == nil { + ts.cfg.Logger.Info("successfully deleted mng (previously failed for delete)", zap.String("name", name)) + delete(failedMNGs, name) + break + } + ts.cfg.Logger.Warn("failed to retry-delete mng; retrying", zap.String("name", name), zap.Error(derr)) + select { + case <-ts.cfg.Stopc: + ts.cfg.Logger.Warn("aborted") + return nil + case <-time.After(time.Minute): } - break } if derr != nil { if err == nil { @@ -272,9 +264,7 @@ func (ts *tester) Delete() error { } else { err = fmt.Errorf("%v; %v", err, derr) } - continue } - delete(failedMNGs, name) } if err != nil { errs = append(errs, err.Error())