diff --git a/dns-controller/pkg/dns/dnscontroller.go b/dns-controller/pkg/dns/dnscontroller.go index f1d44ac06246c..68bc586b6d5b4 100644 --- a/dns-controller/pkg/dns/dnscontroller.go +++ b/dns-controller/pkg/dns/dnscontroller.go @@ -36,6 +36,7 @@ import ( var zoneListCacheValidity = time.Minute * 15 const DefaultTTL = time.Minute +const MaxBackoff = time.Minute * 3 // DNSController applies the desired DNS state to the DNS backend type DNSController struct { @@ -56,6 +57,8 @@ type DNSController struct { // changeCount is a change-counter, which helps us avoid computation when nothing has changed changeCount uint64 + // failCount is a fail-counter for exponential backoff + failCount uint64 // update loop frequency (seconds) updateInterval time.Duration } @@ -120,10 +123,15 @@ func (c *DNSController) runWatcher(stopCh <-chan struct{}) { } if err != nil { - klog.Warningf("Unexpected error in DNS controller, will retry: %v", err) - time.Sleep(2 * c.updateInterval) + fails := atomic.AddUint64(&c.failCount, 1) + backoff := 1 << fails * c.updateInterval + if backoff > MaxBackoff { + backoff = MaxBackoff + } + klog.Warningf("Unexpected error in DNS controller, will retry in %s: %v", backoff, err) + time.Sleep(backoff) } else { - // Simple debouncing; DNS servers are typically pretty slow anyway + atomic.StoreUint64(&c.failCount, 0) time.Sleep(c.updateInterval) } } @@ -308,7 +316,7 @@ func (c *DNSController) runOnce() error { continue } - klog.V(2).Infof("applying DNS changeset for zone %s", key) + klog.V(2).Infof("Applying DNS changeset for zone %s", key) if err := changeset.Apply(ctx); err != nil { klog.Warningf("error applying DNS changeset for zone %s: %v", key, err) errors = append(errors, fmt.Errorf("error applying DNS changeset for zone %s: %v", key, err)) @@ -351,7 +359,7 @@ func (c *DNSController) RemoveRecordsImmediate(records []Record) error { } for key, changeset := range op.changesets { - klog.V(2).Infof("applying DNS changeset for zone %s", key) + klog.V(2).Infof("Applying DNS changeset for zone %s", key) if err := changeset.Apply(ctx); err != nil { klog.Warningf("error applying DNS changeset for zone %s: %v", key, err) errors = append(errors, fmt.Errorf("error applying DNS changeset for zone %s: %v", key, err))