Use exponential backoff for DNS updates

kubernetes · Mar 9, 2021 · d468bc2 · d468bc2
1 parent 6ce35f9
commit d468bc2
Showing 1 changed file with 13 additions and 5 deletions.
diff --git a/dns-controller/pkg/dns/dnscontroller.go b/dns-controller/pkg/dns/dnscontroller.go
@@ -36,6 +36,7 @@ import (
 var zoneListCacheValidity = time.Minute * 15
 
 const DefaultTTL = time.Minute
+const MaxBackoff = time.Minute * 3
 
 // DNSController applies the desired DNS state to the DNS backend
 type DNSController struct {
@@ -56,6 +57,8 @@ type DNSController struct {
 	// changeCount is a change-counter, which helps us avoid computation when nothing has changed
 	changeCount uint64
 
+	// failCount is a fail-counter for exponential backoff
+	failCount uint64
 	// update loop frequency (seconds)
 	updateInterval time.Duration
 }
@@ -120,10 +123,15 @@ func (c *DNSController) runWatcher(stopCh <-chan struct{}) {
 		}
 
 		if err != nil {
-			klog.Warningf("Unexpected error in DNS controller, will retry: %v", err)
-			time.Sleep(2 * c.updateInterval)
+			fails := atomic.AddUint64(&c.failCount, 1)
+			backoff := 1 << fails * c.updateInterval
+			if backoff > MaxBackoff {
+				backoff = MaxBackoff
+			}
+			klog.Warningf("Unexpected error in DNS controller, will retry in %s: %v", backoff, err)
+			time.Sleep(backoff)
 		} else {
-			// Simple debouncing; DNS servers are typically pretty slow anyway
+			atomic.StoreUint64(&c.failCount, 0)
 			time.Sleep(c.updateInterval)
 		}
 	}
@@ -308,7 +316,7 @@ func (c *DNSController) runOnce() error {
 			continue
 		}
 
-		klog.V(2).Infof("applying DNS changeset for zone %s", key)
+		klog.V(2).Infof("Applying DNS changeset for zone %s", key)
 		if err := changeset.Apply(ctx); err != nil {
 			klog.Warningf("error applying DNS changeset for zone %s: %v", key, err)
 			errors = append(errors, fmt.Errorf("error applying DNS changeset for zone %s: %v", key, err))
@@ -351,7 +359,7 @@ func (c *DNSController) RemoveRecordsImmediate(records []Record) error {
 	}
 
 	for key, changeset := range op.changesets {
-		klog.V(2).Infof("applying DNS changeset for zone %s", key)
+		klog.V(2).Infof("Applying DNS changeset for zone %s", key)
 		if err := changeset.Apply(ctx); err != nil {
 			klog.Warningf("error applying DNS changeset for zone %s: %v", key, err)
 			errors = append(errors, fmt.Errorf("error applying DNS changeset for zone %s: %v", key, err))