lease: randomize expiry on initial refresh call

Randomize the very first expiry on lease recovery
to prevent recovered leases from expiring all at
the same time.

Address etcd-io#8096.

Signed-off-by: Gyu-Ho Lee <[email protected]>

lease/lessor.go

@@ -18,6 +18,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"math"
+	"math/rand"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -147,6 +148,9 @@ type lessor struct {
 	stopC chan struct{}
 	// doneC is a channel whose closure indicates that the lessor is stopped.
 	doneC chan struct{}
+
+	// 'true' when lessor just started promoting after recovery
+	promoting bool
 }
 
 func NewLessor(b backend.Backend, minLeaseTTL int64) Lessor {
@@ -160,9 +164,10 @@ func newLessor(b backend.Backend, minLeaseTTL int64) *lessor {
 		b:           b,
 		minLeaseTTL: minLeaseTTL,
 		// expiredC is a small buffered chan to avoid unnecessary blocking.
-		expiredC: make(chan []*Lease, 16),
-		stopC:    make(chan struct{}),
-		doneC:    make(chan struct{}),
+		expiredC:  make(chan []*Lease, 16),
+		stopC:     make(chan struct{}),
+		doneC:     make(chan struct{}),
+		promoting: true,
 	}
 	l.initAndRecover()
 
@@ -323,11 +328,29 @@ func (le *lessor) Promote(extend time.Duration) {
 	defer le.mu.Unlock()
 
 	le.demotec = make(chan struct{})
+	promoting := le.promoting
 
 	// refresh the expiries of all leases.
 	for _, l := range le.leaseMap {
-		l.refresh(extend)
+		ext := extend
+		if promoting {
+			// randomize expiry with 士10%, otherwise leases of same TTL
+			// will expire all at the same time,
+			ext += computeRandomDelta(l.ttl)
+		}
+		l.refresh(ext)
+	}
+	le.promoting = false
+}
+
+func computeRandomDelta(seconds int64) time.Duration {
+	var delta int64
+	if seconds > 10 {
+		delta = int64(float64(seconds) * 0.1 * rand.Float64())
+	} else {
+		delta = rand.Int63n(10)
 	}
+	return time.Duration(delta) * time.Second
 }
 
 func (le *lessor) Demote() {

lease/lessor_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/coreos/etcd/mvcc/backend"
+	"github.com/coreos/etcd/pkg/monotime"
 )
 
 const (
@@ -210,6 +211,52 @@ func TestLessorRenew(t *testing.T) {
 	}
 }
 
+// TestLessorRenewRandomize ensures Lessor renews with randomized expiry.
+func TestLessorRenewRandomize(t *testing.T) {
+	dir, be := NewTestBackend(t)
+	defer os.RemoveAll(dir)
+
+	le := newLessor(be, minLeaseTTL)
+	for i := LeaseID(1); i <= 10; i++ {
+		if _, err := le.Grant(i, 3600); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// simulate stop and recovery
+	le.Stop()
+	be.Close()
+	bcfg := backend.DefaultBackendConfig()
+	bcfg.Path = filepath.Join(dir, "be")
+	be = backend.New(bcfg)
+	defer be.Close()
+	le = newLessor(be, minLeaseTTL)
+
+	now := monotime.Now()
+
+	// first extend after recovery should randomize expiries
+	le.Promote(0)
+
+	for _, l := range le.leaseMap {
+		leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9))
+		pc := (float64(leftSeconds-3600) / float64(3600)) * 100
+		if pc > 10.0 || pc < -10.0 || pc == 0 { // should be within 士10%
+			t.Fatalf("expected randomized expiry, got %d seconds (ttl: 3600)", leftSeconds)
+		}
+	}
+
+	// second extend should not be randomized
+	le.Promote(0)
+
+	for _, l := range le.leaseMap {
+		leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9))
+		pc := (float64(leftSeconds-3600) / float64(3600)) * 100
+		if pc > .5 || pc < -.5 { // should be close to original ttl 3600
+			t.Fatalf("expected 3600-sec left, got %d", leftSeconds)
+		}
+	}
+}
+
 func TestLessorDetach(t *testing.T) {
 	dir, be := NewTestBackend(t)
 	defer os.RemoveAll(dir)