Track the last time we got a heartbeat response.

go/vt/vttablet/tabletserver/repltracker/reader.go

@@ -61,6 +61,7 @@ type heartbeatReader struct {
 
 	lagMu          sync.Mutex
 	lastKnownLag   time.Duration
+	lastKnownTime  time.Time
 	lastKnownError error
 }
 
@@ -106,6 +107,7 @@ func (r *heartbeatReader) Open() {
 	r.pool.Open(r.env.Config().DB.AppWithDB(), r.env.Config().DB.DbaWithDB(), r.env.Config().DB.AppDebugWithDB())
 	r.ticks.Start(func() { r.readHeartbeat() })
 	r.isOpen = true
+	r.lastKnownTime = r.now()
 }
 
 // Close cancels the watchHeartbeat periodic ticker and closes the db pool.
@@ -131,6 +133,12 @@ func (r *heartbeatReader) Close() {
 func (r *heartbeatReader) Status() (time.Duration, error) {
 	r.lagMu.Lock()
 	defer r.lagMu.Unlock()
+
+	// Return an error if we didn't receive a heartbeat for more than two seconds
+	if r.now().Sub(r.lastKnownTime) > 2*r.interval {
+		return 0, fmt.Errorf("no heartbeat received in over 2x the heartbeat interval")
+	}
+
 	if r.lastKnownError != nil {
 		return 0, r.lastKnownError
 	}
@@ -163,6 +171,7 @@ func (r *heartbeatReader) readHeartbeat() {
 	reads.Add(1)
 
 	r.lagMu.Lock()
+	r.lastKnownTime = r.now()
 	r.lastKnownLag = lag
 	r.lastKnownError = nil
 	r.lagMu.Unlock()