Improve logging and flip if/else for readability

Signed-off-by: Matt Lord <[email protected]>

go/vt/discovery/healthcheck.go

@@ -385,7 +385,7 @@ func (hc *HealthCheckImpl) deleteTablet(tablet *topodata.Tablet) {
 	// delete from authoritative map
 	th, ok := hc.healthByAlias[tabletAlias]
 	if !ok {
-		log.Infof("We have no health data for tablet: %v, it might have been deleted already", tabletAlias)
+		log.Infof("We have no health data for tablet: %v, it might have been deleted already", tablet)
 		return
 	}
 	// Calling this will end the context associated with th.checkConn,
@@ -421,7 +421,7 @@ func (hc *HealthCheckImpl) updateHealth(th *TabletHealth, prevTarget *query.Targ
 	// tablet record that was deleted
 	_, ok := hc.healthByAlias[tabletAlias]
 	if !ok {
-		log.Infof("Tablet %s has been deleted, skipping health update", tabletAlias)
+		log.Infof("Tablet %v has been deleted, skipping health update", th.Tablet)
 		return
 	}
 

go/vt/discovery/tablet_health_check.go

@@ -289,10 +289,13 @@ func (thc *tabletHealthCheck) checkConn(hc *HealthCheckImpl) {
 
 		if err != nil {
 			hcErrorCounters.Add([]string{thc.Target.Keyspace, thc.Target.Shard, topoproto.TabletTypeLString(thc.Target.TabletType)}, 1)
-			// We have reason to suspect the tablet healthcheck record is corrupted or invalid so let's remove the tablet's record
-			// from the healthcheck cache and the corrected tablet record will be fetched from the topology and added to the
-			// healthcheck cache again via the topology watcher.
+			// This means that another tablet has taken over the host:port that we were connected to.
+			// So let's remove the tablet's data from the healthcheck, and if it is still a part of the
+			// cluster, the new tablet record will be fetched from the topology server and re-added to
+			// the healthcheck cache again via the topology watcher.
+			// WARNING: Under no other circumstances should we be deleting the tablet here.
 			if strings.Contains(err.Error(), "health stats mismatch") {
+				log.Warningf("deleting tablet %v from healthcheck due to health stats mismatch", thc.Tablet)
 				hc.deleteTablet(thc.Tablet)
 				return
 			}
@@ -329,7 +332,7 @@ func (thc *tabletHealthCheck) checkConn(hc *HealthCheckImpl) {
 }
 
 func (thc *tabletHealthCheck) closeConnection(ctx context.Context, err error) {
-	log.Warningf("tablet %v healthcheck stream error: %v", thc.Tablet.Alias, err)
+	log.Warningf("tablet %v healthcheck stream error: %v", thc.Tablet, err)
 	thc.setServingState(false, err.Error())
 	thc.LastError = err
 	_ = thc.Conn.Close(ctx)

go/vt/discovery/topology_watcher.go

@@ -216,19 +216,20 @@ func (tw *TopologyWatcher) loadTablets() {
 
 	for alias, newVal := range newTablets {
 		// trust the alias from topo and add it if it doesn't exist
-		if val, ok := tw.tablets[alias]; !ok {
-			tw.tabletRecorder.AddTablet(newVal.tablet)
-			topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1)
-		} else {
-			// check if the host and port have changed. If yes, replace tablet
+		if val, ok := tw.tablets[alias]; ok {
+			// check if the host and port have changed. If yes, replace tablet.
 			oldKey := TabletToMapKey(val.tablet)
 			newKey := TabletToMapKey(newVal.tablet)
 			if oldKey != newKey {
 				// This is the case where the same tablet alias is now reporting
-				// a different address key.
+				// a different address (host:port) key.
 				tw.tabletRecorder.ReplaceTablet(val.tablet, newVal.tablet)
 				topologyWatcherOperations.Add(topologyWatcherOpReplaceTablet, 1)
 			}
+		} else {
+			// This is a new tablet record, let's add it to the healthcheck
+			tw.tabletRecorder.AddTablet(newVal.tablet)
+			topologyWatcherOperations.Add(topologyWatcherOpAddTablet, 1)
 		}
 	}