kv: reduce verbose logging on cross-locality metrics

As part of cockroachdb#103983, metrics were added to track when
requests cross region or zone boundaries, however verbose logging was
added to the necessary locality comparisons. Whenever a cluster does not
have locality tiers defined - as is the default on new clusters, as well
as in tests - an error would be generated, and this error would be
logged (with a full stack trace) at the same level as the transport
debugging information, on every request. This change reduces that
verbosity and removes the unnecessary stack trace from being printed on
these locally generated and handled errors.

Epic: none

Release note: None

pkg/kv/kvclient/kvcoord/dist_sender.go

@@ -2545,21 +2545,21 @@ func (ds *DistSender) getLocalityComparison(
 ) roachpb.LocalityComparisonType {
 	gatewayNodeDesc, err := ds.nodeDescs.GetNodeDescriptor(fromNodeID)
 	if err != nil {
-		log.VEventf(ctx, 2, "failed to perform look up for node descriptor %+v", err)
+		log.VEventf(ctx, 3, "failed to perform look up for node descriptor: %v", err)
 		return roachpb.LocalityComparisonType_UNDEFINED
 	}
 	destinationNodeDesc, err := ds.nodeDescs.GetNodeDescriptor(toNodeID)
 	if err != nil {
-		log.VEventf(ctx, 2, "failed to perform look up for node descriptor %+v", err)
+		log.VEventf(ctx, 3, "failed to perform look up for node descriptor: %v", err)
 		return roachpb.LocalityComparisonType_UNDEFINED
 	}
 
 	comparisonResult, regionErr, zoneErr := gatewayNodeDesc.Locality.CompareWithLocality(destinationNodeDesc.Locality)
 	if regionErr != nil {
-		log.VEventf(ctx, 2, "unable to determine if the given nodes are cross region %+v", regionErr)
+		log.VEventf(ctx, 5, "unable to determine if the given nodes are cross region: %v", regionErr)
 	}
 	if zoneErr != nil {
-		log.VEventf(ctx, 2, "unable to determine if the given nodes are cross zone %+v", zoneErr)
+		log.VEventf(ctx, 5, "unable to determine if the given nodes are cross zone: %v", zoneErr)
 	}
 	return comparisonResult
 }

pkg/kv/kvserver/store_snapshot.go

@@ -989,10 +989,10 @@ func (s *Store) getLocalityComparison(
 	secLocality := s.cfg.StorePool.GetNodeLocality(toNodeID)
 	comparisonResult, regionErr, zoneErr := firstLocality.CompareWithLocality(secLocality)
 	if regionErr != nil {
-		log.VEventf(ctx, 2, "unable to determine if the given nodes are cross region %+v", regionErr)
+		log.VEventf(ctx, 5, "unable to determine if the given nodes are cross region: %v", regionErr)
 	}
 	if zoneErr != nil {
-		log.VEventf(ctx, 2, "unable to determine if the given nodes are cross zone %+v", zoneErr)
+		log.VEventf(ctx, 5, "unable to determine if the given nodes are cross zone: %v", zoneErr)
 	}
 	return comparisonResult
 }

pkg/server/node.go

@@ -1353,23 +1353,23 @@ func (n *Node) getLocalityComparison(
 ) roachpb.LocalityComparisonType {
 	gossip := n.storeCfg.Gossip
 	if gossip == nil {
-		log.VEventf(ctx, 2, "gossip is not configured")
+		log.VEventf(ctx, 3, "gossip is not configured")
 		return roachpb.LocalityComparisonType_UNDEFINED
 	}
 
 	gatewayNodeDesc, err := gossip.GetNodeDescriptor(gatewayNodeID)
 	if err != nil {
-		log.VEventf(ctx, 2,
-			"failed to perform look up for node descriptor %v", err)
+		log.VEventf(ctx, 3,
+			"failed to perform look up for node descriptor: %v", err)
 		return roachpb.LocalityComparisonType_UNDEFINED
 	}
 
 	comparisonResult, regionErr, zoneErr := n.Descriptor.Locality.CompareWithLocality(gatewayNodeDesc.Locality)
 	if regionErr != nil {
-		log.VEventf(ctx, 2, "unable to determine if the given nodes are cross region %+v", regionErr)
+		log.VEventf(ctx, 5, "unable to determine if the given nodes are cross region: %v", regionErr)
 	}
 	if zoneErr != nil {
-		log.VEventf(ctx, 2, "unable to determine if the given nodes are cross zone %+v", zoneErr)
+		log.VEventf(ctx, 5, "unable to determine if the given nodes are cross zone: %v", zoneErr)
 	}
 
 	return comparisonResult