server: consider node unhealthy when liveness status is not LIVE

Issue #22424 brought to light the fact that a load balancer using
the /health endpoint would not consider draining nodes unhealthy, and
would route requests to it. This is the first step to ensuring that load
balancers do not route requests to draining nodes.

Release note (bug fix): Health endpoint now returns a node as unhealthy
when draining or decommissioning.

pkg/server/admin.go

@@ -1034,12 +1034,12 @@ func (s *adminServer) Cluster(
 func (s *adminServer) Health(
 	ctx context.Context, req *serverpb.HealthRequest,
 ) (*serverpb.HealthResponse, error) {
-	isLive, err := s.server.nodeLiveness.IsLive(s.server.NodeID())
+	isHealthy, err := s.server.nodeLiveness.IsHealthy(s.server.NodeID())
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, err.Error())
 	}
-	if !isLive {
-		return nil, status.Errorf(codes.Unavailable, "node is not live")
+	if !isHealthy {
+		return nil, status.Errorf(codes.Unavailable, "node is not healthy")
 	}
 	return &serverpb.HealthResponse{}, nil
 }

pkg/storage/node_liveness.go

@@ -352,6 +352,22 @@ func (nl *NodeLiveness) IsLive(nodeID roachpb.NodeID) (bool, error) {
 	return liveness.IsLive(nl.clock.Now(), nl.clock.MaxOffset()), nil
 }
 
+// IsHealthy returns whether or not the specified node is considered healthy
+// based on its current liveness status. It is an error if the specified node is
+// not in the local liveness table.
+func (nl *NodeLiveness) IsHealthy(nodeID roachpb.NodeID) (bool, error) {
+	liveness, err := nl.GetLiveness(nodeID)
+	if err != nil {
+		return false, err
+	}
+	ls := liveness.LivenessStatus(
+		nl.clock.Now().GoTime(),
+		nl.GetLivenessThreshold(),
+		nl.clock.MaxOffset(),
+	)
+	return ls == NodeLivenessStatus_LIVE, nil
+}
+
 // StartHeartbeat starts a periodic heartbeat to refresh this node's
 // last heartbeat in the node liveness table. The optionally provided
 // HeartbeatCallback will be invoked whenever this node updates its own liveness.