tidb_client: improve behavior when no alive tidb instance

pkg/tidb/client.go

@@ -108,10 +108,9 @@ func (c *Client) OpenSQLConn(user string, pass string) (*gorm.DB, error) {
 
 	addr := c.sqlAPIAddress
 	if addr == "" {
-		if overrideEndpoint != "" {
-			addr = overrideEndpoint
-		} else {
-			addr = fmt.Sprintf("127.0.0.1:%d", c.forwarder.sqlPort)
+		var err error
+		if addr, err = c.forwarder.resolveSqlAddr(overrideEndpoint); err != nil {
+			return nil, err
 		}
 	}
 
@@ -133,6 +132,9 @@ func (c *Client) OpenSQLConn(user string, pass string) (*gorm.DB, error) {
 			if strings.HasPrefix(addr, "0.0.0.0:") {
 				log.Warn("TiDB reported its address to be 0.0.0.0. Please specify `-advertise-address` command line parameter when running TiDB")
 			}
+			if c.forwarder.sqlProxy.noAliveRemote.Load() {
+				return nil, ErrNoAliveTiDB.NewWithNoMessage()
+			}
 			return nil, ErrTiDBConnFailed.Wrap(err, "failed to connect to TiDB")
 		} else if mysqlErr, ok := err.(*mysql.MySQLError); ok {
 			if mysqlErr.Number == mysqlerr.ER_ACCESS_DENIED_ERROR {
@@ -155,13 +157,18 @@ func (c *Client) SendGetRequest(path string) ([]byte, error) {
 
 	addr := c.statusAPIAddress
 	if addr == "" {
-		if overrideEndpoint != "" {
-			addr = overrideEndpoint
-		} else {
-			addr = fmt.Sprintf("127.0.0.1:%d", c.forwarder.statusPort)
+		var err error
+		if addr, err = c.forwarder.resolveStatusAddr(overrideEndpoint); err != nil {
+			return nil, err
 		}
 	}
 
 	uri := fmt.Sprintf("%s://%s%s", c.statusAPIHTTPScheme, addr, path)
-	return c.statusAPIHTTPClient.WithTimeout(c.statusAPITimeout).SendRequest(c.lifecycleCtx, uri, http.MethodGet, nil, ErrTiDBClientRequestFailed, "TiDB")
+	result, err := c.statusAPIHTTPClient.
+		WithTimeout(c.statusAPITimeout).
+		SendRequest(c.lifecycleCtx, uri, http.MethodGet, nil, ErrTiDBClientRequestFailed, "TiDB")
+	if err != nil && c.forwarder.statusProxy.noAliveRemote.Load() {
+		return nil, ErrNoAliveTiDB.NewWithNoMessage()
+	}
+	return result, err
 }

pkg/tidb/forwarder.go

@@ -20,11 +20,10 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
-	"github.com/joomcode/errorx"
+	"github.com/pingcap/log"
+	"github.com/pingcap/tidb-dashboard/pkg/utils/topology"
 	"go.etcd.io/etcd/clientv3"
 	"go.uber.org/fx"
-
-	"github.com/pingcap/tidb-dashboard/pkg/utils/topology"
 )
 
 var (
@@ -93,10 +92,8 @@ func (f *Forwarder) pollingForTiDB() {
 			return err
 		}, bo)
 		if err != nil {
-			if errorx.IsOfType(err, ErrNoAliveTiDB) {
-				f.sqlProxy.updateRemotes(nil)
-				f.statusProxy.updateRemotes(nil)
-			}
+			f.sqlProxy.updateRemotes(nil)
+			f.statusProxy.updateRemotes(nil)
 		} else {
 			statusEndpoints := make(map[string]struct{}, len(allTiDB))
 			tidbEndpoints := make(map[string]struct{}, len(allTiDB))
@@ -118,6 +115,28 @@ func (f *Forwarder) pollingForTiDB() {
 	}
 }
 
+func (f *Forwarder) resolveSqlAddr(override string) (string, error) {
+	if override != "" {
+		return override, nil
+	}
+	if f.sqlProxy.noAliveRemote.Load() {
+		log.Warn("Unable to resolve sql connection address since no alive TiDB instance")
+		return "", ErrNoAliveTiDB.NewWithNoMessage()
+	}
+	return fmt.Sprintf("127.0.0.1:%d", f.sqlPort), nil
+}
+
+func (f *Forwarder) resolveStatusAddr(override string) (string, error) {
+	if override != "" {
+		return override, nil
+	}
+	if f.statusProxy.noAliveRemote.Load() {
+		log.Warn("Unable to resolve status connection address since no alive TiDB instance")
+		return "", ErrNoAliveTiDB.NewWithNoMessage()
+	}
+	return fmt.Sprintf("127.0.0.1:%d", f.statusPort), nil
+}
+
 func newForwarder(lc fx.Lifecycle, etcdClient *clientv3.Client) *Forwarder {
 	f := &Forwarder{
 		config: &forwarderConfig{

pkg/tidb/proxy.go

@@ -41,8 +41,9 @@ type proxy struct {
 	checkInterval time.Duration
 	dialTimeout   time.Duration
 
-	remotes sync.Map
-	current string
+	noAliveRemote *atomic.Bool
+	remotes       sync.Map
+	current       string
 }
 
 func newProxy(l net.Listener, endpoints map[string]string, checkInterval time.Duration, timeout time.Duration) *proxy {
@@ -57,6 +58,7 @@ func newProxy(l net.Listener, endpoints map[string]string, checkInterval time.Du
 		remotes:       sync.Map{},
 		dialTimeout:   timeout,
 		checkInterval: checkInterval,
+		noAliveRemote: atomic.NewBool(len(endpoints) == 0),
 	}
 	for key, e := range endpoints {
 		p.remotes.Store(key, &remote{addr: e, inactive: atomic.NewBool(true)})
@@ -74,6 +76,7 @@ func (p *proxy) updateRemotes(remotes map[string]struct{}) {
 			p.remotes.Delete(key)
 			return true
 		})
+		p.noAliveRemote.Store(true)
 		return
 	}
 	// update or create new remote
@@ -95,12 +98,32 @@ func (p *proxy) updateRemotes(remotes map[string]struct{}) {
 		}
 		return true
 	})
+	p.noAliveRemote.Store(false)
 }
 
 func (p *proxy) serve(in net.Conn) {
+	out := p.pickActiveConn()
+	if out == nil {
+		log.Warn("no alive remote, drop incoming conn")
+		in.Close()
+		return
+	}
+	// bidirectional copy
+	go func() {
+		//nolint
+		io.Copy(in, out)
+		in.Close()
+		out.Close()
+	}()
+	//nolint
+	io.Copy(out, in)
+	out.Close()
+	in.Close()
+}
+
+func (p *proxy) pickActiveConn() (out net.Conn) {
 	var (
 		err    error
-		out    net.Conn
 		picked *remote
 	)
 	for {
@@ -116,23 +139,8 @@ func (p *proxy) serve(in net.Conn) {
 		picked.becomeInactive()
 		log.Warn("remote become inactive", zap.String("remote", picked.addr))
 	}
-	if out == nil {
-		log.Warn("no alive remote, drop incoming conn")
-		// Do we need issue a error here?
-		in.Close()
-		return
-	}
-	// bidirectional copy
-	go func() {
-		//nolint
-		io.Copy(in, out)
-		in.Close()
-		out.Close()
-	}()
-	//nolint
-	io.Copy(out, in)
-	out.Close()
-	in.Close()
+	p.noAliveRemote.Store(out == nil)
+	return
 }
 
 // pick returns an active remote if there is any

ui/lib/client/translations/en.yaml

@@ -13,7 +13,7 @@ error:
       export_no_data: No statements can be exported
     other: Other error
   tidb:
-    no_alive_tidb: No live TiDB instance in the cluster
+    no_alive_tidb: No alive TiDB instance
     pd_access_failed: Failed to access PD node
     tidb_conn_failed: Failed to connect to TiDB
     tidb_auth_failed: TiDB authentication failed

ui/lib/client/translations/zh.yaml

@@ -13,7 +13,7 @@ error:
       export_no_data: 没有可导出的语句
     other: 其他错误
   tidb:
-    no_alive_tidb: 集群未启动 TiDB 实例
+    no_alive_tidb: 没有正在运行的 TiDB 实例
     pd_access_failed: 无法访问 PD 节点
     tidb_conn_failed: 无法连接到 TiDB
     tidb_auth_failed: TiDB 登录验证失败