From 2c9d3ab6ef0894eae1d89e1d2e9717354c4f4278 Mon Sep 17 00:00:00 2001 From: Suhaha Date: Wed, 12 May 2021 11:39:09 +0800 Subject: [PATCH] feat: debug api (#898) --- pkg/apiserver/apiserver.go | 2 + pkg/apiserver/debugapi/client.go | 106 +++++++++++ pkg/apiserver/debugapi/endpoint.go | 172 ++++++++++++++++++ pkg/apiserver/debugapi/endpoint_def.go | 155 ++++++++++++++++ pkg/apiserver/debugapi/endpoint_models.go | 18 ++ pkg/apiserver/debugapi/endpoint_test.go | 100 ++++++++++ pkg/apiserver/debugapi/module.go | 21 +++ pkg/apiserver/debugapi/service.go | 116 ++++++++++++ pkg/tidb/client.go | 105 ++++++----- ui/dashboardApp/index.ts | 2 + ui/dashboardApp/layout/main/Sider/index.tsx | 5 +- ui/lib/apps/DebugAPI/apilist/ApiForm.tsx | 135 ++++++++++++++ .../apps/DebugAPI/apilist/ApiFormWidgets.tsx | 50 +++++ .../apps/DebugAPI/apilist/ApiList.module.less | 14 ++ ui/lib/apps/DebugAPI/apilist/ApiList.tsx | 153 ++++++++++++++++ ui/lib/apps/DebugAPI/apilist/index.ts | 3 + ui/lib/apps/DebugAPI/index.meta.ts | 9 + ui/lib/apps/DebugAPI/index.tsx | 12 ++ ui/lib/apps/DebugAPI/translations/en.yaml | 23 +++ ui/lib/apps/DebugAPI/translations/zh.yaml | 23 +++ 20 files changed, 1178 insertions(+), 46 deletions(-) create mode 100644 pkg/apiserver/debugapi/client.go create mode 100644 pkg/apiserver/debugapi/endpoint.go create mode 100644 pkg/apiserver/debugapi/endpoint_def.go create mode 100644 pkg/apiserver/debugapi/endpoint_models.go create mode 100644 pkg/apiserver/debugapi/endpoint_test.go create mode 100644 pkg/apiserver/debugapi/module.go create mode 100644 pkg/apiserver/debugapi/service.go create mode 100644 ui/lib/apps/DebugAPI/apilist/ApiForm.tsx create mode 100644 ui/lib/apps/DebugAPI/apilist/ApiFormWidgets.tsx create mode 100644 ui/lib/apps/DebugAPI/apilist/ApiList.module.less create mode 100644 ui/lib/apps/DebugAPI/apilist/ApiList.tsx create mode 100644 ui/lib/apps/DebugAPI/apilist/index.ts create mode 100644 ui/lib/apps/DebugAPI/index.meta.ts create mode 100644 ui/lib/apps/DebugAPI/index.tsx create mode 100644 ui/lib/apps/DebugAPI/translations/en.yaml create mode 100644 ui/lib/apps/DebugAPI/translations/zh.yaml diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 5a535dae05..7be5dc2a23 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/tidb-dashboard/pkg/apiserver/clusterinfo" "github.com/pingcap/tidb-dashboard/pkg/apiserver/configuration" + "github.com/pingcap/tidb-dashboard/pkg/apiserver/debugapi" "github.com/pingcap/tidb-dashboard/pkg/apiserver/diagnose" "github.com/pingcap/tidb-dashboard/pkg/apiserver/info" "github.com/pingcap/tidb-dashboard/pkg/apiserver/logsearch" @@ -133,6 +134,7 @@ func (s *Service) Start(ctx context.Context) error { profiling.Module, statement.Module, slowquery.Module, + debugapi.Module, fx.Populate(&s.apiHandlerEngine), fx.Invoke( user.RegisterRouter, diff --git a/pkg/apiserver/debugapi/client.go b/pkg/apiserver/debugapi/client.go new file mode 100644 index 0000000000..fb1f0f8158 --- /dev/null +++ b/pkg/apiserver/debugapi/client.go @@ -0,0 +1,106 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debugapi + +import ( + "fmt" + + "go.uber.org/fx" + + "github.com/pingcap/tidb-dashboard/pkg/apiserver/model" + "github.com/pingcap/tidb-dashboard/pkg/pd" + "github.com/pingcap/tidb-dashboard/pkg/tidb" + "github.com/pingcap/tidb-dashboard/pkg/tiflash" + "github.com/pingcap/tidb-dashboard/pkg/tikv" +) + +type Client interface { + Send(request *Request) ([]byte, error) + Get(request *Request) ([]byte, error) +} + +type ClientMap map[model.NodeKind]Client + +func newClientMap(tidbImpl tidbImplement, tikvImpl tikvImplement, tiflashImpl tiflashImplement, pdImpl pdImplement) *ClientMap { + clientMap := ClientMap{ + model.NodeKindTiDB: &tidbImpl, + model.NodeKindTiKV: &tikvImpl, + model.NodeKindTiFlash: &tiflashImpl, + model.NodeKindPD: &pdImpl, + } + return &clientMap +} + +func defaultSendRequest(client Client, req *Request) ([]byte, error) { + switch req.Method { + case EndpointMethodGet: + return client.Get(req) + default: + return nil, fmt.Errorf("invalid request method `%s`, host: %s, path: %s", req.Method, req.Host, req.Path) + } +} + +type tidbImplement struct { + fx.In + Client *tidb.Client +} + +func (impl *tidbImplement) Get(req *Request) ([]byte, error) { + return impl.Client.WithEnforcedStatusAPIAddress(req.Host, req.Port).SendGetRequest(req.Path) +} + +func (impl *tidbImplement) Send(req *Request) ([]byte, error) { + return defaultSendRequest(impl, req) +} + +// TODO: tikv/tiflash/pd forwarder impl + +type tikvImplement struct { + fx.In + Client *tikv.Client +} + +func (impl *tikvImplement) Get(req *Request) ([]byte, error) { + return impl.Client.SendGetRequest(req.Host, req.Port, req.Path) +} + +func (impl *tikvImplement) Send(req *Request) ([]byte, error) { + return defaultSendRequest(impl, req) +} + +type tiflashImplement struct { + fx.In + Client *tiflash.Client +} + +func (impl *tiflashImplement) Get(req *Request) ([]byte, error) { + return impl.Client.SendGetRequest(req.Host, req.Port, req.Path) +} + +func (impl *tiflashImplement) Send(req *Request) ([]byte, error) { + return defaultSendRequest(impl, req) +} + +type pdImplement struct { + fx.In + Client *pd.Client +} + +func (impl *pdImplement) Get(req *Request) ([]byte, error) { + return impl.Client.SendGetRequest(req.Path) +} + +func (impl *pdImplement) Send(req *Request) ([]byte, error) { + return defaultSendRequest(impl, req) +} diff --git a/pkg/apiserver/debugapi/endpoint.go b/pkg/apiserver/debugapi/endpoint.go new file mode 100644 index 0000000000..5d530f28a5 --- /dev/null +++ b/pkg/apiserver/debugapi/endpoint.go @@ -0,0 +1,172 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debugapi + +import ( + "net/http" + "net/url" + "regexp" + + "github.com/pingcap/tidb-dashboard/pkg/apiserver/model" +) + +var ( + ErrMissingRequiredParam = ErrNS.NewType("missing_require_parameter") + ErrInvalidParam = ErrNS.NewType("invalid_parameter") +) + +type EndpointAPIModel struct { + ID string `json:"id"` + Component model.NodeKind `json:"component"` + Path string `json:"path"` + Method EndpointMethod `json:"method"` + PathParams []EndpointAPIParam `json:"path_params"` // e.g. /stats/dump/{db}/{table} -> db, table + QueryParams []EndpointAPIParam `json:"query_params"` // e.g. /debug/pprof?seconds=1 -> seconds +} + +type EndpointMethod string + +const ( + EndpointMethodGet EndpointMethod = http.MethodGet +) + +type Request struct { + Method EndpointMethod + Host string + Port int + Path string + Query string +} + +func (e *EndpointAPIModel) NewRequest(host string, port int, value map[string]string) (*Request, error) { + req := &Request{ + Method: e.Method, + Host: host, + Port: port, + } + + pathValues, err := transformValues(e.PathParams, value) + if err != nil { + return nil, err + } + path, err := e.PopulatePath(pathValues) + if err != nil { + return nil, err + } + req.Path = path + + queryValues, err := transformValues(e.QueryParams, value) + if err != nil { + return nil, err + } + query, err := e.EncodeQuery(queryValues) + if err != nil { + return nil, err + } + req.Query = query + + return req, nil +} + +var paramRegexp *regexp.Regexp = regexp.MustCompile(`\{(\w+)\}`) + +func (e *EndpointAPIModel) PopulatePath(valMap map[string]string) (string, error) { + var returnErr error + replacedPath := e.Path + replacedPath = paramRegexp.ReplaceAllStringFunc(replacedPath, func(s string) string { + if returnErr != nil { + return s + } + + key := paramRegexp.ReplaceAllString(s, "${1}") + val, ok := valMap[key] + // means the param can be found in the endpoint path, but not in the param value map + if !ok { + returnErr = ErrMissingRequiredParam.New("missing required path param, path: %s, param: %s", e.Path, key) + return s + } + + return val + }) + return replacedPath, returnErr +} + +func (e *EndpointAPIModel) EncodeQuery(valMap map[string]string) (string, error) { + query := url.Values{} + for _, q := range e.QueryParams { + // cuz paramValues was generated by e.Query, paramValues can always find param by q.Name + val, ok := valMap[q.Name] + if q.Required && !ok { + return "", ErrMissingRequiredParam.New("missing required query param: %s", q.Name) + } + query.Add(q.Name, val) + } + return query.Encode(), nil +} + +func transformValues(params []EndpointAPIParam, values map[string]string) (map[string]string, error) { + pvMap := map[string]string{} + for _, p := range params { + v, ok := values[p.Name] + if !ok { + continue + } + tVal, err := p.Transform(v) + if err != nil { + return nil, ErrInvalidParam.WrapWithNoMessage(err) + } + pvMap[p.Name] = tVal + } + return pvMap, nil +} + +type EndpointAPIParam struct { + Name string `json:"name"` + Required bool `json:"required"` + // represents what param is + Model EndpointAPIParamModel `json:"model"` + PreModelTransformer ModelTransformer `json:"-"` + PostModelTransformer ModelTransformer `json:"-"` +} + +// Transform incoming param's value by transformer at endpoint / model definition +func (p *EndpointAPIParam) Transform(value string) (string, error) { + transfomers := []ModelTransformer{ + p.PreModelTransformer, + p.Model.Transformer, + p.PostModelTransformer, + } + + for _, t := range transfomers { + if t == nil { + continue + } + v, err := t(value) + if err != nil { + return "", err + } + value = v + } + + return value, nil +} + +// Transformer can transform the incoming param's value in special scenarios +// Also, now are used as validation function +type ModelTransformer func(value string) (string, error) + +type EndpointAPIParamModel struct { + Type string `json:"type"` + Transformer ModelTransformer `json:"-"` +} diff --git a/pkg/apiserver/debugapi/endpoint_def.go b/pkg/apiserver/debugapi/endpoint_def.go new file mode 100644 index 0000000000..9d30b96746 --- /dev/null +++ b/pkg/apiserver/debugapi/endpoint_def.go @@ -0,0 +1,155 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debugapi + +import ( + "github.com/pingcap/tidb-dashboard/pkg/apiserver/model" +) + +var tidbStatsDump EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_stats_dump", + Component: model.NodeKindTiDB, + Path: "/stats/dump/{db}/{table}", + Method: EndpointMethodGet, + PathParams: []EndpointAPIParam{ + { + Name: "db", + Model: EndpointAPIParamModelText, + }, + { + Name: "table", + Model: EndpointAPIParamModelText, + }, + }, +} + +var tidbStatsDumpWithTimestamp EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_stats_dump_timestamp", + Component: model.NodeKindTiDB, + Path: "/stats/dump/{db}/{table}/{yyyyMMddHHmmss}", + Method: EndpointMethodGet, + PathParams: []EndpointAPIParam{ + { + Name: "db", + Model: EndpointAPIParamModelText, + }, + { + Name: "table", + Model: EndpointAPIParamModelText, + }, + { + Name: "yyyyMMddHHmmss", + Model: EndpointAPIParamModelText, + }, + }, +} + +var tidbConfig EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_config", + Component: model.NodeKindTiDB, + Path: "/settings", + Method: EndpointMethodGet, +} + +var tidbSchema EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_schema", + Component: model.NodeKindTiDB, + Path: "/schema", + Method: EndpointMethodGet, +} + +var tidbSchemaWithDB EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_schema_db", + Component: model.NodeKindTiDB, + Path: "/schema/{db}", + Method: EndpointMethodGet, + PathParams: []EndpointAPIParam{ + { + Name: "db", + Model: EndpointAPIParamModelText, + }, + }, +} + +var tidbSchemaWithDBTable EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_schema_db_table", + Component: model.NodeKindTiDB, + Path: "/schema/{db}/{table}", + Method: EndpointMethodGet, + PathParams: []EndpointAPIParam{ + { + Name: "db", + Model: EndpointAPIParamModelText, + }, + { + Name: "table", + Model: EndpointAPIParamModelText, + }, + }, +} + +var tidbSchemaWithTableID EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_schema_tableid", + Component: model.NodeKindTiDB, + Path: "/db-table/{tableID}", + Method: EndpointMethodGet, + PathParams: []EndpointAPIParam{ + { + Name: "tableID", + Model: EndpointAPIParamModelText, + }, + }, +} + +var tidbDDLHistory EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_ddl_history", + Component: model.NodeKindTiDB, + Path: "/ddl/history", + Method: EndpointMethodGet, +} + +var tidbInfo EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_info", + Component: model.NodeKindTiDB, + Path: "/info", + Method: EndpointMethodGet, +} + +var tidbInfoAll EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_info_all", + Component: model.NodeKindTiDB, + Path: "/info/all", + Method: EndpointMethodGet, +} + +var tidbRegionsMeta EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_regions_meta", + Component: model.NodeKindTiDB, + Path: "/regions/meta", + Method: EndpointMethodGet, +} + +var endpointAPIList []EndpointAPIModel = []EndpointAPIModel{ + tidbStatsDump, + tidbStatsDumpWithTimestamp, + tidbConfig, + tidbSchema, + tidbSchemaWithDB, + tidbSchemaWithDBTable, + tidbSchemaWithTableID, + tidbDDLHistory, + tidbInfo, + tidbInfoAll, + tidbRegionsMeta, +} diff --git a/pkg/apiserver/debugapi/endpoint_models.go b/pkg/apiserver/debugapi/endpoint_models.go new file mode 100644 index 0000000000..9e2109c614 --- /dev/null +++ b/pkg/apiserver/debugapi/endpoint_models.go @@ -0,0 +1,18 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debugapi + +var EndpointAPIParamModelText EndpointAPIParamModel = EndpointAPIParamModel{ + Type: "text", +} diff --git a/pkg/apiserver/debugapi/endpoint_test.go b/pkg/apiserver/debugapi/endpoint_test.go new file mode 100644 index 0000000000..0a06235833 --- /dev/null +++ b/pkg/apiserver/debugapi/endpoint_test.go @@ -0,0 +1,100 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debugapi + +import ( + "fmt" + "net/http" + "testing" + + "github.com/joomcode/errorx" + . "github.com/pingcap/check" + + "github.com/pingcap/tidb-dashboard/pkg/apiserver/model" +) + +func TestT(t *testing.T) { + CustomVerboseFlag = true + TestingT(t) +} + +var _ = Suite(&testSchemaSuite{}) + +type testSchemaSuite struct{} + +var testTiDBStatsDump EndpointAPIModel = EndpointAPIModel{ + ID: "tidb_stats_dump", + Component: model.NodeKindTiDB, + Path: "/stats/dump/{db}/{table}", + Method: http.MethodGet, + PathParams: []EndpointAPIParam{ + { + Name: "db", + Model: EndpointAPIParamModelText, + }, + { + Name: "table", + Model: EndpointAPIParamModelText, + }, + }, + QueryParams: []EndpointAPIParam{ + { + Name: "debug", + Required: true, + Model: EndpointAPIParamModelText, + }, + }, +} + +func (t *testSchemaSuite) Test_new_request_success(c *C) { + db := "test" + table := "users" + debugFlag := "1" + + vals := map[string]string{ + "db": db, + "table": table, + "debug": debugFlag, + } + req, err := testTiDBStatsDump.NewRequest("127.0.0.1", 10080, vals) + + if err == nil { + c.Assert(req.Path, Equals, fmt.Sprintf("/stats/dump/%s/%s", db, table)) + c.Assert(req.Query, Equals, fmt.Sprintf("debug=%s", debugFlag)) + } else { + c.ExpectFailure(err.Error()) + } +} + +func (t *testSchemaSuite) Test_new_request_err_missing_required_path_params(c *C) { + vals := map[string]string{ + "db": "test", + "debug": "1", + } + _, err := testTiDBStatsDump.NewRequest("127.0.0.1", 10080, vals) + + c.Log(err) + c.Assert(errorx.IsOfType(err, ErrMissingRequiredParam), Equals, true) +} + +func (t *testSchemaSuite) Test_new_request_err_missing_required_query_params(c *C) { + vals := map[string]string{ + "db": "test", + "table": "users", + } + _, err := testTiDBStatsDump.NewRequest("127.0.0.1", 10080, vals) + + c.Log(err) + c.Assert(errorx.IsOfType(err, ErrMissingRequiredParam), Equals, true) +} diff --git a/pkg/apiserver/debugapi/module.go b/pkg/apiserver/debugapi/module.go new file mode 100644 index 0000000000..7fc0bc6000 --- /dev/null +++ b/pkg/apiserver/debugapi/module.go @@ -0,0 +1,21 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debugapi + +import "go.uber.org/fx" + +var Module = fx.Options( + fx.Provide(newService, newClientMap), + fx.Invoke(registerRouter), +) diff --git a/pkg/apiserver/debugapi/service.go b/pkg/apiserver/debugapi/service.go new file mode 100644 index 0000000000..c5839ed3bb --- /dev/null +++ b/pkg/apiserver/debugapi/service.go @@ -0,0 +1,116 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debugapi + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/joomcode/errorx" + + "github.com/pingcap/tidb-dashboard/pkg/apiserver/user" + "github.com/pingcap/tidb-dashboard/pkg/apiserver/utils" +) + +var ( + ErrNS = errorx.NewNamespace("error.api.debugapi") + ErrComponentClient = ErrNS.NewType("invalid_component_client") + ErrEndpointConfig = ErrNS.NewType("invalid_endpoint_config") + ErrInvalidStatusPort = ErrNS.NewType("invalid_status_port") +) + +func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) { + endpoint := r.Group("/debugapi") + endpoint.Use(auth.MWAuthRequired()) + + endpoint.POST("/request_endpoint", s.RequestEndpoint) + endpoint.GET("/endpoints", s.GetEndpointList) +} + +type endpoint struct { + EndpointAPIModel + Client Client +} + +type Service struct { + endpointMap map[string]endpoint +} + +func newService(clientMap *ClientMap) (*Service, error) { + s := &Service{endpointMap: map[string]endpoint{}} + + for _, e := range endpointAPIList { + client, ok := (*clientMap)[e.Component] + if !ok { + panic(ErrComponentClient.New("%s type client not found, id: %s", e.Component, e.ID)) + } + s.endpointMap[e.ID] = endpoint{EndpointAPIModel: e, Client: client} + } + + return s, nil +} + +type EndpointRequest struct { + ID string `json:"id"` + Host string `json:"host"` + Port int `json:"port"` + Params map[string]string `json:"params"` +} + +// @Summary RequestEndpoint send request to tidb/tikv/tiflash/pd http api +// @Security JwtAuth +// @Param req body EndpointRequest true "endpoint request param" +// @Success 200 {object} string +// @Failure 400 {object} utils.APIError "Bad request" +// @Failure 401 {object} utils.APIError "Unauthorized failure" +// @Failure 500 {object} utils.APIError +// @Router /debugapi/request_endpoint [post] +func (s *Service) RequestEndpoint(c *gin.Context) { + var req EndpointRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.MakeInvalidRequestErrorFromError(c, err) + return + } + + endpoint, ok := s.endpointMap[req.ID] + if !ok { + _ = c.Error(ErrEndpointConfig.New("invalid endpoint id: %s", req.ID)) + return + } + + endpointReq, err := endpoint.NewRequest(req.Host, req.Port, req.Params) + if err != nil { + _ = c.Error(err) + return + } + + resp, err := endpoint.Client.Send(endpointReq) + if err != nil { + _ = c.Error(err) + return + } + + c.JSON(200, string(resp)) +} + +// @Summary Get all endpoint configs +// @Security JwtAuth +// @Success 200 {array} EndpointAPIModel +// @Failure 400 {object} utils.APIError "Bad request" +// @Failure 401 {object} utils.APIError "Unauthorized failure" +// @Failure 500 {object} utils.APIError +// @Router /debugapi/endpoints [get] +func (s *Service) GetEndpointList(c *gin.Context) { + c.JSON(http.StatusOK, endpointAPIList) +} diff --git a/pkg/tidb/client.go b/pkg/tidb/client.go index 535e30dc79..0cc731d926 100644 --- a/pkg/tidb/client.go +++ b/pkg/tidb/client.go @@ -43,14 +43,15 @@ const ( ) type Client struct { - lifecycleCtx context.Context - forwarder *Forwarder - statusAPIHTTPScheme string - statusAPIAddress string // Empty means to use address provided by forwarder - statusAPIHTTPClient *httpc.Client - statusAPITimeout time.Duration - sqlAPITLSKey string // Non empty means use this key as MySQL TLS config - sqlAPIAddress string // Empty means to use address provided by forwarder + lifecycleCtx context.Context + forwarder *Forwarder + statusAPIHTTPScheme string + statusAPIAddress string // Empty means to use address provided by forwarder + enforceStatusAPIAddresss bool // enforced status api address and ignore env override config + statusAPIHTTPClient *httpc.Client + statusAPITimeout time.Duration + sqlAPITLSKey string // Non empty means use this key as MySQL TLS config + sqlAPIAddress string // Empty means to use address provided by forwarder } func NewTiDBClient(lc fx.Lifecycle, config *config.Config, etcdClient *clientv3.Client, httpClient *httpc.Client) *Client { @@ -61,14 +62,15 @@ func NewTiDBClient(lc fx.Lifecycle, config *config.Config, etcdClient *clientv3. } client := &Client{ - lifecycleCtx: nil, - forwarder: newForwarder(lc, etcdClient), - statusAPIHTTPScheme: config.GetClusterHTTPScheme(), - statusAPIAddress: "", - statusAPIHTTPClient: httpClient, - statusAPITimeout: defaultTiDBStatusAPITimeout, - sqlAPITLSKey: sqlAPITLSKey, - sqlAPIAddress: "", + lifecycleCtx: nil, + forwarder: newForwarder(lc, etcdClient), + statusAPIHTTPScheme: config.GetClusterHTTPScheme(), + statusAPIAddress: "", + enforceStatusAPIAddresss: false, + statusAPIHTTPClient: httpClient, + statusAPITimeout: defaultTiDBStatusAPITimeout, + sqlAPITLSKey: sqlAPITLSKey, + sqlAPIAddress: "", } lc.Append(fx.Hook{ @@ -81,40 +83,47 @@ func NewTiDBClient(lc fx.Lifecycle, config *config.Config, etcdClient *clientv3. return client } -func (c *Client) WithStatusAPITimeout(timeout time.Duration) *Client { - c2 := *c - c2.statusAPITimeout = timeout - return &c2 +func (c Client) WithStatusAPITimeout(timeout time.Duration) *Client { + c.statusAPITimeout = timeout + return &c } -func (c *Client) WithStatusAPIAddress(host string, statusPort int) *Client { - c2 := *c - c2.statusAPIAddress = fmt.Sprintf("%s:%d", host, statusPort) - return &c2 +func (c Client) WithStatusAPIAddress(host string, statusPort int) *Client { + c.statusAPIAddress = fmt.Sprintf("%s:%d", host, statusPort) + return &c } -func (c *Client) WithSQLAPIAddress(host string, sqlPort int) *Client { - c2 := *c - c2.sqlAPIAddress = fmt.Sprintf("%s:%d", host, sqlPort) - return &c2 +func (c Client) WithEnforcedStatusAPIAddress(host string, statusPort int) *Client { + c.enforceStatusAPIAddresss = true + c.statusAPIAddress = fmt.Sprintf("%s:%d", host, statusPort) + return &c +} + +func (c Client) WithSQLAPIAddress(host string, sqlPort int) *Client { + c.sqlAPIAddress = fmt.Sprintf("%s:%d", host, sqlPort) + return &c } func (c *Client) OpenSQLConn(user string, pass string) (*gorm.DB, error) { + var err error + overrideEndpoint := os.Getenv(tidbOverrideSQLEndpointEnvVar) + // the `tidbOverrideSQLEndpointEnvVar` and the `Client.sqlAPIAddress` have the same override priority, if both exist, an error is returned if overrideEndpoint != "" && c.sqlAPIAddress != "" { log.Warn(fmt.Sprintf("Reject to establish a target specified TiDB SQL connection since `%s` is set", tidbOverrideSQLEndpointEnvVar)) return nil, ErrTiDBConnFailed.New("TiDB Dashboard is configured to only connect to specified TiDB host") } - addr := c.sqlAPIAddress + var addr string + switch { + case overrideEndpoint != "": + addr = overrideEndpoint + default: + addr = c.sqlAPIAddress + } if addr == "" { - if overrideEndpoint != "" { - addr = overrideEndpoint - } else { - var err error - if addr, err = c.forwarder.getEndpointAddr(c.forwarder.sqlPort); err != nil { - return nil, err - } + if addr, err = c.forwarder.getEndpointAddr(c.forwarder.sqlPort); err != nil { + return nil, err } } @@ -153,21 +162,27 @@ func (c *Client) OpenSQLConn(user string, pass string) (*gorm.DB, error) { } func (c *Client) SendGetRequest(path string) ([]byte, error) { + var err error + overrideEndpoint := os.Getenv(tidbOverrideStatusEndpointEnvVar) - if overrideEndpoint != "" && c.statusAPIAddress != "" { + // the `tidbOverrideStatusEndpointEnvVar` and the `Client.statusAPIAddress` have the same override priority, if both exist and have not enforced `Client.statusAPIAddress` then an error is returned + if overrideEndpoint != "" && c.statusAPIAddress != "" && !c.enforceStatusAPIAddresss { log.Warn(fmt.Sprintf("Reject to establish a target specified TiDB status connection since `%s` is set", tidbOverrideStatusEndpointEnvVar)) return nil, ErrTiDBConnFailed.New("TiDB Dashboard is configured to only connect to specified TiDB host") } - addr := c.statusAPIAddress + var addr string + switch { + case c.enforceStatusAPIAddresss: + addr = c.sqlAPIAddress + case overrideEndpoint != "": + addr = overrideEndpoint + default: + addr = c.sqlAPIAddress + } if addr == "" { - if overrideEndpoint != "" { - addr = overrideEndpoint - } else { - var err error - if addr, err = c.forwarder.getEndpointAddr(c.forwarder.statusPort); err != nil { - return nil, err - } + if addr, err = c.forwarder.getEndpointAddr(c.forwarder.statusPort); err != nil { + return nil, err } } diff --git a/ui/dashboardApp/index.ts b/ui/dashboardApp/index.ts index f2f6e2c2df..e5aee25435 100644 --- a/ui/dashboardApp/index.ts +++ b/ui/dashboardApp/index.ts @@ -32,6 +32,7 @@ import AppSearchLogs from '@lib/apps/SearchLogs/index.meta' import AppInstanceProfiling from '@lib/apps/InstanceProfiling/index.meta' import AppQueryEditor from '@lib/apps/QueryEditor/index.meta' import AppConfiguration from '@lib/apps/Configuration/index.meta' +import AppDebugAPI from '@lib/apps/DebugAPI/index.meta' // import __APP_NAME__ from '@lib/apps/__APP_NAME__/index.meta' // NOTE: Don't remove above comment line, it is a placeholder for code generator @@ -118,6 +119,7 @@ async function main() { .register(AppInstanceProfiling) .register(AppQueryEditor) .register(AppConfiguration) + .register(AppDebugAPI) // .register(__APP_NAME__) // NOTE: Don't remove above comment line, it is a placeholder for code generator diff --git a/ui/dashboardApp/layout/main/Sider/index.tsx b/ui/dashboardApp/layout/main/Sider/index.tsx index 883ffed41b..19c078bcd3 100644 --- a/ui/dashboardApp/layout/main/Sider/index.tsx +++ b/ui/dashboardApp/layout/main/Sider/index.tsx @@ -57,7 +57,10 @@ function Sider({ client.getInstance().infoGet(reqConfig) ) - const debugSubMenuItems = [useAppMenuItem(registry, 'instance_profiling')] + const debugSubMenuItems = [ + useAppMenuItem(registry, 'instance_profiling'), + useAppMenuItem(registry, 'debug_api'), + ] const debugSubMenu = ( `${component}_host`, [component]) + const params = [...(path_params ?? []), ...(query_params ?? [])] + const [loading, setLoading] = useState(false) + + const download = useCallback( + async (values: any) => { + let data: string + let headers: any + try { + setLoading(true) + const { [endpointHostParamKey]: host, ...params } = values + const [hostname, port] = host.split(':') + const resp = await client.getInstance().debugapiRequestEndpointPost({ + id, + host: hostname, + port: Number(port), + params, + }) + data = resp.data + headers = resp.headers + } catch (e) { + setLoading(false) + console.error(e) + return + } + + const blob = new Blob([data], { type: headers['content-type'] }) + const link = document.createElement('a') + const fileName = `${id}_${Date.now()}.json` + + // quick view backdoor + blob.text().then((t) => console.log(t)) + + link.href = window.URL.createObjectURL(blob) + link.download = fileName + link.click() + window.URL.revokeObjectURL(link.href) + + setLoading(false) + }, + [id, endpointHostParamKey] + ) + + const endpointParam = useMemo( + () => ({ + name: endpointHostParamKey, + model: { + type: 'host', + }, + }), + [endpointHostParamKey] + ) + const EndpointHost = () => ( + + ) + + return ( +
+ + {params.map((param) => ( + + ))} + + + + + ) +} + +function ApiFormItem({ param, endpoint, topology }: ApiFormWidgetConfig) { + return ( + + {(widgetsMap[param.model?.type!] || widgetsMap.text)({ + param, + endpoint, + topology, + })} + + ) +} diff --git a/ui/lib/apps/DebugAPI/apilist/ApiFormWidgets.tsx b/ui/lib/apps/DebugAPI/apilist/ApiFormWidgets.tsx new file mode 100644 index 0000000000..552be25adc --- /dev/null +++ b/ui/lib/apps/DebugAPI/apilist/ApiFormWidgets.tsx @@ -0,0 +1,50 @@ +import React from 'react' +import { Input, Select } from 'antd' +import { useTranslation } from 'react-i18next' +import { DebugapiEndpointAPIModel, DebugapiEndpointAPIParam } from '@lib/client' +import type { Topology } from './ApiForm' + +export interface ApiFormWidget { + (config: ApiFormWidgetConfig): JSX.Element +} + +export interface ApiFormWidgetConfig { + param: DebugapiEndpointAPIParam + endpoint: DebugapiEndpointAPIModel + topology: Topology +} + +const TextWidget: ApiFormWidget = ({ param }) => { + const { t } = useTranslation() + return ( + + ) +} + +const HostSelectWidget: ApiFormWidget = ({ endpoint, topology }) => { + const { t } = useTranslation() + const componentEndpoints = topology[endpoint.component!] + + return ( + + ) +} + +export const widgetsMap: { [type: string]: ApiFormWidget } = { + text: TextWidget, + host: HostSelectWidget, +} diff --git a/ui/lib/apps/DebugAPI/apilist/ApiList.module.less b/ui/lib/apps/DebugAPI/apilist/ApiList.module.less new file mode 100644 index 0000000000..4497d3e5a3 --- /dev/null +++ b/ui/lib/apps/DebugAPI/apilist/ApiList.module.less @@ -0,0 +1,14 @@ +.collapse_panel:not(:last-child) { + border-bottom: 1px solid #eee; +} + +.header { + user-select: none; + p { + margin-bottom: 0; + } +} + +.schema { + color: #999; +} diff --git a/ui/lib/apps/DebugAPI/apilist/ApiList.tsx b/ui/lib/apps/DebugAPI/apilist/ApiList.tsx new file mode 100644 index 0000000000..a54bc72ee8 --- /dev/null +++ b/ui/lib/apps/DebugAPI/apilist/ApiList.tsx @@ -0,0 +1,153 @@ +import React, { useEffect, useMemo, useState } from 'react' +import { Collapse, Space, Input, Empty } from 'antd' +import { useTranslation } from 'react-i18next' +import { TFunction } from 'i18next' +import { SearchOutlined } from '@ant-design/icons' +import { debounce } from 'lodash' + +import { AnimatedSkeleton, Card } from '@lib/components' +import { useClientRequest } from '@lib/utils/useClientRequest' +import client, { DebugapiEndpointAPIModel } from '@lib/client' + +import style from './ApiList.module.less' +import ApiForm, { Topology } from './ApiForm' + +const useFilterEndpoints = (endpoints?: DebugapiEndpointAPIModel[]) => { + const [keywords, setKeywords] = useState('') + const nonNullEndpoints = useMemo(() => endpoints || [], [endpoints]) + const [filteredEndpoints, setFilteredEndpoints] = useState< + DebugapiEndpointAPIModel[] + >(nonNullEndpoints) + + useEffect(() => { + const k = keywords.trim() + if (!!k) { + setFilteredEndpoints( + nonNullEndpoints.filter((e) => e.id?.includes(k) || e.path?.includes(k)) + ) + } else { + setFilteredEndpoints(nonNullEndpoints) + } + }, [nonNullEndpoints, keywords]) + + return { + endpoints: filteredEndpoints, + filterBy: debounce(setKeywords, 300), + } +} + +export default function Page() { + const { t } = useTranslation() + const { + data: endpointData, + isLoading: isEndpointLoading, + } = useClientRequest((reqConfig) => + client.getInstance().debugapiEndpointsGet(reqConfig) + ) + const { endpoints, filterBy } = useFilterEndpoints(endpointData) + + const groups = useMemo( + () => + endpoints.reduce((prev, endpoint) => { + const groupName = endpoint.component! + if (!prev[groupName]) { + prev[groupName] = [] + } + prev[groupName].push(endpoint) + return prev + }, {} as { [group: string]: DebugapiEndpointAPIModel[] }), + [endpoints] + ) + const sortingOfGroups = useMemo(() => ['tidb', 'tikv', 'tiflash', 'pd'], []) + // TODO: other components topology + const { + data: tidbTopology = [], + isLoading: isTopologyLoading, + } = useClientRequest((reqConfig) => + client.getInstance().getTiDBTopology(reqConfig) + ) + const topology: Topology = { + tidb: tidbTopology!, + } + + const EndpointGroups = () => + endpoints.length ? ( + <> + {sortingOfGroups + .filter((sortKey) => groups[sortKey]) + .map((sortKey) => { + const g = groups[sortKey] + return ( + + + {g.map((endpoint) => ( + } + key={endpoint.id!} + > + + + ))} + + + ) + })} + + ) : ( + + ) + + return ( + + + + } + onChange={(e) => filterBy(e.target.value)} + /> + + + + + ) +} + +function CustomHeader({ + endpoint, + t, +}: { + endpoint: DebugapiEndpointAPIModel + t: TFunction +}) { + return ( +
+ + +

+ {t(`debug_api.${endpoint.component}.endpoint_ids.${endpoint.id}`)} +

+ + + +
+ ) +} + +// e.g. http://{tidb_ip}/stats/dump/{db}/{table}?queryName={queryName} +function Schema({ endpoint }: { endpoint: DebugapiEndpointAPIModel }) { + const query = + endpoint.query_params?.reduce((prev, { name }, i) => { + if (i === 0) { + prev += '?' + } + prev += `${name}={${name}}` + return prev + }, '') || '' + return ( +

+ {`http://{${endpoint.component}_host}${endpoint.path}${query}`} +

+ ) +} diff --git a/ui/lib/apps/DebugAPI/apilist/index.ts b/ui/lib/apps/DebugAPI/apilist/index.ts new file mode 100644 index 0000000000..3b4ebcfbb5 --- /dev/null +++ b/ui/lib/apps/DebugAPI/apilist/index.ts @@ -0,0 +1,3 @@ +import ApiList from './ApiList' + +export { ApiList } diff --git a/ui/lib/apps/DebugAPI/index.meta.ts b/ui/lib/apps/DebugAPI/index.meta.ts new file mode 100644 index 0000000000..4e0bd93ba5 --- /dev/null +++ b/ui/lib/apps/DebugAPI/index.meta.ts @@ -0,0 +1,9 @@ +import { ApiOutlined } from '@ant-design/icons' + +export default { + id: 'debug_api', + routerPrefix: '/debug_api', + icon: ApiOutlined, + translations: require.context('./translations/', false, /\.yaml$/), + reactRoot: () => import(/* webpackChunkName: "app_debug_api" */ '.'), +} diff --git a/ui/lib/apps/DebugAPI/index.tsx b/ui/lib/apps/DebugAPI/index.tsx new file mode 100644 index 0000000000..76f5f5a634 --- /dev/null +++ b/ui/lib/apps/DebugAPI/index.tsx @@ -0,0 +1,12 @@ +import React from 'react' + +import { Root } from '@lib/components' +import { ApiList } from './apilist' + +const App = () => ( + + + +) + +export default App diff --git a/ui/lib/apps/DebugAPI/translations/en.yaml b/ui/lib/apps/DebugAPI/translations/en.yaml new file mode 100644 index 0000000000..a4d7d4aee7 --- /dev/null +++ b/ui/lib/apps/DebugAPI/translations/en.yaml @@ -0,0 +1,23 @@ +debug_api: + nav_title: Debugging API + keyword_search: Keyword Search + endpoints_not_found: Endpoints not found + form: + download: Download + widgets: + text: Please enter the {{param}} + host_select_placeholder: Please select the {{endpointType}} host + tidb: + name: TiDB + endpoint_ids: + tidb_stats_dump: Stats Dump + tidb_stats_dump_timestamp: Stats Dump With Timestamp + tidb_config: Current TiDB Config + tidb_schema: TiDB Schema + tidb_schema_db: DB Schema + tidb_schema_db_table: Table Schema + tidb_schema_tableid: Table Schema By Table ID + tidb_ddl_history: DDL History + tidb_info: Info + tidb_info_all: All Info + tidb_regions_meta: Regions Meta diff --git a/ui/lib/apps/DebugAPI/translations/zh.yaml b/ui/lib/apps/DebugAPI/translations/zh.yaml new file mode 100644 index 0000000000..8188625e1a --- /dev/null +++ b/ui/lib/apps/DebugAPI/translations/zh.yaml @@ -0,0 +1,23 @@ +debug_api: + nav_title: 信息查询接口 + keyword_search: 关键词搜索 + endpoints_not_found: 找不到对应接口 + form: + download: 下载 + widgets: + text: 请输入 {{param}} + host_select_placeholder: 请选择对应的 {{endpointType}} host + tidb: + name: TiDB + endpoint_ids: + tidb_stats_dump: Stats Dump + tidb_stats_dump_timestamp: Stats Dump With Timestamp + tidb_config: Current TiDB Config + tidb_schema: TiDB Schema + tidb_schema_db: DB Schema + tidb_schema_db_table: Table Schema + tidb_schema_tableid: Table Schema By Table ID + tidb_ddl_history: DDL History + tidb_info: Info + tidb_info_all: All Info + tidb_regions_meta: Regions Meta