feat(stmt): support config maximum number of stmt kept in memory

pkg/apiserver/slowquery/model.go

@@ -14,8 +14,6 @@
 package slowquery
 
 import (
-	"strings"
-
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
 )
 
@@ -111,7 +109,7 @@ func getFieldsAndTags() (slowQueryFields []Field) {
 
 	for _, f := range fields {
 		sqf := Field{
-			ColumnName: getGormColumnName(f.Tags["gorm"]),
+			ColumnName: utils.GetGormColumnName(f.Tags["gorm"]),
 			JSONName:   f.Tags["json"],
 			Projection: f.Tags["proj"],
 		}
@@ -121,9 +119,3 @@ func getFieldsAndTags() (slowQueryFields []Field) {
 
 	return
 }
-
-func getGormColumnName(gormStr string) string {
-	// TODO: use go-gorm/gorm/schema ParseTagSetting. Prerequisite: Upgrade to the latest version
-	columnName := strings.Split(gormStr, ":")[1]
-	return columnName
-}

pkg/apiserver/statement/config.go

@@ -0,0 +1,84 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package statement
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/thoas/go-funk"
+
+	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
+)
+
+// incoming configuration field should have the gorm tag `column` used to specify global variables
+// sql will be built like this, gorm:"column:some_global_var" -> @@GLOBAL.some_global_var as some_global_var
+func buildConfigQuerySQL(config interface{}) string {
+	var configType reflect.Type
+	if reflect.ValueOf(config).Kind() == reflect.Ptr {
+		configType = reflect.TypeOf(config).Elem()
+	} else {
+		configType = reflect.TypeOf(config)
+	}
+
+	stmts := []string{}
+	fNum := configType.NumField()
+	for i := 0; i < fNum; i++ {
+		f := configType.Field(i)
+		gormTag, ok := f.Tag.Lookup("gorm")
+		if !ok {
+			continue
+		}
+		column := utils.GetGormColumnName(gormTag)
+		stmts = append(stmts, fmt.Sprintf("@@GLOBAL.%s AS %s", column, column))
+	}
+
+	// skip `SQL string formatting (gosec)` lint
+	return "SELECT " + strings.Join(stmts, ", ") // nolints
+}
+
+// sql will be built like this, gorm:"column:some_global_var" -> @@GLOBAL.some_global_var = some_global_var_value
+func buildConfigUpdateSQL(config interface{}, extract ...string) string {
+	var configType reflect.Type
+	var configValue reflect.Value
+	if reflect.ValueOf(config).Kind() == reflect.Ptr {
+		configType = reflect.TypeOf(config).Elem()
+		configValue = reflect.ValueOf(config).Elem()
+	} else {
+		configType = reflect.TypeOf(config)
+		configValue = reflect.ValueOf(config)
+	}
+
+	stmts := []string{}
+	fNum := configType.NumField()
+	for i := 0; i < fNum; i++ {
+		f := configType.Field(i)
+		// extract fields on demand
+		if len(extract) != 0 && !funk.ContainsString(extract, f.Name) {
+			continue
+		}
+		gormTag, ok := f.Tag.Lookup("gorm")
+		if !ok {
+			continue
+		}
+
+		val := configValue.Field(i)
+		column := utils.GetGormColumnName(gormTag)
+		stmts = append(stmts, fmt.Sprintf("@@GLOBAL.%s = %v", column, val))
+	}
+
+	// skip `SQL string formatting (gosec)` lint
+	return "SET " + strings.Join(stmts, ", ") // nolints
+}

pkg/apiserver/statement/config_test.go

@@ -0,0 +1,74 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package statement
+
+import (
+	"testing"
+
+	. "github.com/pingcap/check"
+)
+
+func TestT(t *testing.T) {
+	CustomVerboseFlag = true
+	TestingT(t)
+}
+
+var _ = Suite(&testConfigSuite{})
+
+type testConfigSuite struct{}
+
+type testConfig struct {
+	Enable          bool `json:"enable" gorm:"column:tidb_enable_stmt_summary"`
+	RefreshInterval int  `json:"refresh_interval" gorm:"column:tidb_stmt_summary_refresh_interval"`
+}
+
+func (t *testConfigSuite) Test_buildConfigQuerySQL_struct_success(c *C) {
+	testConfigStmt := "SELECT @@GLOBAL.tidb_enable_stmt_summary as tidb_enable_stmt_summary,@@GLOBAL.tidb_stmt_summary_refresh_interval as tidb_stmt_summary_refresh_interval"
+	c.Assert(buildConfigQuerySQL(testConfig{}), Equals, testConfigStmt)
+}
+
+func (t *testConfigSuite) Test_buildConfigQuerySQL_ptr_success(c *C) {
+	testConfigStmt := "SELECT @@GLOBAL.tidb_enable_stmt_summary as tidb_enable_stmt_summary,@@GLOBAL.tidb_stmt_summary_refresh_interval as tidb_stmt_summary_refresh_interval"
+	c.Assert(buildConfigQuerySQL(&testConfig{}), Equals, testConfigStmt)
+}
+
+type testConfig2 struct {
+	Enable          bool `json:"enable" gorm:"column:tidb_enable_stmt_summary"`
+	RefreshInterval int  `json:"refresh_interval"`
+}
+
+func (t *testConfigSuite) Test_buildConfigQuerySQL_without_gorm_tag(c *C) {
+	testConfigStmt := "SELECT @@GLOBAL.tidb_enable_stmt_summary as tidb_enable_stmt_summary"
+	c.Assert(buildConfigQuerySQL(&testConfig2{}), Equals, testConfigStmt)
+}
+
+func (t *testConfigSuite) Test_buildConfigUpdateSQL_struct_success(c *C) {
+	testConfigStmt := "SET @@GLOBAL.tidb_enable_stmt_summary = true,@@GLOBAL.tidb_stmt_summary_refresh_interval = 1800"
+	c.Assert(buildConfigUpdateSQL(testConfig{Enable: true, RefreshInterval: 1800}), Equals, testConfigStmt)
+}
+
+func (t *testConfigSuite) Test_buildConfigUpdateSQL_ptr_success(c *C) {
+	testConfigStmt := "SET @@GLOBAL.tidb_enable_stmt_summary = true,@@GLOBAL.tidb_stmt_summary_refresh_interval = 1800"
+	c.Assert(buildConfigUpdateSQL(&testConfig{Enable: true, RefreshInterval: 1800}), Equals, testConfigStmt)
+}
+
+func (t *testConfigSuite) Test_buildConfigUpdateSQL_without_gorm_tag(c *C) {
+	testConfigStmt := "SET @@GLOBAL.tidb_enable_stmt_summary = true"
+	c.Assert(buildConfigUpdateSQL(&testConfig2{Enable: true, RefreshInterval: 1800}), Equals, testConfigStmt)
+}
+
+func (t *testConfigSuite) Test_buildConfigUpdateSQL_extract_fields(c *C) {
+	testConfigStmt := "SET @@GLOBAL.tidb_enable_stmt_summary = true"
+	c.Assert(buildConfigUpdateSQL(&testConfig{Enable: true, RefreshInterval: 1800}, "Enable"), Equals, testConfigStmt)
+}

pkg/apiserver/statement/models.go

@@ -21,12 +21,6 @@ import (
 	"github.com/pingcap/tidb-dashboard/pkg/apiserver/utils"
 )
 
-type Config struct {
-	Enable          bool `json:"enable"`
-	RefreshInterval int  `json:"refresh_interval"`
-	HistorySize     int  `json:"history_size"`
-}
-
 // TimeRange represents a range of time
 type TimeRange struct {
 	BeginTime int64 `json:"begin_time"`

pkg/apiserver/statement/queries.go

@@ -16,89 +16,15 @@ package statement
 import (
 	"fmt"
 	"regexp"
-	"strconv"
 	"strings"
 
 	"github.com/jinzhu/gorm"
 )
 
 const (
-	statementsTable        = "INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY"
-	stmtEnableVar          = "tidb_enable_stmt_summary"
-	stmtRefreshIntervalVar = "tidb_stmt_summary_refresh_interval"
-	stmtHistorySizeVar     = "tidb_stmt_summary_history_size"
+	statementsTable = "INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY"
 )
 
-// How to get sql variables by GORM
-// https://github.com/jinzhu/gorm/issues/2616
-func querySQLIntVariable(db *gorm.DB, name string) (int, error) {
-	var values []string
-	sql := fmt.Sprintf("SELECT @@GLOBAL.%s as value", name) // nolints
-	err := db.Raw(sql).Pluck("value", &values).Error
-	if err != nil {
-		return 0, err
-	}
-	strVal := values[0]
-	if strVal == "" {
-		return -1, nil
-	}
-	intVal, err := strconv.Atoi(strVal)
-	if err != nil {
-		return 0, err
-	}
-	return intVal, nil
-}
-
-func queryStmtConfig(db *gorm.DB) (*Config, error) {
-	config := Config{}
-
-	enable, err := querySQLIntVariable(db, stmtEnableVar)
-	if err != nil {
-		return nil, err
-	}
-	config.Enable = enable != 0
-
-	refreshInterval, err := querySQLIntVariable(db, stmtRefreshIntervalVar)
-	if err != nil {
-		return nil, err
-	}
-	if refreshInterval == -1 {
-		config.RefreshInterval = 1800
-	} else {
-		config.RefreshInterval = refreshInterval
-	}
-
-	historySize, err := querySQLIntVariable(db, stmtHistorySizeVar)
-	if err != nil {
-		return nil, err
-	}
-	if historySize == -1 {
-		config.HistorySize = 24
-	} else {
-		config.HistorySize = historySize
-	}
-
-	return &config, err
-}
-
-func updateStmtConfig(db *gorm.DB, config *Config) (err error) {
-	var sql string
-	sql = fmt.Sprintf("SET GLOBAL %s = ?", stmtEnableVar)
-	err = db.Exec(sql, config.Enable).Error
-
-	if config.Enable {
-		// update other configurations
-		sql = fmt.Sprintf("SET GLOBAL %s = ?", stmtRefreshIntervalVar)
-		err = db.Exec(sql, config.RefreshInterval).Error
-		if err != nil {
-			return
-		}
-		sql = fmt.Sprintf("SET GLOBAL %s = ?", stmtHistorySizeVar)
-		err = db.Exec(sql, config.HistorySize).Error
-	}
-	return
-}
-
 func queryTimeRanges(db *gorm.DB) (result []*TimeRange, err error) {
 	err = db.
 		Select(`

pkg/apiserver/statement/service.go

@@ -73,14 +73,22 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 	}
 }
 
+type EditableConfig struct {
+	Enable          bool `json:"enable" gorm:"column:tidb_enable_stmt_summary"`
+	RefreshInterval int  `json:"refresh_interval" gorm:"column:tidb_stmt_summary_refresh_interval"`
+	HistorySize     int  `json:"history_size" gorm:"column:tidb_stmt_summary_history_size"`
+	MaxSize         int  `json:"max_size" gorm:"column:tidb_stmt_summary_max_stmt_count"`
+}
+
 // @Summary Get statement configurations
-// @Success 200 {object} statement.Config
+// @Success 200 {object} statement.EditableConfig
 // @Router /statements/config [get]
 // @Security JwtAuth
 // @Failure 401 {object} utils.APIError "Unauthorized failure"
 func (s *Service) configHandler(c *gin.Context) {
 	db := utils.GetTiDBConnection(c)
-	cfg, err := queryStmtConfig(db)
+	cfg := &EditableConfig{}
+	err := db.Raw(buildConfigQuerySQL(cfg)).Find(cfg).Error
 	if err != nil {
 		_ = c.Error(err)
 		return
@@ -89,23 +97,30 @@ func (s *Service) configHandler(c *gin.Context) {
 }
 
 // @Summary Update statement configurations
-// @Param request body statement.Config true "Request body"
+// @Param request body statement.EditableConfig true "Request body"
 // @Success 204 {object} string
 // @Router /statements/config [post]
 // @Security JwtAuth
 // @Failure 401 {object} utils.APIError "Unauthorized failure"
 func (s *Service) modifyConfigHandler(c *gin.Context) {
-	var req Config
-	if err := c.ShouldBindJSON(&req); err != nil {
+	var err error
+	var config EditableConfig
+	if err = c.ShouldBindJSON(&config); err != nil {
 		utils.MakeInvalidRequestErrorFromError(c, err)
 		return
 	}
 	db := utils.GetTiDBConnection(c)
-	err := updateStmtConfig(db, &req)
+
+	if !config.Enable {
+		err = db.Exec(buildConfigUpdateSQL(&config, "Enable")).Error
+	} else {
+		err = db.Exec(buildConfigUpdateSQL(&config)).Error
+	}
 	if err != nil {
 		_ = c.Error(err)
 		return
 	}
+
 	c.Status(http.StatusNoContent)
 }
 

pkg/apiserver/utils/gorm.go

@@ -0,0 +1,22 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import "strings"
+
+func GetGormColumnName(gormStr string) string {
+	// TODO: use go-gorm/gorm/schema ParseTagSetting. Prerequisite: Upgrade to the latest version
+	columnName := strings.Split(gormStr, ":")[1]
+	return columnName
+}