Skip to content

Commit

Permalink
slack-15.0: backport vitessio#15897 (#389)
Browse files Browse the repository at this point in the history 
* Add sql text counts stats to `vtcombo`,`vtgate`+`vttablet`

Signed-off-by: Tim Vaillancourt <[email protected]>

* initialize sqlTextCounts where missed

Signed-off-by: Harshit Gangal <[email protected]>

* missing fix

Signed-off-by: Tim Vaillancourt <[email protected]>

* fix accidental line delete

Signed-off-by: Tim Vaillancourt <[email protected]>

---------

Signed-off-by: Tim Vaillancourt <[email protected]>
Signed-off-by: Harshit Gangal <[email protected]>
Co-authored-by: Harshit Gangal <[email protected]>
  • Loading branch information
@timvaillancourt @harshit-gangal
timvaillancourt and harshit-gangal authored Jun 1, 2024
1 parent c801fd2 commit b6812a7
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 19 deletions.
12 changes: 9 additions & 3 deletions go/vt/vtgate/vtgate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,9 +184,10 @@ type VTGate struct {
// stats objects.
// TODO(sougou): This needs to be cleaned up. There
// are global vars that depend on this member var.
timings *stats.MultiTimings
rowsReturned *stats.CountersWithMultiLabels
rowsAffected *stats.CountersWithMultiLabels
timings *stats.MultiTimings
rowsReturned *stats.CountersWithMultiLabels
rowsAffected *stats.CountersWithMultiLabels
sqlTextCounts *stats.CountersWithMultiLabels

// the throttled loggers for all errors, one per API entry
logExecute *logutil.ThrottledLogger
Expand Down Expand Up @@ -304,6 +305,10 @@ func Init(
"VtgateApiRowsAffected",
"Rows affected by a write (DML) operation through the VTgate API",
[]string{"Operation", "Keyspace", "DbType"}),
sqlTextCounts: stats.NewCountersWithMultiLabels(
"VtgateSQLTextCounts",
"Vtgate API query SQL text counts",
[]string{"Operation", "Keyspace", "DbType"}),

logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second),
logStreamExecute: logutil.NewThrottledLogger("StreamExecute", 5*time.Second),
Expand Down Expand Up @@ -433,6 +438,7 @@ func (vtg *VTGate) Execute(ctx context.Context, session *vtgatepb.Session, sql s
if err == nil {
vtg.rowsReturned.Add(statsKey, int64(len(qr.Rows)))
vtg.rowsAffected.Add(statsKey, int64(qr.RowsAffected))
vtg.sqlTextCounts.Add(statsKey, int64(len(sql)))
return session, qr, nil
}

Expand Down
12 changes: 8 additions & 4 deletions go/vt/vttablet/tabletserver/query_engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ type QueryEngine struct {
consolidatorMode sync2.AtomicString

// stats
queryCounts, queryTimes, queryRowCounts, queryErrorCounts, queryRowsAffected, queryRowsReturned *stats.CountersWithMultiLabels
queryCounts, queryTimes, queryRowCounts, queryErrorCounts, queryRowsAffected, queryRowsReturned, querySQLTextCounts *stats.CountersWithMultiLabels

// stats flags
enablePerWorkloadTableMetrics bool
Expand Down Expand Up @@ -261,6 +261,7 @@ func NewQueryEngine(env tabletenv.Env, se *schema.Engine) *QueryEngine {
qe.queryRowCounts = env.Exporter().NewCountersWithMultiLabels("QueryRowCounts", "(DEPRECATED - use QueryRowsAffected and QueryRowsReturned instead) query row counts", labels)
qe.queryRowsAffected = env.Exporter().NewCountersWithMultiLabels("QueryRowsAffected", "query rows affected", labels)
qe.queryRowsReturned = env.Exporter().NewCountersWithMultiLabels("QueryRowsReturned", "query rows returned", labels)
qe.querySQLTextCounts = env.Exporter().NewCountersWithMultiLabels("QuerySQLTextCounts", "query sql text counts", labels)
qe.queryErrorCounts = env.Exporter().NewCountersWithMultiLabels("QueryErrorCounts", "query error counts", labels)

env.Exporter().HandleFunc("/debug/hotrows", qe.txSerializer.ServeHTTP)
Expand Down Expand Up @@ -489,9 +490,9 @@ func (qe *QueryEngine) QueryPlanCacheLen() int {
}

// AddStats adds the given stats for the planName.tableName
func (qe *QueryEngine) AddStats(planType planbuilder.PlanType, tableName, workload string, queryCount int64, duration, mysqlTime time.Duration, rowsAffected, rowsReturned, errorCount int64) {
func (qe *QueryEngine) AddStats(plan *TabletPlan, tableName, workload string, queryCount int64, duration, mysqlTime time.Duration, rowsAffected, rowsReturned, errorCount int64) {
// table names can contain "." characters, replace them!
keys := []string{tableName, planType.String()}
keys := []string{tableName, plan.PlanID.String()}
// Only use the workload as a label if that's enabled in the configuration.
if qe.enablePerWorkloadTableMetrics {
keys = append(keys, workload)
Expand All @@ -500,11 +501,14 @@ func (qe *QueryEngine) AddStats(planType planbuilder.PlanType, tableName, worklo
qe.queryTimes.Add(keys, int64(duration))
qe.queryRowCounts.Add(keys, rowsAffected)
qe.queryErrorCounts.Add(keys, errorCount)
if plan.FullQuery != nil {
qe.querySQLTextCounts.Add(keys, int64(len(plan.FullQuery.Query)))
}

// For certain plan types like select, we only want to add their metrics to rows returned
// But there are special cases like `SELECT ... INTO OUTFILE ''` which return positive rows affected
// So we check if it is positive and add that too.
switch planType {
switch plan.PlanID {
case planbuilder.PlanSelect, planbuilder.PlanSelectStream, planbuilder.PlanSelectImpossible, planbuilder.PlanShow, planbuilder.PlanOtherRead:
qe.queryRowsReturned.Add(keys, rowsReturned)
if rowsAffected > 0 {
Expand Down
42 changes: 32 additions & 10 deletions go/vt/vttablet/tabletserver/query_engine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,9 +573,21 @@ func TestPlanCachePollution(t *testing.T) {
}

func TestAddQueryStats(t *testing.T) {
fakeSelectPlan := &TabletPlan{
Plan: &planbuilder.Plan{
PlanID: planbuilder.PlanSelect,
FullQuery: &sqlparser.ParsedQuery{Query: `select * from something where something=123`}, // 43 length
},
}
fakeInsertPlan := &TabletPlan{
Plan: &planbuilder.Plan{
PlanID: planbuilder.PlanInsert,
FullQuery: &sqlparser.ParsedQuery{Query: `insert into something (id, msg) values(123, 'hello world!')`}, // 59 length
},
}
testcases := []struct {
name string
planType planbuilder.PlanType
plan *TabletPlan
tableName string
queryCount int64
duration time.Duration
Expand All @@ -590,12 +602,13 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes string
expectedQueryRowsAffected string
expectedQueryRowsReturned string
expectedQuerySQLTextCounts string
expectedQueryErrorCounts string
expectedQueryRowCounts string
}{
{
name: "select query",
planType: planbuilder.PlanSelect,
plan: fakeSelectPlan,
tableName: "A",
queryCount: 1,
duration: 10,
Expand All @@ -609,11 +622,12 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes: `{"A.Select": 10}`,
expectedQueryRowsAffected: `{}`,
expectedQueryRowsReturned: `{"A.Select": 15}`,
expectedQuerySQLTextCounts: `{"A.Select": 43}`,
expectedQueryRowCounts: `{"A.Select": 0}`,
expectedQueryErrorCounts: `{"A.Select": 0}`,
}, {
name: "select into query",
planType: planbuilder.PlanSelect,
plan: fakeSelectPlan,
tableName: "A",
queryCount: 1,
duration: 10,
Expand All @@ -627,11 +641,12 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes: `{"A.Select": 10}`,
expectedQueryRowsAffected: `{"A.Select": 15}`,
expectedQueryRowsReturned: `{"A.Select": 0}`,
expectedQuerySQLTextCounts: `{"A.Select": 43}`,
expectedQueryRowCounts: `{"A.Select": 15}`,
expectedQueryErrorCounts: `{"A.Select": 0}`,
}, {
name: "error",
planType: planbuilder.PlanSelect,
plan: fakeSelectPlan,
tableName: "A",
queryCount: 1,
duration: 10,
Expand All @@ -645,11 +660,12 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes: `{"A.Select": 10}`,
expectedQueryRowsAffected: `{}`,
expectedQueryRowsReturned: `{"A.Select": 0}`,
expectedQuerySQLTextCounts: `{"A.Select": 43}`,
expectedQueryRowCounts: `{"A.Select": 0}`,
expectedQueryErrorCounts: `{"A.Select": 1}`,
}, {
name: "insert query",
planType: planbuilder.PlanInsert,
plan: fakeInsertPlan,
tableName: "A",
queryCount: 1,
duration: 10,
Expand All @@ -663,11 +679,12 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes: `{"A.Insert": 10}`,
expectedQueryRowsAffected: `{"A.Insert": 15}`,
expectedQueryRowsReturned: `{}`,
expectedQuerySQLTextCounts: `{"A.Insert": 59}`,
expectedQueryRowCounts: `{"A.Insert": 15}`,
expectedQueryErrorCounts: `{"A.Insert": 0}`,
}, {
name: "select query with per workload metrics",
planType: planbuilder.PlanSelect,
plan: fakeSelectPlan,
tableName: "A",
queryCount: 1,
duration: 10,
Expand All @@ -681,11 +698,12 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes: `{"A.Select.some-workload": 10}`,
expectedQueryRowsAffected: `{}`,
expectedQueryRowsReturned: `{"A.Select.some-workload": 15}`,
expectedQuerySQLTextCounts: `{"A.Select.some-workload": 43}`,
expectedQueryRowCounts: `{"A.Select.some-workload": 0}`,
expectedQueryErrorCounts: `{"A.Select.some-workload": 0}`,
}, {
name: "select into query with per workload metrics",
planType: planbuilder.PlanSelect,
plan: fakeSelectPlan,
tableName: "A",
queryCount: 1,
duration: 10,
Expand All @@ -699,11 +717,12 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes: `{"A.Select.some-workload": 10}`,
expectedQueryRowsAffected: `{"A.Select.some-workload": 15}`,
expectedQueryRowsReturned: `{"A.Select.some-workload": 0}`,
expectedQuerySQLTextCounts: `{"A.Select.some-workload": 43}`,
expectedQueryRowCounts: `{"A.Select.some-workload": 15}`,
expectedQueryErrorCounts: `{"A.Select.some-workload": 0}`,
}, {
name: "error with per workload metrics",
planType: planbuilder.PlanSelect,
plan: fakeSelectPlan,
tableName: "A",
queryCount: 1,
duration: 10,
Expand All @@ -717,11 +736,12 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes: `{"A.Select.some-workload": 10}`,
expectedQueryRowsAffected: `{}`,
expectedQueryRowsReturned: `{"A.Select.some-workload": 0}`,
expectedQuerySQLTextCounts: `{"A.Select.some-workload": 43}`,
expectedQueryRowCounts: `{"A.Select.some-workload": 0}`,
expectedQueryErrorCounts: `{"A.Select.some-workload": 1}`,
}, {
name: "insert query with per workload metrics",
planType: planbuilder.PlanInsert,
plan: fakeInsertPlan,
tableName: "A",
queryCount: 1,
duration: 10,
Expand All @@ -735,6 +755,7 @@ func TestAddQueryStats(t *testing.T) {
expectedQueryTimes: `{"A.Insert.some-workload": 10}`,
expectedQueryRowsAffected: `{"A.Insert.some-workload": 15}`,
expectedQueryRowsReturned: `{}`,
expectedQuerySQLTextCounts: `{"A.Insert.some-workload": 59}`,
expectedQueryRowCounts: `{"A.Insert.some-workload": 15}`,
expectedQueryErrorCounts: `{"A.Insert.some-workload": 0}`,
},
Expand All @@ -749,12 +770,13 @@ func TestAddQueryStats(t *testing.T) {
env := tabletenv.NewEnv(config, "TestAddQueryStats_"+testcase.name)
se := schema.NewEngine(env)
qe := NewQueryEngine(env, se)
qe.AddStats(testcase.planType, testcase.tableName, testcase.workload, testcase.queryCount, testcase.duration, testcase.mysqlTime, testcase.rowsAffected, testcase.rowsReturned, testcase.errorCount)
qe.AddStats(testcase.plan, testcase.tableName, testcase.workload, testcase.queryCount, testcase.duration, testcase.mysqlTime, testcase.rowsAffected, testcase.rowsReturned, testcase.errorCount)
assert.Equal(t, testcase.expectedQueryCounts, qe.queryCounts.String())
assert.Equal(t, testcase.expectedQueryTimes, qe.queryTimes.String())
assert.Equal(t, testcase.expectedQueryRowsAffected, qe.queryRowsAffected.String())
assert.Equal(t, testcase.expectedQueryRowsReturned, qe.queryRowsReturned.String())
assert.Equal(t, testcase.expectedQueryRowCounts, qe.queryRowCounts.String())
assert.Equal(t, testcase.expectedQuerySQLTextCounts, qe.querySQLTextCounts.String())
assert.Equal(t, testcase.expectedQueryErrorCounts, qe.queryErrorCounts.String())
})
}
Expand Down
4 changes: 2 additions & 2 deletions go/vt/vttablet/tabletserver/query_executor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,11 @@ func (qre *QueryExecutor) Execute() (reply *sqltypes.Result, err error) {
}

if reply == nil {
qre.tsv.qe.AddStats(qre.plan.PlanID, tableName, qre.options.GetWorkloadName(), 1, duration, mysqlTime, 0, 0, 1)
qre.tsv.qe.AddStats(qre.plan, tableName, qre.options.GetWorkloadName(), 1, duration, mysqlTime, 0, 0, 1)
qre.plan.AddStats(1, duration, mysqlTime, 0, 0, 1)
return
}
qre.tsv.qe.AddStats(qre.plan.PlanID, tableName, qre.options.GetWorkloadName(), 1, duration, mysqlTime, int64(reply.RowsAffected), int64(len(reply.Rows)), 0)
qre.tsv.qe.AddStats(qre.plan, tableName, qre.options.GetWorkloadName(), 1, duration, mysqlTime, int64(reply.RowsAffected), int64(len(reply.Rows)), 0)
qre.plan.AddStats(1, duration, mysqlTime, reply.RowsAffected, uint64(len(reply.Rows)), 0)
qre.logStats.RowsAffected = int(reply.RowsAffected)
qre.logStats.Rows = reply.Rows
Expand Down

0 comments on commit b6812a7

Please sign in to comment.