Skip to content

Commit

Permalink
sql: add crdb_internal.{node,cluster}_txn_execution_insights
Browse files Browse the repository at this point in the history 
Closes cockroachdb#93075

This commit adds 2 new virtual tables displaying execution
insights for transaction.
- crdb_internal.cluster_txn_execution_insights
- crdb_internal.node_txn_execution_insights

Release note (sql change):
2 new virtual tables displaying execution insights for transaction:
- crdb_internal.cluster_txn_execution_insights
- crdb_internal.node_txn_execution_insights
  • Loading branch information
@xinhaoz
xinhaoz committed Jan 9, 2023
1 parent 308b37d commit 4435b19
Show file tree
Hide file tree
Showing 9 changed files with 1,186 additions and 1,005 deletions.
2 changes: 2 additions & 0 deletions pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ crdb_internal cluster_settings table admin NULL NULL
crdb_internal cluster_statement_statistics table admin NULL NULL
crdb_internal cluster_transaction_statistics table admin NULL NULL
crdb_internal cluster_transactions table admin NULL NULL
crdb_internal cluster_txn_execution_insights table admin NULL NULL
crdb_internal create_function_statements table admin NULL NULL
crdb_internal create_schema_statements table admin NULL NULL
crdb_internal create_statements table admin NULL NULL
Expand Down Expand Up @@ -79,6 +80,7 @@ crdb_internal node_sessions table admin NULL NULL
crdb_internal node_statement_statistics table admin NULL NULL
crdb_internal node_transaction_statistics table admin NULL NULL
crdb_internal node_transactions table admin NULL NULL
crdb_internal node_txn_execution_insights table admin NULL NULL
crdb_internal node_txn_stats table admin NULL NULL
crdb_internal partitions table admin NULL NULL
crdb_internal pg_catalog_table_is_implemented table admin NULL NULL
Expand Down
172 changes: 167 additions & 5 deletions pkg/sql/crdb_internal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ var crdbInternal = virtualSchema{
catconstants.CrdbInternalClusterContentionEventsTableID: crdbInternalClusterContentionEventsTable,
catconstants.CrdbInternalClusterDistSQLFlowsTableID: crdbInternalClusterDistSQLFlowsTable,
catconstants.CrdbInternalClusterExecutionInsightsTableID: crdbInternalClusterExecutionInsightsTable,
catconstants.CrdbInternalClusterTxnExecutionInsightsTableID: crdbInternalClusterTxnExecutionInsightsTable,
catconstants.CrdbInternalClusterLocksTableID: crdbInternalClusterLocksTable,
catconstants.CrdbInternalClusterQueriesTableID: crdbInternalClusterQueriesTable,
catconstants.CrdbInternalClusterTransactionsTableID: crdbInternalClusterTxnsTable,
Expand Down Expand Up @@ -161,6 +162,7 @@ var crdbInternal = virtualSchema{
catconstants.CrdbInternalLocalMetricsTableID: crdbInternalLocalMetricsTable,
catconstants.CrdbInternalNodeExecutionInsightsTableID: crdbInternalNodeExecutionInsightsTable,
catconstants.CrdbInternalNodeStmtStatsTableID: crdbInternalNodeStmtStatsTable,
catconstants.CrdbInternalNodeTxnExecutionInsightsTableID: crdbInternalNodeTxnExecutionInsightsTable,
catconstants.CrdbInternalNodeTxnStatsTableID: crdbInternalNodeTxnStatsTable,
catconstants.CrdbInternalPartitionsTableID: crdbInternalPartitionsTable,
catconstants.CrdbInternalRangesNoLeasesTableID: crdbInternalRangesNoLeasesTable,
Expand Down Expand Up @@ -6878,7 +6880,165 @@ func populateClusterLocksWithFilter(
return matched, err
}

// This is the table structure for both {cluster,node}_txn_execution_insights.
const txnExecutionInsightsSchemaPattern = `
CREATE TABLE crdb_internal.%s (
txn_id UUID NOT NULL,
txn_fingerprint_id BYTES NOT NULL,
query STRING NOT NULL,
implicit_txn BOOL NOT NULL,
session_id STRING NOT NULL,
start_time TIMESTAMP NOT NULL,
end_time TIMESTAMP NOT NULL,
user_name STRING NOT NULL,
app_name STRING NOT NULL,
rows_read INT8 NOT NULL,
rows_written INT8 NOT NULL,
priority STRING NOT NULL,
retries INT8 NOT NULL,
last_retry_reason STRING,
contention INTERVAL,
problems STRING[] NOT NULL,
causes STRING[] NOT NULL,
stmt_execution_ids STRING[] NOT NULL
)`

var crdbInternalClusterTxnExecutionInsightsTable = virtualSchemaTable{
schema: fmt.Sprintf(txnExecutionInsightsSchemaPattern, "cluster_txn_execution_insights"),
comment: `Cluster transaction execution insights`,
populate: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, addRow func(...tree.Datum) error) (err error) {
return populateTxnExecutionInsights(ctx, p, addRow, &serverpb.ListExecutionInsightsRequest{})
},
}

var crdbInternalNodeTxnExecutionInsightsTable = virtualSchemaTable{
schema: fmt.Sprintf(txnExecutionInsightsSchemaPattern, "node_txn_execution_insights"),
comment: `Node transaction execution insights`,
populate: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, addRow func(...tree.Datum) error) (err error) {
return populateTxnExecutionInsights(ctx, p, addRow, &serverpb.ListExecutionInsightsRequest{NodeID: "local"})
},
}

func populateTxnExecutionInsights(
ctx context.Context,
p *planner,
addRow func(...tree.Datum) error,
request *serverpb.ListExecutionInsightsRequest,
) (err error) {
hasRoleOption, err := p.HasViewActivityOrViewActivityRedactedRole(ctx)
if err != nil {
return err
}
if !hasRoleOption {
return pgerror.Newf(
pgcode.InsufficientPrivilege,
"user %s does not have %s or %s privilege",
p.User(),
roleoption.VIEWACTIVITY,
roleoption.VIEWACTIVITYREDACTED,
)
}

response, err := p.extendedEvalCtx.SQLStatusServer.ListExecutionInsights(ctx, request)
if err != nil {
return err
}

// We should truncate the query if it surpasses some absurd limit.
queryMax := 5000
for _, insight := range response.Insights {
var queryBuilder strings.Builder
for i := range insight.Statements {
// Build query string.
remaining := queryMax - queryBuilder.Len()
if len(insight.Statements[i].Query) > remaining {
if remaining > 0 {
queryBuilder.WriteString(insight.Statements[i].Query[:remaining] + "...")
}
break
}
if i > 0 {
queryBuilder.WriteString(" ; ")
}
queryBuilder.WriteString(insight.Statements[i].Query)
}

problems := tree.NewDArray(types.String)
for i := range insight.Transaction.Problems {
if err = problems.Append(tree.NewDString(insight.Transaction.Problems[i].String())); err != nil {
return err
}
}

causes := tree.NewDArray(types.String)
for i := range insight.Transaction.Causes {
if err = causes.Append(tree.NewDString(insight.Transaction.Causes[i].String())); err != nil {
return err
}
}

var startTimestamp *tree.DTimestamp
startTimestamp, err = tree.MakeDTimestamp(insight.Transaction.StartTime, time.Nanosecond)
if err != nil {
return err
}

var endTimestamp *tree.DTimestamp
endTimestamp, err = tree.MakeDTimestamp(insight.Transaction.EndTime, time.Nanosecond)
if err != nil {
return err
}

autoRetryReason := tree.DNull
if insight.Transaction.AutoRetryReason != "" {
autoRetryReason = tree.NewDString(insight.Transaction.AutoRetryReason)
}

contentionTime := tree.DNull
if insight.Transaction.Contention != nil {
contentionTime = tree.NewDInterval(
duration.MakeDuration(insight.Transaction.Contention.Nanoseconds(), 0, 0),
types.DefaultIntervalTypeMetadata,
)
}

stmtIDs := tree.NewDArray(types.String)
for _, id := range insight.Transaction.StmtExecutionIDs {
if err = stmtIDs.Append(tree.NewDString(hex.EncodeToString(id.GetBytes()))); err != nil {
return err
}
}

err = errors.CombineErrors(err, addRow(
tree.NewDUuid(tree.DUuid{UUID: insight.Transaction.ID}),
tree.NewDBytes(tree.DBytes(sqlstatsutil.EncodeUint64ToBytes(uint64(insight.Transaction.FingerprintID)))),
tree.NewDString(queryBuilder.String()),
tree.MakeDBool(tree.DBool(insight.Transaction.ImplicitTxn)),
tree.NewDString(hex.EncodeToString(insight.Session.ID.GetBytes())),
startTimestamp,
endTimestamp,
tree.NewDString(insight.Transaction.User),
tree.NewDString(insight.Transaction.ApplicationName),
tree.NewDInt(tree.DInt(insight.Transaction.RowsRead)),
tree.NewDInt(tree.DInt(insight.Transaction.RowsWritten)),
tree.NewDString(insight.Transaction.UserPriority),
tree.NewDInt(tree.DInt(insight.Transaction.RetryCount)),
autoRetryReason,
contentionTime,
problems,
causes,
stmtIDs,
))

if err != nil {
return err
}
}
return
}

// This is the table structure for both cluster_execution_insights and node_execution_insights.
// Both contain statement execution insights.
const executionInsightsSchemaPattern = `
CREATE TABLE crdb_internal.%s (
session_id STRING NOT NULL,
Expand Down Expand Up @@ -6910,20 +7070,22 @@ CREATE TABLE crdb_internal.%s (
)`

var crdbInternalClusterExecutionInsightsTable = virtualSchemaTable{
schema: fmt.Sprintf(executionInsightsSchemaPattern, "cluster_execution_insights"),
schema: fmt.Sprintf(executionInsightsSchemaPattern, "cluster_execution_insights"),
comment: `Cluster-wide statement execution insights`,
populate: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, addRow func(...tree.Datum) error) (err error) {
return populateExecutionInsights(ctx, p, addRow, &serverpb.ListExecutionInsightsRequest{})
return populateStmtInsights(ctx, p, addRow, &serverpb.ListExecutionInsightsRequest{})
},
}

var crdbInternalNodeExecutionInsightsTable = virtualSchemaTable{
schema: fmt.Sprintf(executionInsightsSchemaPattern, "node_execution_insights"),
schema: fmt.Sprintf(executionInsightsSchemaPattern, "node_execution_insights"),
comment: `Node statement execution insights`,
populate: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, addRow func(...tree.Datum) error) (err error) {
return populateExecutionInsights(ctx, p, addRow, &serverpb.ListExecutionInsightsRequest{NodeID: "local"})
return populateStmtInsights(ctx, p, addRow, &serverpb.ListExecutionInsightsRequest{NodeID: "local"})
},
}

func populateExecutionInsights(
func populateStmtInsights(
ctx context.Context,
p *planner,
addRow func(...tree.Datum) error,
Expand Down
2 changes: 2 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/crdb_internal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ crdb_internal cluster_settings table admin NULL NULL
crdb_internal cluster_statement_statistics table admin NULL NULL
crdb_internal cluster_transaction_statistics table admin NULL NULL
crdb_internal cluster_transactions table admin NULL NULL
crdb_internal cluster_txn_execution_insights table admin NULL NULL
crdb_internal create_function_statements table admin NULL NULL
crdb_internal create_schema_statements table admin NULL NULL
crdb_internal create_statements table admin NULL NULL
Expand Down Expand Up @@ -71,6 +72,7 @@ crdb_internal node_sessions table admin NULL NULL
crdb_internal node_statement_statistics table admin NULL NULL
crdb_internal node_transaction_statistics table admin NULL NULL
crdb_internal node_transactions table admin NULL NULL
crdb_internal node_txn_execution_insights table admin NULL NULL
crdb_internal node_txn_stats table admin NULL NULL
crdb_internal partitions table admin NULL NULL
crdb_internal pg_catalog_table_is_implemented table admin NULL NULL
Expand Down
Loading

0 comments on commit 4435b19

Please sign in to comment.