Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add schema to StatsQualifier for schema databases #2762

Merged
merged 2 commits into from
Nov 27, 2024
Merged

Add schema to StatsQualifier for schema databases #2762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0945a0b
Add schema to StatsQualifier for schema databases
tbantle22 Nov 22, 2024
077393d
Fix comment
tbantle22 Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions enginetest/queries/stats_queries.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ var StatisticsQueries = []ScriptTest{
SkipResultCheckOnServerEngine: true, // the non-interface types are not identified over the wire result
Query: "SELECT * FROM information_schema.column_statistics",
Expected: []sql.Row{
{"mydb", "t", "i", stats.NewStatistic(3, 3, 0, 24, time.Now(), sql.NewStatQualifier("mydb", "t", "primary"), []string{"i"}, []sql.Type{types.Int64}, []sql.HistogramBucket{
{"mydb", "t", "i", stats.NewStatistic(3, 3, 0, 24, time.Now(), sql.NewStatQualifier("mydb", "", "t", "primary"), []string{"i"}, []sql.Type{types.Int64}, []sql.HistogramBucket{
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(1)}, nil, nil),
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(2)}, nil, nil),
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(3)}, nil, nil),
Expand All @@ -60,7 +60,7 @@ var StatisticsQueries = []ScriptTest{
SkipResultCheckOnServerEngine: true, // the non-interface types are not identified over the wire result
Query: "SELECT * FROM information_schema.column_statistics",
Expected: []sql.Row{
{"mydb", "t", "i", stats.NewStatistic(40, 40, 1, 0, time.Now(), sql.NewStatQualifier("mydb", "t", "primary"), []string{"i"}, []sql.Type{types.Int64}, []sql.HistogramBucket{
{"mydb", "t", "i", stats.NewStatistic(40, 40, 1, 0, time.Now(), sql.NewStatQualifier("mydb", "", "t", "primary"), []string{"i"}, []sql.Type{types.Int64}, []sql.HistogramBucket{
stats.NewHistogramBucket(20, 20, 0, 1, sql.Row{float64(50)}, nil, nil),
stats.NewHistogramBucket(20, 20, 0, 1, sql.Row{float64(80)}, nil, nil),
}, sql.IndexClassDefault, nil),
Expand Down Expand Up @@ -89,13 +89,13 @@ var StatisticsQueries = []ScriptTest{
SkipResultCheckOnServerEngine: true, // the non-interface types are not identified over the wire result
Query: "SELECT * FROM information_schema.column_statistics",
Expected: []sql.Row{
{"mydb", "t", "i", stats.NewStatistic(3, 3, 0, 48, time.Now(), sql.NewStatQualifier("mydb", "t", "primary"), []string{"i"}, []sql.Type{types.Int64}, []sql.HistogramBucket{
{"mydb", "t", "i", stats.NewStatistic(3, 3, 0, 48, time.Now(), sql.NewStatQualifier("mydb", "", "t", "primary"), []string{"i"}, []sql.Type{types.Int64}, []sql.HistogramBucket{
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(1)}, nil, []sql.Row{}),
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(2)}, nil, []sql.Row{}),
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(3)}, nil, []sql.Row{}),
}, sql.IndexClassDefault, nil),
},
{"mydb", "t", "j", stats.NewStatistic(3, 3, 0, 48, time.Now(), sql.NewStatQualifier("mydb", "t", "j"), []string{"j"}, []sql.Type{types.Int64}, []sql.HistogramBucket{
{"mydb", "t", "j", stats.NewStatistic(3, 3, 0, 48, time.Now(), sql.NewStatQualifier("mydb", "", "t", "j"), []string{"j"}, []sql.Type{types.Int64}, []sql.HistogramBucket{
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(4)}, nil, []sql.Row{}),
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(5)}, nil, []sql.Row{}),
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{int64(6)}, nil, []sql.Row{}),
Expand All @@ -117,7 +117,7 @@ var StatisticsQueries = []ScriptTest{
SkipResultCheckOnServerEngine: true, // the non-interface types are not identified over the wire result
Query: "SELECT * FROM information_schema.column_statistics",
Expected: []sql.Row{
{"mydb", "t", "i", stats.NewStatistic(4, 4, 0, 32, time.Now(), sql.NewStatQualifier("mydb", "t", "primary"), []string{"i"}, []sql.Type{types.Float64}, []sql.HistogramBucket{
{"mydb", "t", "i", stats.NewStatistic(4, 4, 0, 32, time.Now(), sql.NewStatQualifier("mydb", "", "t", "primary"), []string{"i"}, []sql.Type{types.Float64}, []sql.HistogramBucket{
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{float64(1.25)}, nil, []sql.Row{}),
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{float64(7.5)}, nil, []sql.Row{}),
stats.NewHistogramBucket(1, 1, 0, 1, sql.Row{float64(10.5)}, nil, []sql.Row{}),
Expand Down
16 changes: 13 additions & 3 deletions memory/stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,17 @@ func (s *StatsProv) estimateStats(ctx *sql.Context, table sql.Table, keys map[st
types = append(types, sch[i].Type)
}

qual, err := sql.NewQualifierFromString(string(key))
var schemaName string
if tabSch, ok := table.(sql.DatabaseSchemaTable); ok {
schemaName = tabSch.DatabaseSchema().SchemaName()
}

var qual sql.StatQualifier
if schemaName == "" {
qual, err = sql.NewQualifierFromString(string(key))
} else {
qual, err = sql.NewSchemaQualifierFromString(string(key))
}
if err != nil {
return err
}
Expand Down Expand Up @@ -260,7 +270,7 @@ func (s *StatsProv) GetStats(ctx *sql.Context, qual sql.StatQualifier, cols []st

func (s *StatsProv) DropStats(ctx *sql.Context, qual sql.StatQualifier, cols []string) error {
colsSuff := strings.Join(cols, ",") + ")"
for key, _ := range s.colStats {
for key := range s.colStats {
if strings.HasPrefix(string(key), qual.String()) && strings.HasSuffix(string(key), colsSuff) {
delete(s.colStats, key)
}
Expand Down Expand Up @@ -295,7 +305,7 @@ func (s *StatsProv) DataLength(ctx *sql.Context, db string, table sql.Table) (ui
}

func (s *StatsProv) DropDbStats(ctx *sql.Context, db string, flush bool) error {
for key, _ := range s.colStats {
for key := range s.colStats {
if strings.HasPrefix(string(key), db) {
delete(s.colStats, key)
}
Expand Down
21 changes: 15 additions & 6 deletions sql/analyzer/costed_index_scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,17 @@ func getCostedIndexScan(ctx *sql.Context, statsProv sql.StatsProvider, rt sql.Ta
if dbTab, ok := rt.UnderlyingTable().(sql.Databaseable); ok {
dbName = strings.ToLower(dbTab.Database())
}
tableName := strings.ToLower(rt.UnderlyingTable().Name())
table := rt.UnderlyingTable()
var schemaName string
if schTab, ok := table.(sql.DatabaseSchemaTable); ok {
schemaName = strings.ToLower(schTab.DatabaseSchema().SchemaName())
}
tableName := strings.ToLower(table.Name())

if len(qualToStat) > 0 {
// don't mix and match real and default stats
for _, idx := range indexes {
qual := sql.NewStatQualifier(dbName, tableName, strings.ToLower(idx.ID()))
qual := sql.NewStatQualifier(dbName, schemaName, tableName, strings.ToLower(idx.ID()))
_, ok := qualToStat[qual]
if !ok {
qualToStat = nil
Expand All @@ -188,7 +193,7 @@ func getCostedIndexScan(ctx *sql.Context, statsProv sql.StatsProvider, rt sql.Ta
}

for _, idx := range indexes {
qual := sql.NewStatQualifier(dbName, tableName, strings.ToLower(idx.ID()))
qual := sql.NewStatQualifier(dbName, schemaName, tableName, strings.ToLower(idx.ID()))
stat, ok := qualToStat[qual]
if !ok {
stat, err = uniformDistStatisticsForIndex(ctx, statsProv, iat, idx)
Expand Down Expand Up @@ -1536,6 +1541,10 @@ func uniformDistStatisticsForIndex(ctx *sql.Context, statsProv sql.StatsProvider
if dbTable, ok := iat.(sql.Databaseable); ok {
dbName = strings.ToLower(dbTable.Database())
}
var schemaName string
if schTab, ok := iat.(sql.DatabaseSchemaTable); ok {
schemaName = strings.ToLower(schTab.DatabaseSchema().SchemaName())
}
tableName := strings.ToLower(iat.Name())

var sch sql.Schema
Expand All @@ -1545,7 +1554,7 @@ func uniformDistStatisticsForIndex(ctx *sql.Context, statsProv sql.StatsProvider
sch = iat.Schema()
}

return newUniformDistStatistic(dbName, tableName, sch, idx, rowCount, avgSize)
return newUniformDistStatistic(dbName, schemaName, tableName, sch, idx, rowCount, avgSize)
}

func indexFds(tableName string, sch sql.Schema, idx sql.Index) (*sql.FuncDepSet, sql.ColSet, error) {
Expand Down Expand Up @@ -1588,7 +1597,7 @@ func indexFds(tableName string, sch sql.Schema, idx sql.Index) (*sql.FuncDepSet,
return sql.NewTablescanFDs(all, strictKeys, laxKeys, notNull), idxCols, nil
}

func newUniformDistStatistic(dbName, tableName string, sch sql.Schema, idx sql.Index, rowCount, avgSize uint64) (sql.Statistic, error) {
func newUniformDistStatistic(dbName, schemaName, tableName string, sch sql.Schema, idx sql.Index, rowCount, avgSize uint64) (sql.Statistic, error) {
tablePrefix := fmt.Sprintf("%s.", tableName)

distinctCount := rowCount
Expand All @@ -1615,7 +1624,7 @@ func newUniformDistStatistic(dbName, tableName string, sch sql.Schema, idx sql.I
class = sql.IndexClassDefault
}

qual := sql.NewStatQualifier(dbName, tableName, strings.ToLower(idx.ID()))
qual := sql.NewStatQualifier(dbName, schemaName, tableName, strings.ToLower(idx.ID()))
stat := stats.NewStatistic(rowCount, distinctCount, nullCount, avgSize, time.Now(), qual, cols, types, nil, class, nil)

fds, idxCols, err := indexFds(tableName, sch, idx)
Expand Down
2 changes: 1 addition & 1 deletion sql/analyzer/costed_index_scan_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -583,7 +583,7 @@ func TestRangeBuilder(t *testing.T) {
idx = idx_1
}

stat, err := newUniformDistStatistic("mydb", testTable, sch, idx, 10, 10)
stat, err := newUniformDistStatistic("mydb", "", testTable, sch, idx, 10, 10)
require.NoError(t, err)

err = c.cost(root, stat, idx)
Expand Down
7 changes: 6 additions & 1 deletion sql/analyzer/indexed_joins.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1344,7 +1344,12 @@ func makeIndexScan(ctx *sql.Context, statsProv sql.StatsProvider, tab plan.Table
for _, e := range idx.SqlIdx().Expressions() {
cols = append(cols, strings.TrimPrefix(e, tablePrefix))
}
stats, _ := statsProv.GetStats(ctx, sql.NewStatQualifier(tn.Database().Name(), tn.Name(), idx.SqlIdx().ID()), cols)
var schemaName string
if schTab, ok := tn.(sql.DatabaseSchemaTable); ok {
schemaName = strings.ToLower(schTab.DatabaseSchema().SchemaName())
}

stats, _ := statsProv.GetStats(ctx, sql.NewStatQualifier(tn.Database().Name(), schemaName, tn.Name(), idx.SqlIdx().ID()), cols)

return &memo.IndexScan{
Table: ret,
Expand Down
17 changes: 11 additions & 6 deletions sql/plan/histogram.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,15 +79,16 @@ func (u *UpdateHistogram) IsReadOnly() bool {
return false
}

func NewDropHistogram(db, table string, cols []string) *DropHistogram {
return &DropHistogram{db: db, cols: cols, table: table}
func NewDropHistogram(db, schema, table string, cols []string) *DropHistogram {
return &DropHistogram{db: db, schema: schema, cols: cols, table: table}
}

type DropHistogram struct {
db string
table string
cols []string
prov sql.StatsProvider
db string
schema string
table string
cols []string
prov sql.StatsProvider
}

var _ sql.Node = (*DropHistogram)(nil)
Expand All @@ -106,6 +107,10 @@ func (d *DropHistogram) Db() string {
return d.db
}

func (d *DropHistogram) SchemaName() string {
return d.schema
}

func (d *DropHistogram) Table() string {
return d.table
}
Expand Down
8 changes: 4 additions & 4 deletions sql/planbuilder/analyze.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func (b *Builder) buildAnalyze(inScope *scope, n *ast.Analyze, query string) (ou
switch n.Action {
case ast.UpdateStr:
sch := tableScope.node.Schema()
return b.buildAnalyzeUpdate(inScope, n, strings.ToLower(n.Tables[0].DbQualifier.String()), strings.ToLower(n.Tables[0].Name.String()), sch, columns, types)
return b.buildAnalyzeUpdate(inScope, n, strings.ToLower(n.Tables[0].DbQualifier.String()), strings.ToLower(n.Tables[0].SchemaQualifier.String()), strings.ToLower(n.Tables[0].Name.String()), sch, columns, types)
case ast.DropStr:
outScope = inScope.push()
dbName := n.Tables[0].DbQualifier.String()
Expand All @@ -81,7 +81,7 @@ func (b *Builder) buildAnalyze(inScope *scope, n *ast.Analyze, query string) (ou
b.handleErr(sql.ErrNoDatabaseSelected.New())
}

outScope.node = plan.NewDropHistogram(strings.ToLower(dbName), strings.ToLower(n.Tables[0].Name.String()), columns).WithProvider(b.cat)
outScope.node = plan.NewDropHistogram(strings.ToLower(dbName), strings.ToLower(n.Tables[0].SchemaQualifier.String()), strings.ToLower(n.Tables[0].Name.String()), columns).WithProvider(b.cat)
default:
err := fmt.Errorf("invalid ANALYZE action: %s, expected UPDATE or DROP", n.Action)
b.handleErr(err)
Expand Down Expand Up @@ -117,7 +117,7 @@ func (b *Builder) buildAnalyzeTables(inScope *scope, n *ast.Analyze, query strin
return
}

func (b *Builder) buildAnalyzeUpdate(inScope *scope, n *ast.Analyze, dbName, tableName string, sch sql.Schema, columns []string, types []sql.Type) (outScope *scope) {
func (b *Builder) buildAnalyzeUpdate(inScope *scope, n *ast.Analyze, dbName, schemaName, tableName string, sch sql.Schema, columns []string, types []sql.Type) (outScope *scope) {
if dbName == "" {
dbName = b.ctx.GetCurrentDatabase()
}
Expand Down Expand Up @@ -155,7 +155,7 @@ func (b *Builder) buildAnalyzeUpdate(inScope *scope, n *ast.Analyze, dbName, tab

statistic := statisticJ.ToStatistic()

statistic.SetQualifier(sql.NewStatQualifier(strings.ToLower(dbName), tableName, strings.ToLower(indexName)))
statistic.SetQualifier(sql.NewStatQualifier(strings.ToLower(dbName), strings.ToLower(schemaName), tableName, strings.ToLower(indexName)))
statistic.SetColumns(columns)
statistic.SetTypes(types)

Expand Down
1 change: 1 addition & 0 deletions sql/rowexec/other.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ func (b *BaseBuilder) buildDropHistogram(ctx *sql.Context, n *plan.DropHistogram

return &dropHistogramIter{
db: n.Db(),
schema: n.SchemaName(),
table: n.Table(),
columns: n.Cols(),
prov: n.StatsProvider(),
Expand Down
3 changes: 2 additions & 1 deletion sql/rowexec/other_iters.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func (itr *updateHistogramIter) Close(_ *sql.Context) error {

type dropHistogramIter struct {
db string
schema string
table string
columns []string
prov sql.StatsProvider
Expand All @@ -99,7 +100,7 @@ func (itr *dropHistogramIter) Next(ctx *sql.Context) (sql.Row, error) {
defer func() {
itr.done = true
}()
qual := sql.NewStatQualifier(itr.db, itr.table, "")
qual := sql.NewStatQualifier(itr.db, itr.schema, itr.table, "")
err := itr.prov.DropStats(ctx, qual, itr.columns)
if err != nil {
return sql.Row{itr.table, "histogram", "error", err.Error()}, nil
Expand Down
32 changes: 28 additions & 4 deletions sql/statistics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ type MutableStatistic interface {
WithLowerBound(Row) Statistic
}

// NewQualifierFromString creates a new StatQualifier from a string.
func NewQualifierFromString(q string) (StatQualifier, error) {
parts := strings.Split(q, ".")
if len(parts) < 3 {
Expand All @@ -99,23 +100,42 @@ func NewQualifierFromString(q string) (StatQualifier, error) {
return StatQualifier{Database: parts[0], Tab: parts[1], Idx: parts[2]}, nil
}

func NewStatQualifier(db, table, index string) StatQualifier {
return StatQualifier{Database: strings.ToLower(db), Tab: strings.ToLower(table), Idx: strings.ToLower(index)}
// NewSchemaQualifierFromString creates a new StatQualifier from a string,
// assuming the string contains a schema part.
func NewSchemaQualifierFromString(q string) (StatQualifier, error) {
parts := strings.Split(q, ".")
if len(parts) < 4 {
return StatQualifier{}, fmt.Errorf("invalid qualifier string: '%s', expected '<database>.<schema>.<table>.<index>'", q)
}
return StatQualifier{Database: parts[0], Sch: parts[1], Tab: parts[2], Idx: parts[3]}, nil
}

func NewStatQualifier(db, schema, table, index string) StatQualifier {
return StatQualifier{
Database: strings.ToLower(db),
Sch: strings.ToLower(schema),
Tab: strings.ToLower(table),
Idx: strings.ToLower(index)}
}

// StatQualifier is the namespace hierarchy for a given statistic.
// The qualifier and set of columns completely describes a unique stat.
type StatQualifier struct {
Database string `json:"database"`
Sch string `json:"schema"`
Tab string `json:"table"`
Idx string `json:"index"`
}

func (q StatQualifier) String() string {
tableName := q.Tab
if q.Sch != "" {
tableName = fmt.Sprintf("%s.%s", q.Sch, q.Tab)
}
if q.Idx != "" {
return fmt.Sprintf("%s.%s.%s", q.Database, q.Tab, q.Idx)
return fmt.Sprintf("%s.%s.%s", q.Database, tableName, q.Idx)
}
return fmt.Sprintf("%s.%s", q.Database, q.Tab)
return fmt.Sprintf("%s.%s", q.Database, tableName)
}

func (q StatQualifier) Empty() bool {
Expand All @@ -126,6 +146,10 @@ func (q StatQualifier) Db() string {
return q.Database
}

func (q StatQualifier) Schema() string {
return q.Sch
}

func (q StatQualifier) Table() string {
return q.Tab
}
Expand Down
Loading