Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

planner: add optimizer trace framework for physicalOptimize #30335

Merged
merged 21 commits into from
Jan 5, 2022
Merged

planner: add optimizer trace framework for physicalOptimize #30335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
1bcf81f
make fmt
rebelice Dec 2, 2021
4a74fb6
add test
rebelice Dec 2, 2021
888530c
fmt
rebelice Dec 2, 2021
ff0fad0
add a trace case
rebelice Dec 3, 2021
7d28788
address comments
rebelice Dec 27, 2021
b8a632e
address comments
rebelice Dec 30, 2021
7f49dd6
Merge branch 'master' into dev_optimizer_trace_framework
rebelice Dec 30, 2021
37ddb2c
address comments
rebelice Dec 31, 2021
832e765
address comment
rebelice Dec 31, 2021
dc6920a
address the comment
Yisaer Jan 4, 2022
83ba660
address comments
rebelice Jan 4, 2022
80f813c
fmt
rebelice Jan 4, 2022
82dd5cb
fix ci
rebelice Jan 4, 2022
25d1601
fix ci
rebelice Jan 4, 2022
3ddb2ae
add comments
rebelice Jan 4, 2022
a78c13e
fix ci
rebelice Jan 4, 2022
3206981
Merge branch 'master' into dev_optimizer_trace_framework
rebelice Jan 4, 2022
85c67ce
use testkit
rebelice Jan 5, 2022
2761fb2
Merge branch 'master' into dev_optimizer_trace_framework
rebelice Jan 5, 2022
04b4ad7
fix test
rebelice Jan 5, 2022
3520372
Merge branch 'master' into dev_optimizer_trace_framework
ti-chi-bot Jan 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 94 additions & 16 deletions planner/core/find_best_task.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/ranger"
"github.com/pingcap/tidb/util/set"
"github.com/pingcap/tidb/util/tracing"
"go.uber.org/zap"
)

Expand Down Expand Up @@ -136,7 +137,7 @@ func GetPropByOrderByItemsContainScalarFunc(items []*util.ByItems) (*property.Ph
return &property.PhysicalProperty{SortItems: propItems}, true, onlyColumn
}

func (p *LogicalTableDual) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (p *LogicalTableDual) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
// If the required property is not empty and the row count > 1,
// we cannot ensure this required property.
// But if the row count is 0 or 1, we don't need to care about the property.
Expand All @@ -151,7 +152,7 @@ func (p *LogicalTableDual) findBestTask(prop *property.PhysicalProperty, planCou
return &rootTask{p: dual, isEmpty: p.RowCount == 0}, 1, nil
}

func (p *LogicalShow) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (p *LogicalShow) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
if !prop.IsEmpty() || planCounter.Empty() {
return invalidTask, 0, nil
}
Expand All @@ -161,7 +162,7 @@ func (p *LogicalShow) findBestTask(prop *property.PhysicalProperty, planCounter
return &rootTask{p: pShow}, 1, nil
}

func (p *LogicalShowDDLJobs) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (p *LogicalShowDDLJobs) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
if !prop.IsEmpty() || planCounter.Empty() {
return invalidTask, 0, nil
}
Expand All @@ -172,7 +173,7 @@ func (p *LogicalShowDDLJobs) findBestTask(prop *property.PhysicalProperty, planC
}

// rebuildChildTasks rebuilds the childTasks to make the clock_th combination.
func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan, childCnts []int64, planCounter int64, TS uint64) error {
func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan, childCnts []int64, planCounter int64, TS uint64, opt *physicalOptimizeOp) error {
// The taskMap of children nodes should be rolled back first.
for _, child := range p.children {
child.rollBackTaskMap(TS)
Expand All @@ -187,7 +188,7 @@ func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan,
for j, child := range p.children {
multAll /= childCnts[j]
curClock = PlanCounterTp((planCounter-1)/multAll + 1)
childTask, _, err := child.findBestTask(pp.GetChildReqProps(j), &curClock)
childTask, _, err := child.findBestTask(pp.GetChildReqProps(j), &curClock, opt)
planCounter = (planCounter-1)%multAll + 1
if err != nil {
return err
Expand All @@ -203,21 +204,23 @@ func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]task, pp PhysicalPlan,
return nil
}

func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, addEnforcer bool, planCounter *PlanCounterTp) (task, int64, error) {
func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPlan, prop *property.PhysicalProperty, addEnforcer bool, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
var bestTask task = invalidTask
var curCntPlan, cntPlan int64
childTasks := make([]task, 0, len(p.children))
childCnts := make([]int64, len(p.children))
cntPlan = 0
for _, pp := range physicalPlans {
candidateInfo := opt.appendCandidate(p, pp, prop.String())
// Find best child tasks firstly.
childTasks = childTasks[:0]
// The curCntPlan records the number of possible plans for pp
curCntPlan = 1
TimeStampNow := p.GetLogicalTS4TaskMap()
savedPlanID := p.ctx.GetSessionVars().PlanID
for j, child := range p.children {
childTask, cnt, err := child.findBestTask(pp.GetChildReqProps(j), &PlanCounterDisabled)
childProp := pp.GetChildReqProps(j)
childTask, cnt, err := child.findBestTask(childProp, &PlanCounterDisabled, opt)
childCnts[j] = cnt
if err != nil {
return nil, 0, err
Expand All @@ -227,6 +230,9 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
break
}
childTasks = append(childTasks, childTask)
if opt != nil && childTask != nil {
opt.appendChildToCandidate(candidateInfo, childTask.plan())
}
}

// This check makes sure that there is no invalid child task.
Expand All @@ -238,7 +244,7 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
if planCounter.IsForce() && int64(*planCounter) <= curCntPlan {
p.ctx.GetSessionVars().PlanID = savedPlanID
curCntPlan = int64(*planCounter)
err := p.rebuildChildTasks(&childTasks, pp, childCnts, int64(*planCounter), TimeStampNow)
err := p.rebuildChildTasks(&childTasks, pp, childCnts, int64(*planCounter), TimeStampNow, opt)
if err != nil {
return nil, 0, err
}
Expand Down Expand Up @@ -274,6 +280,9 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
bestTask = curTask
break
}
if candidateInfo != nil {
candidateInfo.SetCost(curTask.cost())
}
// Get the most efficient one.
if curTask.cost() < bestTask.cost() || (bestTask.invalid() && !curTask.invalid()) {
bestTask = curTask
Expand All @@ -282,8 +291,71 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task(physicalPlans []PhysicalPl
return bestTask, cntPlan, nil
}

type physicalOptimizeOp struct {
// tracer is goring to track optimize steps during physical optimizing
tracer *tracing.PhysicalOptimizeTracer
}

func defaultPhysicalOptimizeOption() *physicalOptimizeOp {
return &physicalOptimizeOp{}
}

func (op *physicalOptimizeOp) withEnableOptimizeTracer(tracer *tracing.PhysicalOptimizeTracer) *physicalOptimizeOp {
op.tracer = tracer
return op
}

func (op *physicalOptimizeOp) buildPhysicalOptimizeTraceInfo(p LogicalPlan, prop string) *tracing.PhysicalOptimizeTraceInfo {
if op == nil || op.tracer == nil {
return nil
}
name := tracing.CodecPlanName(p.TP(), p.ID())
if _, ok := op.tracer.State[name]; !ok {
op.tracer.State[name] = make(map[string]*tracing.PhysicalOptimizeTraceInfo)
}
if info, ok := op.tracer.State[name][prop]; ok {
return info
}
traceInfo := &tracing.PhysicalOptimizeTraceInfo{Property: prop}
op.tracer.State[name][prop] = traceInfo
return traceInfo
}

func (op *physicalOptimizeOp) appendChildToCandidate(candidateInfo *tracing.PhysicalPlanTrace, plan PhysicalPlan) {
if op == nil || op.tracer == nil || candidateInfo == nil {
return
}
childPhysicalPlanTrace := &tracing.PhysicalPlanTrace{TP: plan.TP(), ID: plan.ID(), Info: plan.ExplainInfo(), Cost: plan.Cost()}
candidateInfo.Children = append(candidateInfo.Children, childPhysicalPlanTrace)
}

func (op *physicalOptimizeOp) setBest(lp LogicalPlan, pp PhysicalPlan, prop string) {
if op == nil || op.tracer == nil {
return
}
traceInfo := op.tracer.State[fmt.Sprintf("%v_%v", lp.TP(), lp.ID())][prop]
if traceInfo == nil {
return
}
traceInfo.BestTask = &tracing.PhysicalPlanTrace{ID: pp.ID(), TP: pp.TP(), Info: pp.ExplainInfo(), Cost: pp.Cost()}
}

func (op *physicalOptimizeOp) appendCandidate(logicalPlan *baseLogicalPlan, physicalPlan PhysicalPlan, prop string) *tracing.PhysicalPlanTrace {
if op == nil || op.tracer == nil {
return nil
}
PhysicalPlanTrace := &tracing.PhysicalPlanTrace{TP: physicalPlan.TP(), ID: physicalPlan.ID(), Info: physicalPlan.ExplainInfo()}
name := tracing.CodecPlanName(logicalPlan.TP(), logicalPlan.ID())
traceInfo := op.tracer.State[name][prop]
if traceInfo == nil {
return nil
}
traceInfo.Candidates = append(traceInfo.Candidates, PhysicalPlanTrace)
return PhysicalPlanTrace
}

// findBestTask implements LogicalPlan interface.
func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (bestTask task, cntPlan int64, err error) {
func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (bestTask task, cntPlan int64, err error) {
// If p is an inner plan in an IndexJoin, the IndexJoin will generate an inner plan by itself,
// and set inner child prop nil, so here we do nothing.
if prop == nil {
Expand Down Expand Up @@ -353,15 +425,18 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun

var cnt int64
var curTask task
if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, false, planCounter); err != nil {
if opt != nil {
opt.buildPhysicalOptimizeTraceInfo(p, newProp.String())
}
if bestTask, cnt, err = p.enumeratePhysicalPlans4Task(plansFitsProp, newProp, false, planCounter, opt); err != nil {
return nil, 0, err
}
cntPlan += cnt
if planCounter.Empty() {
goto END
}

curTask, cnt, err = p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp, true, planCounter)
curTask, cnt, err = p.enumeratePhysicalPlans4Task(plansNeedEnforce, newProp, true, planCounter, opt)
if err != nil {
return nil, 0, err
}
Expand All @@ -376,10 +451,13 @@ func (p *baseLogicalPlan) findBestTask(prop *property.PhysicalProperty, planCoun

END:
p.storeTask(prop, bestTask)
if opt != nil {
opt.setBest(p.self, bestTask.plan(), prop.String())
}
return bestTask, cntPlan, nil
}

func (p *LogicalMemTable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) {
func (p *LogicalMemTable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (t task, cntPlan int64, err error) {
if !prop.IsEmpty() || planCounter.Empty() {
return invalidTask, 0, nil
}
Expand Down Expand Up @@ -689,7 +767,7 @@ func (ds *DataSource) isPointGetConvertableSchema() bool {

// findBestTask implements the PhysicalPlan interface.
// It will enumerate all the available indices and choose a plan with least cost.
func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) {
func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (t task, cntPlan int64, err error) {
// If ds is an inner plan in an IndexJoin, the IndexJoin will generate an inner plan by itself,
// and set inner child prop nil, so here we do nothing.
if prop == nil {
Expand All @@ -710,7 +788,7 @@ func (ds *DataSource) findBestTask(prop *property.PhysicalProperty, planCounter
if prop.CanAddEnforcer {
// First, get the bestTask without enforced prop
prop.CanAddEnforcer = false
t, cnt, err = ds.findBestTask(prop, planCounter)
t, cnt, err = ds.findBestTask(prop, planCounter, opt)
if err != nil {
return nil, 0, err
}
Expand Down Expand Up @@ -2169,7 +2247,7 @@ func (ds *DataSource) getOriginalPhysicalIndexScan(prop *property.PhysicalProper
return is, cost, rowCount
}

func (p *LogicalCTE) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) {
func (p *LogicalCTE) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (t task, cntPlan int64, err error) {
if !prop.IsEmpty() && !prop.CanAddEnforcer {
return invalidTask, 1, nil
}
Expand All @@ -2187,7 +2265,7 @@ func (p *LogicalCTE) findBestTask(prop *property.PhysicalProperty, planCounter *
return t, 1, nil
}

func (p *LogicalCTETable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (t task, cntPlan int64, err error) {
func (p *LogicalCTETable) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (t task, cntPlan int64, err error) {
if !prop.IsEmpty() {
return nil, 1, nil
}
Expand Down
16 changes: 8 additions & 8 deletions planner/core/find_best_task_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (ds mockDataSource) Init(ctx sessionctx.Context) *mockDataSource {
return &ds
}

func (ds *mockDataSource) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp) (task, int64, error) {
func (ds *mockDataSource) findBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, opt *physicalOptimizeOp) (task, int64, error) {
// It can satisfy any of the property!
// Just use a TableDual for convenience.
p := PhysicalTableDual{}.Init(ds.ctx, &property.StatsInfo{RowCount: 1}, 0)
Expand Down Expand Up @@ -144,7 +144,7 @@ func TestCostOverflow(t *testing.T) {
mockPlan.SetChildren(mockDS)
// An empty property is enough for this test.
prop := property.NewPhysicalProperty(property.RootTaskType, nil, false, 0, false)
task, _, err := mockPlan.findBestTask(prop, &PlanCounterDisabled)
task, _, err := mockPlan.findBestTask(prop, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
// The cost should be overflowed, but the task shouldn't be invalid.
require.False(t, task.invalid())
Expand All @@ -171,7 +171,7 @@ func TestEnforcedProperty(t *testing.T) {
CanAddEnforcer: false,
}
// should return invalid task because no physical plan can match this property.
task, _, err := mockPlan.findBestTask(prop0, &PlanCounterDisabled)
task, _, err := mockPlan.findBestTask(prop0, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.True(t, task.invalid())

Expand All @@ -180,7 +180,7 @@ func TestEnforcedProperty(t *testing.T) {
CanAddEnforcer: true,
}
// should return the valid task when the property is enforced.
task, _, err = mockPlan.findBestTask(prop1, &PlanCounterDisabled)
task, _, err = mockPlan.findBestTask(prop1, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
}
Expand All @@ -203,7 +203,7 @@ func TestHintCannotFitProperty(t *testing.T) {
SortItems: items,
CanAddEnforcer: true,
}
task, _, err := mockPlan0.findBestTask(prop0, &PlanCounterDisabled)
task, _, err := mockPlan0.findBestTask(prop0, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
_, enforcedSort := task.plan().(*PhysicalSort)
Expand All @@ -219,7 +219,7 @@ func TestHintCannotFitProperty(t *testing.T) {
SortItems: items,
CanAddEnforcer: false,
}
task, _, err = mockPlan0.findBestTask(prop1, &PlanCounterDisabled)
task, _, err = mockPlan0.findBestTask(prop1, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
_, enforcedSort = task.plan().(*PhysicalSort)
Expand All @@ -240,7 +240,7 @@ func TestHintCannotFitProperty(t *testing.T) {
canGeneratePlan2: false,
}.Init(ctx)
mockPlan1.SetChildren(mockDS)
task, _, err = mockPlan1.findBestTask(prop2, &PlanCounterDisabled)
task, _, err = mockPlan1.findBestTask(prop2, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
require.Equal(t, uint16(1), ctx.GetSessionVars().StmtCtx.WarningCount())
Expand All @@ -256,7 +256,7 @@ func TestHintCannotFitProperty(t *testing.T) {
SortItems: items,
CanAddEnforcer: true,
}
task, _, err = mockPlan1.findBestTask(prop3, &PlanCounterDisabled)
task, _, err = mockPlan1.findBestTask(prop3, &PlanCounterDisabled, defaultPhysicalOptimizeOption())
require.NoError(t, err)
require.False(t, task.invalid())
require.Equal(t, uint16(1), ctx.GetSessionVars().StmtCtx.WarningCount())
Expand Down
12 changes: 11 additions & 1 deletion planner/core/optimizer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,8 +452,18 @@ func physicalOptimize(logic LogicalPlan, planCounter *PlanCounterTp) (PhysicalPl
ExpectedCnt: math.MaxFloat64,
}

opt := defaultPhysicalOptimizeOption()
stmtCtx := logic.SCtx().GetSessionVars().StmtCtx
if stmtCtx.EnableOptimizeTrace {
tracer := &tracing.PhysicalOptimizeTracer{State: make(map[string]map[string]*tracing.PhysicalOptimizeTraceInfo)}
opt = opt.withEnableOptimizeTracer(tracer)
defer func() {
stmtCtx.PhysicalOptimizeTrace = tracer
}()
}

logic.SCtx().GetSessionVars().StmtCtx.TaskMapBakTS = 0
t, _, err := logic.findBestTask(prop, planCounter)
t, _, err := logic.findBestTask(prop, planCounter, opt)
if err != nil {
return nil, 0, err
}
Expand Down
Loading