From 44de903d0701ef35bf734f2d8b6c2410c3cde85c Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Wed, 23 Dec 2020 21:54:42 +0100 Subject: [PATCH] add greedy option for large queries Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/querygraph.go | 2 +- go/vt/vtgate/planbuilder/route_planning.go | 78 ++++++++++++++++--- go/vt/vtgate/planbuilder/testdata/onecase.txt | 40 +--------- 3 files changed, 70 insertions(+), 50 deletions(-) diff --git a/go/vt/vtgate/planbuilder/querygraph.go b/go/vt/vtgate/planbuilder/querygraph.go index 550cd88a900..f24427c178d 100644 --- a/go/vt/vtgate/planbuilder/querygraph.go +++ b/go/vt/vtgate/planbuilder/querygraph.go @@ -162,7 +162,7 @@ func (qg *queryGraph) addNoDepsPredicate(predicate sqlparser.Expr) { } } -func (qg *queryGraph) tryMerge(a, b joinTree, joinPredicates []sqlparser.Expr) joinTree { +func tryMerge(a, b joinTree, joinPredicates []sqlparser.Expr) joinTree { aRoute, ok := a.(*routePlan) if !ok { return nil diff --git a/go/vt/vtgate/planbuilder/route_planning.go b/go/vt/vtgate/planbuilder/route_planning.go index 60144023284..8c2785ff4a7 100644 --- a/go/vt/vtgate/planbuilder/route_planning.go +++ b/go/vt/vtgate/planbuilder/route_planning.go @@ -42,7 +42,12 @@ func newBuildSelectPlan(sel *sqlparser.Select, vschema ContextVSchema) (engine.P return nil, err } - tree, err := solve(qgraph, semTable, vschema) + var tree joinTree + if len(qgraph.tables) <= 10 { + tree, err = dpSolve(qgraph, semTable, vschema) + } else { + tree, err = greedySolve(qgraph, semTable, vschema) + } if err != nil { return nil, err } @@ -218,7 +223,7 @@ func (jp *joinPlan) cost() int { we use dynamic programming to find the cheapest route/join tree possible, where the cost of a plan is the number of joins */ -func solve(qg *queryGraph, semTable *semantics.SemTable, vschema ContextVSchema) (joinTree, error) { +func dpSolve(qg *queryGraph, semTable *semantics.SemTable, vschema ContextVSchema) (joinTree, error) { size := len(qg.tables) dpTable := makeDPTable() @@ -251,14 +256,7 @@ func solve(qg *queryGraph, semTable *semantics.SemTable, vschema ContextVSchema) continue } joinPredicates := qg.crossTable[solves] - newPlan := qg.tryMerge(lhs, rhs, joinPredicates) - if newPlan == nil { - newPlan = &joinPlan{ - lhs: lhs, - rhs: rhs, - predicates: joinPredicates, - } - } + newPlan := createJoin(lhs, rhs, joinPredicates) if oldPlan == nil || newPlan.cost() < oldPlan.cost() { dpTable.add(newPlan) } @@ -269,6 +267,66 @@ func solve(qg *queryGraph, semTable *semantics.SemTable, vschema ContextVSchema) return dpTable.planFor(allTables), nil } +func createJoin(lhs joinTree, rhs joinTree, joinPredicates []sqlparser.Expr) joinTree { + newPlan := tryMerge(lhs, rhs, joinPredicates) + if newPlan == nil { + newPlan = &joinPlan{ + lhs: lhs, + rhs: rhs, + predicates: joinPredicates, + } + } + return newPlan +} + +/* + + */ +func greedySolve(qg *queryGraph, semTable *semantics.SemTable, vschema ContextVSchema) (joinTree, error) { + plans := make([]joinTree, len(qg.tables)) + planCache := map[semantics.TableSet]joinTree{} + + // we start by seeding the table with the single routes + for i, table := range qg.tables { + solves := semTable.TableSetFor(table.alias) + plan, err := createRoutePlan(table, solves, vschema) + if err != nil { + return nil, err + } + plans[i] = plan + } + + // loop while we have un-joined query parts left + for len(plans) > 1 { + var lIdx, rIdx int + var bestPlan joinTree + for i, lhs := range plans { + for j := i + 1; j < len(plans); j++ { + rhs := plans[j] + solves := lhs.solves() | rhs.solves() + joinPredicates := qg.crossTable[solves] + plan := planCache[solves] + if plan == nil { + plan = createJoin(lhs, rhs, joinPredicates) + planCache[solves] = plan + } + + if bestPlan == nil || plan.cost() < bestPlan.cost() { + bestPlan = plan + // remember which plans we based on, so we can remove them later + lIdx = i + rIdx = j + } + } + } + plans = append(plans[:rIdx], plans[rIdx+1:]...) + plans = append(plans[:lIdx], plans[lIdx+1:]...) + plans = append(plans, bestPlan) + } + + return plans[0], nil +} + func createRoutePlan(table *queryTable, solves semantics.TableSet, vschema ContextVSchema) (*routePlan, error) { vschemaTable, _, _, _, _, err := vschema.FindTableOrVindex(table.table) if err != nil { diff --git a/go/vt/vtgate/planbuilder/testdata/onecase.txt b/go/vt/vtgate/planbuilder/testdata/onecase.txt index 0ce1a45f0c9..16a368ddafd 100644 --- a/go/vt/vtgate/planbuilder/testdata/onecase.txt +++ b/go/vt/vtgate/planbuilder/testdata/onecase.txt @@ -1,39 +1 @@ -# Add your test case here for debugging and run go test -run=One. -# ',' join -"select music.col from user, music" -{ - "QueryType": "SELECT", - "Original": "select music.col from user, music", - "Instructions": { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "1", - "TableName": "user_music", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from user where 1 != 1", - "Query": "select 1 from user", - "Table": "user" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select music.col from music where 1 != 1", - "Query": "select music.col from music", - "Table": "music" - } - ] - } -} -{ -} +# Add your test case here for debugging and run go test -run=One. \ No newline at end of file