Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gen4: Subquery dependencies update #8998

Merged
merged 5 commits into from
Oct 14, 2021
Merged

gen4: Subquery dependencies update #8998

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
897cfa7
added failing testcases as unit tests
GuptaManan100 Oct 13, 2021
3954083
fixed subquery binding
GuptaManan100 Oct 14, 2021
4090a69
clean up code and add queries to test the new dependency code
systay Oct 14, 2021
c0d1609
update test assertions
systay Oct 14, 2021
438a96d
update query to test a more interesting situation
systay Oct 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions go/vt/vtgate/planbuilder/abstract/querygraph.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@ limitations under the License.
package abstract

import (
vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
"vitess.io/vitess/go/vt/sqlparser"
"vitess.io/vitess/go/vt/vterrors"
"vitess.io/vitess/go/vt/vtgate/semantics"
)

Expand Down Expand Up @@ -139,7 +137,8 @@ func (qg *QueryGraph) collectPredicate(predicate sqlparser.Expr, semTable *seman
case 1:
found := qg.addToSingleTable(deps, predicate)
if !found {
return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "table %v for predicate %v not found", deps, sqlparser.String(predicate))
// this could be a predicate that only has dependencies from outside this QG
qg.addJoinPredicates(deps, predicate)
}
default:
qg.addJoinPredicates(deps, predicate)
Expand Down
2 changes: 1 addition & 1 deletion go/vt/vtgate/planbuilder/route_planning.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func (c planningContext) isSubQueryToReplace(e sqlparser.Expr) bool {
return false
}
for _, extractedSubq := range c.semTable.GetSubqueryNeedingRewrite() {
if extractedSubq.NeedsRewrite && sqlparser.EqualsRefOfSubquery(&sqlparser.Subquery{Select: extractedSubq.Subquery.Select}, ext) {
if extractedSubq.NeedsRewrite && sqlparser.EqualsRefOfSubquery(extractedSubq.Subquery, ext) {
return true
}
}
Expand Down
147 changes: 147 additions & 0 deletions go/vt/vtgate/planbuilder/testdata/select_cases.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2201,3 +2201,150 @@ Gen4 plan same as above
}
}
Gen4 plan same as above

"select (select col from user limit 1) as a from user join user_extra order by a + 1"
"unsupported: in scatter query: complex order by expression: a + 1"
systay marked this conversation as resolved.
Show resolved Hide resolved
Gen4 plan same as above

"select t.a from (select (select col from user limit 1) as a from user join user_extra) t"
{
"QueryType": "SELECT",
"Original": "select t.a from (select (select col from user limit 1) as a from user join user_extra) t",
"Instructions": {
"OperatorType": "SimpleProjection",
"Columns": [
0
],
"Inputs": [
{
"OperatorType": "Subquery",
"Variant": "PulloutValue",
"PulloutVars": [
"__sq_has_values1",
"__sq1"
],
"Inputs": [
{
"OperatorType": "Limit",
"Count": 1,
"Inputs": [
{
"OperatorType": "Route",
"Variant": "SelectScatter",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select col from `user` where 1 != 1",
"Query": "select col from `user` limit :__upper_limit",
"Table": "`user`"
}
]
},
{
"OperatorType": "Join",
"Variant": "Join",
"JoinColumnIndexes": "-1",
"TableName": "`user`_user_extra",
"Inputs": [
{
"OperatorType": "Route",
"Variant": "SelectScatter",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select :__sq1 as a from `user` where 1 != 1",
"Query": "select :__sq1 as a from `user`",
"Table": "`user`"
},
{
"OperatorType": "Route",
"Variant": "SelectScatter",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select 1 from user_extra where 1 != 1",
"Query": "select 1 from user_extra",
"Table": "user_extra"
}
]
}
]
}
]
}
}
{
"QueryType": "SELECT",
"Original": "select t.a from (select (select col from user limit 1) as a from user join user_extra) t",
"Instructions": {
"OperatorType": "SimpleProjection",
"Columns": [
0
],
"Inputs": [
{
"OperatorType": "Subquery",
"Variant": "PulloutValue",
"PulloutVars": [
"__sq1"
],
"Inputs": [
{
"OperatorType": "Limit",
"Count": 1,
"Inputs": [
{
"OperatorType": "Route",
"Variant": "SelectScatter",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select col from `user` where 1 != 1",
"Query": "select col from `user` limit :__upper_limit",
"Table": "`user`"
}
]
},
{
"OperatorType": "Join",
"Variant": "Join",
"JoinColumnIndexes": "-1",
"TableName": "`user`_user_extra",
"Inputs": [
{
"OperatorType": "Route",
"Variant": "SelectScatter",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select :__sq1 as a from `user` where 1 != 1",
"Query": "select :__sq1 as a from `user`",
"Table": "`user`"
},
{
"OperatorType": "Route",
"Variant": "SelectScatter",
"Keyspace": {
"Name": "user",
"Sharded": true
},
"FieldQuery": "select 1 from user_extra where 1 != 1",
"Query": "select 1 from user_extra",
"Table": "user_extra"
}
]
}
]
}
]
}
}

"select (select col from user where user_extra.id = 4 limit 1) as a from user join user_extra"
"unsupported: cross-shard correlated subquery"
Gen4 plan same as above
105 changes: 61 additions & 44 deletions go/vt/vtgate/semantics/binder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,55 +56,15 @@ func (b *binder) up(cursor *sqlparser.Cursor) error {
switch node := cursor.Node().(type) {
case *sqlparser.Subquery:
currScope := b.scoper.currentScope()
if currScope.selectStmt == nil {
return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unable to bind subquery to select statement")
}

sq := &sqlparser.ExtractedSubquery{
Subquery: node,
Original: node,
OpCode: int(engine.PulloutValue),
}

switch par := cursor.Parent().(type) {
case *sqlparser.ComparisonExpr:
switch par.Operator {
case sqlparser.InOp:
sq.OpCode = int(engine.PulloutIn)
case sqlparser.NotInOp:
sq.OpCode = int(engine.PulloutNotIn)
}
subq, exp := GetSubqueryAndOtherSide(par)
sq.Original = &sqlparser.ComparisonExpr{
Left: exp,
Operator: par.Operator,
Right: subq,
}
sq.OtherSide = exp
case *sqlparser.ExistsExpr:
sq.OpCode = int(engine.PulloutExists)
sq.Original = par
sq, err := b.createExtractedSubquery(cursor, currScope, node)
if err != nil {
return err
}

b.subqueryMap[currScope.selectStmt] = append(b.subqueryMap[currScope.selectStmt], sq)
b.subqueryRef[node] = sq

subqRecursiveDeps := b.recursive.dependencies(node)
subqDirectDeps := b.direct.dependencies(node)

tablesToKeep := EmptyTableSet()
sco := currScope
for sco != nil {
for _, table := range sco.tables {
tablesToKeep.MergeInPlace(table.getTableSet(b.org))
}
sco = sco.parent
}

subqDirectDeps.KeepOnly(tablesToKeep)
subqRecursiveDeps.KeepOnly(tablesToKeep)
b.recursive[node] = subqRecursiveDeps
b.direct[node] = subqDirectDeps
b.setSubQueryDependencies(node, currScope)

case *sqlparser.ColName:
deps, err := b.resolveColumn(node, b.scoper.currentScope())
Expand Down Expand Up @@ -138,6 +98,62 @@ func (b *binder) up(cursor *sqlparser.Cursor) error {
return nil
}

// setSubQueryDependencies sets the correct dependencies for the subquery
// the binder usually only sets the dependencies of ColNames, but we need to
// handle the subquery dependencies differently, so they are set manually here
// this method will only keep dependencies to tables outside the subquery
func (b *binder) setSubQueryDependencies(subq *sqlparser.Subquery, currScope *scope) {
subqRecursiveDeps := b.recursive.dependencies(subq)
subqDirectDeps := b.direct.dependencies(subq)

tablesToKeep := EmptyTableSet()
sco := currScope
for sco != nil {
for _, table := range sco.tables {
tablesToKeep.MergeInPlace(table.getTableSet(b.org))
}
sco = sco.parent
}

subqDirectDeps.KeepOnly(tablesToKeep)
subqRecursiveDeps.KeepOnly(tablesToKeep)
b.recursive[subq] = subqRecursiveDeps
b.direct[subq] = subqDirectDeps
}

func (b *binder) createExtractedSubquery(cursor *sqlparser.Cursor, currScope *scope, subq *sqlparser.Subquery) (*sqlparser.ExtractedSubquery, error) {
if currScope.selectStmt == nil {
return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unable to bind subquery to select statement")
}

sq := &sqlparser.ExtractedSubquery{
Subquery: subq,
Original: subq,
OpCode: int(engine.PulloutValue),
}

switch par := cursor.Parent().(type) {
case *sqlparser.ComparisonExpr:
switch par.Operator {
case sqlparser.InOp:
sq.OpCode = int(engine.PulloutIn)
case sqlparser.NotInOp:
sq.OpCode = int(engine.PulloutNotIn)
}
subq, exp := GetSubqueryAndOtherSide(par)
sq.Original = &sqlparser.ComparisonExpr{
Left: exp,
Operator: par.Operator,
Right: subq,
}
sq.OtherSide = exp
case *sqlparser.ExistsExpr:
sq.OpCode = int(engine.PulloutExists)
sq.Original = par
}
return sq, nil
}

func (b *binder) resolveColumn(colName *sqlparser.ColName, current *scope) (deps dependency, err error) {
var thisDeps dependencies
for current != nil {
Expand Down Expand Up @@ -187,6 +203,7 @@ func makeAmbiguousError(colName *sqlparser.ColName, err error) error {
return err
}

// GetSubqueryAndOtherSide returns the subquery and other side of a comparison, iff one of the sides is a SubQuery
func GetSubqueryAndOtherSide(node *sqlparser.ComparisonExpr) (*sqlparser.Subquery, sqlparser.Expr) {
var subq *sqlparser.Subquery
var exp sqlparser.Expr
Expand Down