Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

opt: fix computation of apply join outer columns #35171

Merged
merged 1 commit into from
Feb 26, 2019
Merged

opt: fix computation of apply join outer columns #35171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions pkg/sql/opt/memo/logical_props_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,13 +402,6 @@ func (b *logicalPropsBuilder) buildJoinProps(join RelExpr, rel *props.Relational
// bound by the input columns.
inputCols := h.leftProps.OutputCols.Union(h.rightProps.OutputCols)
rel.OuterCols.DifferenceWith(inputCols)
if opt.IsJoinApplyOp(join) {
// Outer columns of right side of apply join can be bound by output columns
// of left side of apply join. Since this is apply join, there is always a
// right input.
rightOuterCols := join.Child(1).(RelExpr).Relational().OuterCols
rel.OuterCols.DifferenceWith(rightOuterCols)
}

// Functional Dependencies
// -----------------------
Expand Down
16 changes: 12 additions & 4 deletions pkg/sql/opt/norm/reject_nulls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,20 +78,28 @@ func DeriveRejectNullCols(in memo.RelExpr) opt.ColSet {
switch in.Op() {
case opt.InnerJoinOp, opt.InnerJoinApplyOp:
// Pass through null-rejecting columns from both inputs.
relProps.Rule.RejectNullCols = DeriveRejectNullCols(in.Child(0).(memo.RelExpr))
relProps.Rule.RejectNullCols.UnionWith(DeriveRejectNullCols(in.Child(1).(memo.RelExpr)))
if in.Child(0).(memo.RelExpr).Relational().OuterCols.Empty() {
relProps.Rule.RejectNullCols.UnionWith(DeriveRejectNullCols(in.Child(0).(memo.RelExpr)))
}
if in.Child(1).(memo.RelExpr).Relational().OuterCols.Empty() {
relProps.Rule.RejectNullCols.UnionWith(DeriveRejectNullCols(in.Child(1).(memo.RelExpr)))
}

case opt.LeftJoinOp, opt.LeftJoinApplyOp:
// Pass through null-rejection columns from left input, and request null-
// rejection on right columns.
relProps.Rule.RejectNullCols = DeriveRejectNullCols(in.Child(0).(memo.RelExpr))
if in.Child(0).(memo.RelExpr).Relational().OuterCols.Empty() {
relProps.Rule.RejectNullCols.UnionWith(DeriveRejectNullCols(in.Child(0).(memo.RelExpr)))
}
relProps.Rule.RejectNullCols.UnionWith(in.Child(1).(memo.RelExpr).Relational().OutputCols)

case opt.RightJoinOp, opt.RightJoinApplyOp:
// Pass through null-rejection columns from right input, and request null-
// rejection on left columns.
relProps.Rule.RejectNullCols = in.Child(0).(memo.RelExpr).Relational().OutputCols
relProps.Rule.RejectNullCols.UnionWith(DeriveRejectNullCols(in.Child(1).(memo.RelExpr)))
if in.Child(1).(memo.RelExpr).Relational().OuterCols.Empty() {
relProps.Rule.RejectNullCols.UnionWith(DeriveRejectNullCols(in.Child(1).(memo.RelExpr)))
}

case opt.FullJoinOp, opt.FullJoinApplyOp:
// Request null-rejection on all output columns.
Expand Down
82 changes: 47 additions & 35 deletions pkg/sql/opt/norm/testdata/rules/decorrelate
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4327,7 +4327,8 @@ limit
│ └── const: 0 [type=int]
└── const: 10 [type=int]

opt expect=TryDecorrelateProjectSet
# TODO(justin): figure out how to get this to decorrelate again.
opt
SELECT * FROM articles, xy WHERE EXISTS(
SELECT * FROM ROWS FROM (generate_series(x, id), length(title), upper(title), unnest(tag_list))
)
Expand All @@ -4336,51 +4337,62 @@ project
├── columns: id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp) x:10(int!null) y:11(int)
├── side-effects
├── key: (1,10)
├── fd: (1)-->(2-9), (10)-->(11), (1,10)-->(2-9,11)
├── fd: (1)-->(2-9), (1,10)-->(2-9,11)
└── select
├── columns: id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp) x:10(int!null) y:11(int) true_agg:17(bool!null)
├── side-effects
├── key: (1,10)
├── fd: (1)-->(2-9), (10)-->(11), (1,10)-->(2-9,11,17)
├── fd: (1)-->(2-9), (1,10)-->(2-9,11,17)
├── group-by
│ ├── columns: id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp) x:10(int!null) y:11(int) true_agg:17(bool)
│ ├── grouping columns: id:1(int!null) x:10(int!null)
│ ├── side-effects
│ ├── key: (1,10)
│ ├── fd: (1)-->(2-9), (10)-->(11), (1,10)-->(2-9,11,17)
│ ├── project
│ │ ├── columns: true:16(bool!null) id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp) x:10(int!null) y:11(int)
│ ├── fd: (1)-->(2-9), (1,10)-->(2-9,11,17)
│ ├── inner-join-apply
│ │ ├── columns: id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp) x:10(int!null) y:11(int) true:16(bool)
│ │ ├── side-effects
│ │ ├── fd: ()-->(16), (1)-->(2-9), (10)-->(11)
│ │ ├── project-set
│ │ │ ├── columns: id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp) x:10(int!null) y:11(int) generate_series:12(int) length:13(int) upper:14(string) unnest:15(string)
│ │ ├── fd: (1)-->(2-9), (1,10)-->(11)
│ │ ├── scan articles
│ │ │ ├── columns: id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp)
│ │ │ ├── key: (1)
│ │ │ └── fd: (1)-->(2-9)
│ │ ├── left-join-apply
│ │ │ ├── columns: x:10(int!null) y:11(int) true:16(bool)
│ │ │ ├── outer: (1,4,6)
│ │ │ ├── side-effects
│ │ │ ├── fd: (1)-->(2-9), (10)-->(11)
│ │ │ ├── inner-join
│ │ │ │ ├── columns: id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp) x:10(int!null) y:11(int)
│ │ │ │ ├── key: (1,10)
│ │ │ │ ├── fd: (1)-->(2-9), (10)-->(11)
│ │ │ │ ├── scan articles
│ │ │ │ │ ├── columns: id:1(int!null) body:2(string) description:3(string) title:4(string) slug:5(string) tag_list:6(string[]) user_id:7(string) created_at:8(timestamp) updated_at:9(timestamp)
│ │ │ │ │ ├── key: (1)
│ │ │ │ │ └── fd: (1)-->(2-9)
│ │ │ │ ├── scan xy
│ │ │ │ │ ├── columns: x:10(int!null) y:11(int)
│ │ │ │ │ ├── key: (10)
│ │ │ │ │ └── fd: (10)-->(11)
│ │ │ │ └── filters (true)
│ │ │ └── zip
│ │ │ ├── function: generate_series [type=int, outer=(1,10), side-effects]
│ │ │ │ ├── variable: x [type=int]
│ │ │ │ └── variable: id [type=int]
│ │ │ ├── function: length [type=int, outer=(4)]
│ │ │ │ └── variable: title [type=string]
│ │ │ ├── function: upper [type=string, outer=(4)]
│ │ │ │ └── variable: title [type=string]
│ │ │ └── function: unnest [type=string, outer=(6), side-effects]
│ │ │ └── variable: tag_list [type=string[]]
│ │ └── projections
│ │ └── true [type=bool]
│ │ │ ├── fd: (10)-->(11)
│ │ │ ├── scan xy
│ │ │ │ ├── columns: x:10(int!null) y:11(int)
│ │ │ │ ├── key: (10)
│ │ │ │ └── fd: (10)-->(11)
│ │ │ ├── project
│ │ │ │ ├── columns: true:16(bool!null)
│ │ │ │ ├── outer: (1,4,6,10)
│ │ │ │ ├── side-effects
│ │ │ │ ├── fd: ()-->(16)
│ │ │ │ ├── project-set
│ │ │ │ │ ├── columns: generate_series:12(int) length:13(int) upper:14(string) unnest:15(string)
│ │ │ │ │ ├── outer: (1,4,6,10)
│ │ │ │ │ ├── side-effects
│ │ │ │ │ ├── values
│ │ │ │ │ │ ├── cardinality: [1 - 1]
│ │ │ │ │ │ ├── key: ()
│ │ │ │ │ │ └── tuple [type=tuple]
│ │ │ │ │ └── zip
│ │ │ │ │ ├── function: generate_series [type=int, outer=(1,10), side-effects]
│ │ │ │ │ │ ├── variable: x [type=int]
│ │ │ │ │ │ └── variable: id [type=int]
│ │ │ │ │ ├── function: length [type=int, outer=(4)]
│ │ │ │ │ │ └── variable: title [type=string]
│ │ │ │ │ ├── function: upper [type=string, outer=(4)]
│ │ │ │ │ │ └── variable: title [type=string]
│ │ │ │ │ └── function: unnest [type=string, outer=(6), side-effects]
│ │ │ │ │ └── variable: tag_list [type=string[]]
│ │ │ │ └── projections
│ │ │ │ └── true [type=bool]
│ │ │ └── filters (true)
│ │ └── filters (true)
│ └── aggregations
│ ├── const-not-null-agg [type=bool, outer=(16)]
│ │ └── variable: true [type=bool]
Expand Down
173 changes: 129 additions & 44 deletions pkg/sql/opt/norm/testdata/rules/reject_nulls
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,51 +419,64 @@ project


# Ignore ConstAgg aggregates on other columns.
opt expect=RejectNullsGroupBy
SELECT * FROM a WHERE (SELECT sum(x) FROM xy WHERE x=k)>i
exprnorm expect=RejectNullsGroupBy
(Root
(Select
(ScalarGroupBy
(LeftJoin
(Scan [ (Table "xy") (Cols "x,y") ])
(Scan [ (Table "uv") (Cols "u,v") ])
[ ]
[ ]
)
[
(AggregationsItem (Sum (Var "v")) (NewColumn "sum" "int"))
(AggregationsItem (ConstAgg (Var "u")) (NewColumn "const" "int"))
]
[ ]
)
[ (Eq (Var "sum") (Const 10)) ]
)
(Presentation "u,v")
(NoOrdering)
)
----
project
├── columns: k:1(int!null) i:2(int!null) f:3(float) s:4(string)
├── key: (1)
├── fd: (1)-->(2-4)
└── select
├── columns: k:1(int!null) i:2(int!null) f:3(float) s:4(string) sum:7(decimal!null)
├── key: (1)
├── fd: (1)-->(2-4,7)
├── group-by
│ ├── columns: k:1(int!null) i:2(int) f:3(float) s:4(string) sum:7(decimal)
│ ├── grouping columns: k:1(int!null)
│ ├── internal-ordering: +(1|5)
│ ├── key: (1)
│ ├── fd: (1)-->(2-4,7)
│ ├── inner-join (merge)
│ │ ├── columns: k:1(int!null) i:2(int) f:3(float) s:4(string) x:5(int!null)
│ │ ├── left ordering: +1
│ │ ├── right ordering: +5
│ │ ├── key: (5)
│ │ ├── fd: (1)-->(2-4), (1)==(5), (5)==(1)
│ │ ├── ordering: +(1|5) [actual: +1]
│ │ ├── scan a
│ │ │ ├── columns: k:1(int!null) i:2(int) f:3(float) s:4(string)
│ │ │ ├── key: (1)
│ │ │ ├── fd: (1)-->(2-4)
│ │ │ └── ordering: +1
│ │ ├── scan xy
│ │ │ ├── columns: x:5(int!null)
│ │ │ ├── key: (5)
│ │ │ └── ordering: +5
│ │ └── filters (true)
│ └── aggregations
│ ├── sum [type=decimal, outer=(5)]
│ │ └── variable: x [type=int]
│ ├── const-agg [type=int, outer=(2)]
│ │ └── variable: i [type=int]
│ ├── const-agg [type=float, outer=(3)]
│ │ └── variable: f [type=float]
│ └── const-agg [type=string, outer=(4)]
│ └── variable: s [type=string]
└── filters
└── i < sum [type=bool, outer=(2,7), constraints=(/2: (/NULL - ]; /7: (/NULL - ])]
select
├── columns: u:3(int) v:4(int) [hidden: sum:5(int!null) const:6(int)]
├── cardinality: [0 - 1]
├── key: ()
├── fd: ()-->(5,6)
├── scalar-group-by
│ ├── columns: sum:5(int) const:6(int)
│ ├── cardinality: [1 - 1]
│ ├── key: ()
│ ├── fd: ()-->(5,6)
│ ├── inner-join
│ │ ├── columns: x:1(int!null) y:2(int) u:3(int!null) v:4(int!null)
│ │ ├── key: (1,3)
│ │ ├── fd: (1)-->(2), (3)-->(4)
│ │ ├── scan t.public.xy
│ │ │ ├── columns: x:1(int!null) y:2(int)
│ │ │ ├── key: (1)
│ │ │ └── fd: (1)-->(2)
│ │ ├── select
│ │ │ ├── columns: u:3(int!null) v:4(int!null)
│ │ │ ├── key: (3)
│ │ │ ├── fd: (3)-->(4)
│ │ │ ├── scan t.public.uv
│ │ │ │ ├── columns: u:3(int!null) v:4(int)
│ │ │ │ ├── key: (3)
│ │ │ │ └── fd: (3)-->(4)
│ │ │ └── filters
│ │ │ └── v IS NOT NULL [type=bool, outer=(4), constraints=(/4: (/NULL - ]; tight)]
│ │ └── filters (true)
│ └── aggregations
│ ├── sum [type=decimal, outer=(4)]
│ │ └── variable: v [type=int]
│ └── const-agg [type=int, outer=(3)]
│ └── variable: u [type=int]
└── filters
└── sum = 10 [type=bool, outer=(5), constraints=(/5: [/10 - /10]; tight), fd=()-->(5)]

# Don't reject nulls when multiple columns are used.
opt expect-not=RejectNullsGroupBy
Expand Down Expand Up @@ -599,3 +612,75 @@ project
│ └── variable: exists [type=bool, outer=(16), constraints=(/16: [/true - /true]; tight), fd=()-->(16)]
└── projections
└── const: 1 [type=int]

# Regression test: the not-null filter can't make it all the way down to the
# join that requested it, so ensure that we don't endlessly try to introduce
# them.
exprnorm
(Select
(ScalarGroupBy
(InnerJoinApply
(Scan [ (Table "xy") (Cols "x,y") ])
(LeftJoinApply
(Scan [ (Table "uv") (Cols "u,v") ])
(Select
(Values
[ (Tuple [ (Plus (Var "x") (Var "u")) ] "tuple{int}" ) ]
[ (Cols [ (NewColumn "z" "int") ]) ]
)
[ (Eq (Var "x") (Const 3)) ]
)
[ ]
[ ]
)
[ ]
[ ]
)
[ (AggregationsItem (Sum (Var "z")) (NewColumn "sum" "int")) ]
[ ]
)
[ (Eq (Var "sum") (Const 10)) ]
)
----
select
├── columns: sum:6(int!null)
├── cardinality: [0 - 1]
├── key: ()
├── fd: ()-->(6)
├── scalar-group-by
│ ├── columns: sum:6(int)
│ ├── cardinality: [1 - 1]
│ ├── key: ()
│ ├── fd: ()-->(6)
│ ├── inner-join-apply
│ │ ├── columns: x:1(int!null) y:2(int) u:3(int!null) v:4(int) z:5(int)
│ │ ├── key: (1,3)
│ │ ├── fd: (1)-->(2), (1,3)-->(4,5)
│ │ ├── scan t.public.xy
│ │ │ ├── columns: x:1(int!null) y:2(int)
│ │ │ ├── key: (1)
│ │ │ └── fd: (1)-->(2)
│ │ ├── left-join-apply
│ │ │ ├── columns: u:3(int!null) v:4(int) z:5(int)
│ │ │ ├── outer: (1)
│ │ │ ├── key: (3)
│ │ │ ├── fd: (3)-->(4,5)
│ │ │ ├── scan t.public.uv
│ │ │ │ ├── columns: u:3(int!null) v:4(int)
│ │ │ │ ├── key: (3)
│ │ │ │ └── fd: (3)-->(4)
│ │ │ ├── values
│ │ │ │ ├── columns: z:5(int)
│ │ │ │ ├── outer: (1,3)
│ │ │ │ ├── cardinality: [1 - 1]
│ │ │ │ ├── key: ()
│ │ │ │ ├── fd: ()-->(5)
│ │ │ │ └── (x + u,) [type=tuple{int}]
│ │ │ └── filters
│ │ │ └── x = 3 [type=bool, outer=(1), constraints=(/1: [/3 - /3]; tight), fd=()-->(1)]
│ │ └── filters (true)
│ └── aggregations
│ └── sum [type=decimal, outer=(5)]
│ └── variable: z [type=int]
└── filters
└── sum = 10 [type=bool, outer=(6), constraints=(/6: [/10 - /10]; tight), fd=()-->(6)]
10 changes: 10 additions & 0 deletions pkg/sql/opt/optgen/exprgen/custom_funcs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,16 @@ func (c *customFuncs) ProjectionItem(
}
}

// AggregationsItem creates an AggregationsItem. A list of such items can be used as
// an AggregationsExpr.
func (c *customFuncs) AggregationsItem(agg opt.ScalarExpr, col opt.ColumnID) memo.AggregationsItem {
return memo.AggregationsItem{
Agg: agg,
ColPrivate: memo.ColPrivate{Col: col},
Typ: c.mem.Metadata().ColumnMeta(col).Type,
}
}

// Ordering parses a string like "+a,-b" into an Ordering.
func (c *customFuncs) Ordering(str string) opt.Ordering {
defer func() {
Expand Down
28 changes: 28 additions & 0 deletions pkg/sql/opt/testutils/opttester/opt_tester.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,14 @@ func (ot *OptTester) RunCommand(tb testing.TB, d *datadriven.TestData) string {
ot.postProcess(tb, d, e)
return memo.FormatExpr(e, ot.Flags.ExprFormat)

case "exprnorm":
e, err := ot.ExprNorm()
if err != nil {
d.Fatalf(tb, "%v", err)
}
ot.postProcess(tb, d, e)
return memo.FormatExpr(e, ot.Flags.ExprFormat)

default:
d.Fatalf(tb, "unsupported command: %s", d.Cmd)
return ""
Expand Down Expand Up @@ -569,6 +577,26 @@ func (ot *OptTester) Expr() (opt.Expr, error) {
return exprgen.Build(ot.catalog, &f, ot.sql)
}

// ExprNorm parses the input directly into an expression and runs
// normalization; see exprgen.Build.
func (ot *OptTester) ExprNorm() (opt.Expr, error) {
var f norm.Factory
f.Init(&ot.evalCtx)

f.NotifyOnMatchedRule(func(ruleName opt.RuleName) bool {
// exprgen.Build doesn't run optimization, so we don't need to explicitly
// disallow exploration rules here.

if ot.Flags.DisableRules.Contains(int(ruleName)) {
return false
}
ot.seenRules.Add(int(ruleName))
return true
})

return exprgen.Build(ot.catalog, &f, ot.sql)
}

// RuleStats performs the optimization and returns statistics about how many
// rules were applied.
func (ot *OptTester) RuleStats() (string, error) {
Expand Down