From c219c79923a80e4ac21c2fbd4b8ed49dc4633e3d Mon Sep 17 00:00:00 2001 From: time-and-fate <25057648+time-and-fate@users.noreply.github.com> Date: Thu, 3 Mar 2022 07:14:08 +0800 Subject: [PATCH 1/4] fix --- cmd/explaintest/r/explain_easy.result | 28 +++++++++---------- expression/testdata/expression_suite_out.json | 26 ++++++++--------- planner/core/integration_test.go | 2 +- .../core/testdata/integration_suite_out.json | 8 +++--- .../core/testdata/partition_pruner_out.json | 8 +++--- statistics/selectivity.go | 28 +++++++++++++++++++ util/ranger/testdata/ranger_suite_out.json | 4 +-- 7 files changed, 66 insertions(+), 38 deletions(-) diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index e3d082d5c7542..766b8760a2faf 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -433,10 +433,10 @@ IndexReader 10.00 root index:IndexRangeScan └─IndexRangeScan 10.00 cop[tikv] table:t, index:idx(a, b) range:[1,1], keep order:false, stats:pseudo explain format = 'brief' select * from t where a = 1 and a = 2; id estRows task access object operator info -TableDual 8000.00 root rows:0 +TableDual 0.00 root rows:0 explain format = 'brief' select * from t where b = 1 and b = 2; id estRows task access object operator info -TableDual 8000.00 root rows:0 +TableDual 0.00 root rows:0 explain format = 'brief' select * from t t1 join t t2 where t1.b = t2.b and t2.b is null; id estRows task access object operator info Projection 0.00 root test.t.a, test.t.b, test.t.a, test.t.b @@ -458,7 +458,7 @@ drop table if exists t; create table t(a bigint primary key); explain format = 'brief' select * from t where a = 1 and a = 2; id estRows task access object operator info -TableDual 8000.00 root rows:0 +TableDual 0.00 root rows:0 explain format = 'brief' select null or a > 1 from t; id estRows task access object operator info Projection 10000.00 root or(, gt(test.t.a, 1))->Column#2 @@ -506,17 +506,17 @@ PRIMARY KEY (`id`) explain format = 'brief' SELECT COUNT(1) FROM (SELECT COALESCE(b.region_name, '不详') region_name, SUM(a.registration_num) registration_num FROM (SELECT stat_date, show_date, region_id, 0 registration_num FROM test01 WHERE period = 1 AND stat_date >= 20191202 AND stat_date <= 20191202 UNION ALL SELECT stat_date, show_date, region_id, registration_num registration_num FROM test01 WHERE period = 1 AND stat_date >= 20191202 AND stat_date <= 20191202) a LEFT JOIN test02 b ON a.region_id = b.id WHERE registration_num > 0 AND a.stat_date >= '20191202' AND a.stat_date <= '20191202' GROUP BY a.stat_date , a.show_date , COALESCE(b.region_name, '不详') ) JLS; id estRows task access object operator info StreamAgg 1.00 root funcs:count(1)->Column#22 -└─HashAgg 8000.00 root group by:Column#32, Column#33, Column#34, funcs:count(1)->Column#31 - └─Projection 10000.01 root Column#14, Column#15, coalesce(test.test02.region_name, 不详)->Column#34 - └─HashJoin 10000.01 root left outer join, equal:[eq(Column#16, test.test02.id)] - ├─TableReader(Build) 10000.00 root data:TableFullScan - │ └─TableFullScan 10000.00 cop[tikv] table:b keep order:false, stats:pseudo - └─Union(Probe) 8000.01 root - ├─TableDual 8000.00 root rows:0 - └─Projection 0.01 root test.test01.stat_date, test.test01.show_date, test.test01.region_id - └─TableReader 0.01 root data:Selection - └─Selection 0.01 cop[tikv] eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), gt(cast(test.test01.registration_num, bigint(20) BINARY), 0), le(test.test01.stat_date, 20191202) - └─TableFullScan 10000.00 cop[tikv] table:test01 keep order:false, stats:pseudo +└─HashAgg 1.00 root group by:Column#32, Column#33, Column#34, funcs:count(1)->Column#31 + └─Projection 0.01 root Column#14, Column#15, coalesce(test.test02.region_name, 不详)->Column#34 + └─IndexJoin 0.01 root left outer join, inner:TableReader, outer key:Column#16, inner key:test.test02.id, equal cond:eq(Column#16, test.test02.id) + ├─Union(Build) 0.01 root + │ ├─TableDual 0.00 root rows:0 + │ └─Projection 0.01 root test.test01.stat_date, test.test01.show_date, test.test01.region_id + │ └─TableReader 0.01 root data:Selection + │ └─Selection 0.01 cop[tikv] eq(test.test01.period, 1), ge(test.test01.stat_date, 20191202), gt(cast(test.test01.registration_num, bigint(20) BINARY), 0), le(test.test01.stat_date, 20191202) + │ └─TableFullScan 10000.00 cop[tikv] table:test01 keep order:false, stats:pseudo + └─TableReader(Probe) 1.00 root data:TableRangeScan + └─TableRangeScan 1.00 cop[tikv] table:b range: decided by [Column#16], keep order:false, stats:pseudo drop table if exists t; create table t(a int, nb int not null, nc int not null); explain format = 'brief' select ifnull(a, 0) from t; diff --git a/expression/testdata/expression_suite_out.json b/expression/testdata/expression_suite_out.json index 8b89ee1e4dc81..7047b62ba1156 100644 --- a/expression/testdata/expression_suite_out.json +++ b/expression/testdata/expression_suite_out.json @@ -197,8 +197,8 @@ { "SQL": "explain format = 'brief' select * from t1 left join t2 on false", "Result": [ - "HashJoin 80000000.00 root CARTESIAN left outer join", - "├─TableDual(Build) 8000.00 root rows:0", + "HashJoin 10000.00 root CARTESIAN left outer join", + "├─TableDual(Build) 0.00 root rows:0", "└─TableReader(Probe) 10000.00 root data:TableFullScan", " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ] @@ -206,8 +206,8 @@ { "SQL": "explain format = 'brief' select * from t1 right join t2 on false", "Result": [ - "HashJoin 80000000.00 root CARTESIAN right outer join", - "├─TableDual(Build) 8000.00 root rows:0", + "HashJoin 10000.00 root CARTESIAN right outer join", + "├─TableDual(Build) 0.00 root rows:0", "└─TableReader(Probe) 10000.00 root data:TableFullScan", " └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo" ] @@ -215,8 +215,8 @@ { "SQL": "explain format = 'brief' select * from t1 left join t2 on t1.a = 1 and t1.a = 2", "Result": [ - "HashJoin 80000000.00 root CARTESIAN left outer join", - "├─TableDual(Build) 8000.00 root rows:0", + "HashJoin 10000.00 root CARTESIAN left outer join", + "├─TableDual(Build) 0.00 root rows:0", "└─TableReader(Probe) 10000.00 root data:TableFullScan", " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ] @@ -224,18 +224,18 @@ { "SQL": "explain format = 'brief' select * from t1 left join t2 on t1.a =1 where t1.a = 2", "Result": [ - "HashJoin 80000.00 root CARTESIAN left outer join", - "├─TableReader(Build) 10.00 root data:Selection", - "│ └─Selection 10.00 cop[tikv] eq(test.t1.a, 2)", - "│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo", - "└─TableDual(Probe) 8000.00 root rows:0" + "HashJoin 10.00 root CARTESIAN left outer join", + "├─TableDual(Build) 0.00 root rows:0", + "└─TableReader(Probe) 10.00 root data:Selection", + " └─Selection 10.00 cop[tikv] eq(test.t1.a, 2)", + " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ] }, { "SQL": "explain format = 'brief' select * from t1 left join t2 on t2.a = 1 and t2.a = 2", "Result": [ - "HashJoin 80000000.00 root CARTESIAN left outer join", - "├─TableDual(Build) 8000.00 root rows:0", + "HashJoin 10000.00 root CARTESIAN left outer join", + "├─TableDual(Build) 0.00 root rows:0", "└─TableReader(Probe) 10000.00 root data:TableFullScan", " └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo" ] diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 9a22c6ba2ce55..4ce7a75f1542c 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -3301,7 +3301,7 @@ func TestIssue14481(t *testing.T) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int default null, b int default null, c int default null)") plan := tk.MustQuery("explain format = 'brief' select * from t where a = 1 and a = 2") - plan.Check(testkit.Rows("TableDual 8000.00 root rows:0")) + plan.Check(testkit.Rows("TableDual 0.00 root rows:0")) tk.MustExec("drop table t") } diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index fe09f26006537..31bf95341ed42 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -5574,7 +5574,7 @@ "Plan": [ "IndexMerge 8.00 root ", "├─IndexRangeScan(Build) 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false", - "├─Selection(Build) 0.80 cop[tikv] 1", + "├─Selection(Build) 1.00 cop[tikv] 1", "│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false", "└─TableRowIDScan(Probe) 8.00 cop[tikv] table:t keep order:false" ], @@ -5585,7 +5585,7 @@ "Plan": [ "IndexMerge 8.00 root ", "├─IndexRangeScan(Build) 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false", - "├─Selection(Build) 0.80 cop[tikv] 1", + "├─Selection(Build) 1.00 cop[tikv] 1", "│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false", "└─TableRowIDScan(Probe) 8.00 cop[tikv] table:t keep order:false" ], @@ -5595,9 +5595,9 @@ "SQL": "desc format='brief' select /*+ use_index_merge(t) */ * from t where (a=1 and length(a)=1) or (b=1 and length(b)=1)", "Plan": [ "IndexMerge 8.00 root ", - "├─Selection(Build) 0.80 cop[tikv] 1", + "├─Selection(Build) 1.00 cop[tikv] 1", "│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:a(a) range:[1,1], keep order:false", - "├─Selection(Build) 0.80 cop[tikv] 1", + "├─Selection(Build) 1.00 cop[tikv] 1", "│ └─IndexRangeScan 1.00 cop[tikv] table:t, index:b(b) range:[1,1], keep order:false", "└─TableRowIDScan(Probe) 8.00 cop[tikv] table:t keep order:false" ], diff --git a/planner/core/testdata/partition_pruner_out.json b/planner/core/testdata/partition_pruner_out.json index 7977fc780453d..db74202a1459b 100644 --- a/planner/core/testdata/partition_pruner_out.json +++ b/planner/core/testdata/partition_pruner_out.json @@ -1485,10 +1485,10 @@ "SQL": "select * from t1 where a in (1,6) and (a=3 and b =3)", "Result": null, "Plan": [ - "TableDual 8000.00 root rows:0" + "TableDual 0.00 root rows:0" ], "IndexPlan": [ - "TableDual 8000.00 root rows:0" + "TableDual 0.00 root rows:0" ] }, { @@ -2488,10 +2488,10 @@ "SQL": "select * from t1 where a in (1,6) and (a=3 and b =3)", "Result": null, "Plan": [ - "TableDual 8000.00 root rows:0" + "TableDual 0.00 root rows:0" ], "IndexPlan": [ - "TableDual 8000.00 root rows:0" + "TableDual 0.00 root rows:0" ] }, { diff --git a/statistics/selectivity.go b/statistics/selectivity.go index 0f5f4792acc71..137efd431956e 100644 --- a/statistics/selectivity.go +++ b/statistics/selectivity.go @@ -317,6 +317,34 @@ func (coll *HistColl) Selectivity(ctx sessionctx.Context, exprs []expression.Exp } } + // Try to cover Constants + if mask > 0 { + for i, expr := range remainedExprs { + if mask&(1< 0 { diff --git a/util/ranger/testdata/ranger_suite_out.json b/util/ranger/testdata/ranger_suite_out.json index 8d78c7b1aa4ff..f9d4ede451753 100644 --- a/util/ranger/testdata/ranger_suite_out.json +++ b/util/ranger/testdata/ranger_suite_out.json @@ -123,7 +123,7 @@ { "SQL": "select * from t where a = 1 and (b = 1 or b = 2) and b = 3 and c > 1;", "Plan": [ - "TableDual_5 8000.00 root rows:0" + "TableDual_5 0.00 root rows:0" ], "Result": null }, @@ -169,7 +169,7 @@ { "SQL": "select * from t where a = 1 and b is null and b = 1 and c > 1;", "Plan": [ - "TableDual_5 8000.00 root rows:0" + "TableDual_5 0.00 root rows:0" ], "Result": null } From b6fb633dd0ad5641feeaf2cd965ffed02afcbe3b Mon Sep 17 00:00:00 2001 From: time-and-fate <25057648+time-and-fate@users.noreply.github.com> Date: Thu, 3 Mar 2022 09:46:47 +0800 Subject: [PATCH 2/4] fix --- executor/prepared_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/prepared_test.go b/executor/prepared_test.go index ab251b9e4853d..570d46ed49955 100644 --- a/executor/prepared_test.go +++ b/executor/prepared_test.go @@ -168,7 +168,7 @@ func TestPreparedNullParam(t *testing.T) { ps := []*util.ProcessInfo{tkProcess} tk.Session().SetSessionManager(&mockSessionManager1{PS: ps}) tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows( - "TableDual_5 8000.00 root rows:0")) + "TableDual_5 0.00 root rows:0")) } } From e29600bc7960e6fbc86f8f9540b202d53d6820be Mon Sep 17 00:00:00 2001 From: time-and-fate <25057648+time-and-fate@users.noreply.github.com> Date: Fri, 4 Mar 2022 14:32:18 +0800 Subject: [PATCH 3/4] try to delete unnecessary code --- statistics/selectivity.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/statistics/selectivity.go b/statistics/selectivity.go index 137efd431956e..5fc1f1dcc2b3b 100644 --- a/statistics/selectivity.go +++ b/statistics/selectivity.go @@ -382,24 +382,6 @@ func (coll *HistColl) Selectivity(ctx sessionctx.Context, exprs []expression.Exp if ok { continue } - // where {"0" / 0 / "false" / false / null} or A or B ... the '0' constant item should be ignored. - if c, ok := cond.(*expression.Constant); ok { - if !expression.MaybeOverOptimized4PlanCache(ctx, []expression.Expression{cond}) { - if c.Value.IsNull() { - // constant is null - continue - } - if isTrue, err := c.Value.ToBool(sc); err == nil { - if isTrue == 0 { - // constant == 0 - continue - } - // constant == 1 - selectivity = 1.0 - break - } - } - } var cnfItems []expression.Expression if scalar, ok := cond.(*expression.ScalarFunction); ok && scalar.FuncName.L == ast.LogicAnd { From 0e1513332adbf09b00f5ab6af631221d72f09368 Mon Sep 17 00:00:00 2001 From: time-and-fate <25057648+time-and-fate@users.noreply.github.com> Date: Fri, 4 Mar 2022 15:25:26 +0800 Subject: [PATCH 4/4] add some cases --- cmd/explaintest/r/imdbload.result | 9 +++++++++ cmd/explaintest/t/imdbload.test | 2 ++ 2 files changed, 11 insertions(+) diff --git a/cmd/explaintest/r/imdbload.result b/cmd/explaintest/r/imdbload.result index c627b86d86701..25c16db7c979d 100644 --- a/cmd/explaintest/r/imdbload.result +++ b/cmd/explaintest/r/imdbload.result @@ -279,6 +279,15 @@ id estRows task access object operator info TableReader_7 804024.75 root data:Selection_6 └─Selection_6 804024.75 cop[tikv] or(and(eq(imdbload.char_name.imdb_index, "I"), lt(imdbload.char_name.surname_pcode, "E436")), and(eq(imdbload.char_name.imdb_index, "L"), lt(imdbload.char_name.surname_pcode, "E436"))) └─TableFullScan_5 4314864.00 cop[tikv] table:char_name keep order:false +explain select * from char_name use index (itest2) where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436')); +id estRows task access object operator info +IndexLookUp_7 2010061.87 root +├─IndexRangeScan_5(Build) 2010061.87 cop[tikv] table:char_name, index:itest2(imdb_index, surname_pcode, name_pcode_nf) range:["I" -inf,"I" "E436"), ["L" -inf,"L" "E436"), keep order:false +└─TableRowIDScan_6(Probe) 2010061.87 cop[tikv] table:char_name keep order:false +trace plan target = 'estimation' select * from char_name where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436')); +CE_trace +[{"table_name":"char_name","type":"Column Stats-Range","expr":"((id >= -9223372036854775808 and id <= 9223372036854775807))","row_count":4314864},{"table_name":"char_name","type":"Index Stats-Range","expr":"((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436'))","row_count":2010061},{"table_name":"char_name","type":"Index Stats-Range","expr":"((surname_pcode < 'E436'))","row_count":1005030},{"table_name":"char_name","type":"Table Stats-Expression-CNF","expr":"`or`(`and`(`eq`(imdbload.char_name.imdb_index, 'I'), `lt`(imdbload.char_name.surname_pcode, 'E436')), `and`(`eq`(imdbload.char_name.imdb_index, 'L'), `lt`(imdbload.char_name.surname_pcode, 'E436')))","row_count":804024}] + explain select * from char_name where ((imdb_index = 'V') and (surname_pcode < 'L3416')); id estRows task access object operator info TableReader_7 1927106.39 root data:Selection_6 diff --git a/cmd/explaintest/t/imdbload.test b/cmd/explaintest/t/imdbload.test index 70e4af321dbdd..ba5f090d08e68 100644 --- a/cmd/explaintest/t/imdbload.test +++ b/cmd/explaintest/t/imdbload.test @@ -281,6 +281,8 @@ load stats 's/imdbload_stats/cast_info.json'; -- Actual row count: 1 explain select * from char_name where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436')); +explain select * from char_name use index (itest2) where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436')); +trace plan target = 'estimation' select * from char_name where ((imdb_index = 'I') and (surname_pcode < 'E436')) or ((imdb_index = 'L') and (surname_pcode < 'E436')); -- Actual row count: 0 explain select * from char_name where ((imdb_index = 'V') and (surname_pcode < 'L3416'));