diff --git a/distsql/request_builder.go b/distsql/request_builder.go index 3fbce7e02b51a..da1bd1ffc0e57 100644 --- a/distsql/request_builder.go +++ b/distsql/request_builder.go @@ -183,7 +183,6 @@ func TableRangesToKVRanges(tid int64, ranges []*ranger.Range, fb *statistics.Que if fb == nil || fb.Hist() == nil { return tableRangesToKVRangesWithoutSplit(tid, ranges) } - ranges = fb.Hist().SplitRange(ranges) krs := make([]kv.KeyRange, 0, len(ranges)) feedbackRanges := make([]*ranger.Range, 0, len(ranges)) for _, ran := range ranges { @@ -270,7 +269,7 @@ func IndexRangesToKVRanges(sc *stmtctx.StatementContext, tid, idxID int64, range feedbackRanges = append(feedbackRanges, &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)}, HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true}) } - feedbackRanges = fb.Hist().SplitRange(feedbackRanges) + feedbackRanges = fb.Hist().SplitRange(sc, feedbackRanges) krs := make([]kv.KeyRange, 0, len(feedbackRanges)) for _, ran := range feedbackRanges { low, high := ran.LowVal[0].GetBytes(), ran.HighVal[0].GetBytes() diff --git a/executor/table_reader.go b/executor/table_reader.go index 634b9bbf7a799..00bc3d87dc3a7 100644 --- a/executor/table_reader.go +++ b/executor/table_reader.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/ranger" - tipb "github.com/pingcap/tipb/go-tipb" + "github.com/pingcap/tipb/go-tipb" "golang.org/x/net/context" ) @@ -96,6 +96,10 @@ func (e *TableReaderExecutor) Open(ctx context.Context) error { } e.resultHandler = &tableResultHandler{} + // Split ranges here since the unsigned part and signed part will swap their position when encoding the range to kv ranges. + if e.feedback != nil && e.feedback.Hist() != nil { + e.ranges = e.feedback.Hist().SplitRange(e.ctx.GetSessionVars().StmtCtx, e.ranges) + } firstPartRanges, secondPartRanges := splitRanges(e.ranges, e.keepOrder, e.desc) firstResult, err := e.buildResp(ctx, firstPartRanges) if err != nil { diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 025c5b777a569..73da2788b562c 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -1130,14 +1130,14 @@ func (s *testPlanSuite) TestColumnPruning(c *C) { { sql: "select count(*) from t", ans: map[int][]string{ - 1: {"test.t._tidb_rowid"}, + 1: {"test.t.a"}, }, }, { sql: "select count(*) from t a join t b where a.a < 1", ans: map[int][]string{ 1: {"test.a.a"}, - 2: {"test.t._tidb_rowid"}, + 2: {"test.b.a"}, }, }, { @@ -1178,7 +1178,7 @@ func (s *testPlanSuite) TestColumnPruning(c *C) { { sql: "select exists (select count(*) from t where b = k.a) from t k", ans: map[int][]string{ - 1: {"test.t._tidb_rowid"}, + 1: {"test.k.a"}, }, }, { @@ -1236,8 +1236,8 @@ func (s *testPlanSuite) TestColumnPruning(c *C) { 3: {"test.t21.a"}, 5: {"t2.a"}, 8: {"test.t01.a"}, - 10: {"test.t._tidb_rowid"}, - 12: {"test.t._tidb_rowid"}, + 10: {"test.t3.a"}, + 12: {"test.t4.a"}, }, }, } diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index fccfb9b23ead8..51e99cf7c71f7 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/mysql" ) type columnPruner struct { @@ -165,7 +166,15 @@ func (p *LogicalUnionScan) PruneColumns(parentUsedCols []*expression.Column) { // PruneColumns implements LogicalPlan interface. func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column) { used := getUsedList(parentUsedCols, ds.schema) + var ( + handleCol *expression.Column + handleColInfo *model.ColumnInfo + ) for i := len(used) - 1; i >= 0; i-- { + if ds.tableInfo.PKIsHandle && mysql.HasPriKeyFlag(ds.Columns[i].Flag) { + handleCol = ds.schema.Columns[i] + handleColInfo = ds.Columns[i] + } if !used[i] { ds.schema.Columns = append(ds.schema.Columns[:i], ds.schema.Columns[i+1:]...) ds.Columns = append(ds.Columns[:i], ds.Columns[i+1:]...) @@ -179,8 +188,12 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column) { // For SQL like `select 1 from t`, tikv's response will be empty if no column is in schema. // So we'll force to push one if schema doesn't have any column. if ds.schema.Len() == 0 && !infoschema.IsMemoryDB(ds.DBName.L) { - ds.Columns = append(ds.Columns, model.NewExtraHandleColInfo()) - ds.schema.Append(ds.newExtraHandleSchemaCol()) + if handleCol == nil { + handleCol = ds.newExtraHandleSchemaCol() + handleColInfo = model.NewExtraHandleColInfo() + } + ds.Columns = append(ds.Columns, handleColInfo) + ds.schema.Append(handleCol) } } diff --git a/statistics/feedback.go b/statistics/feedback.go index 3423eced30d84..c0af492ec20cb 100644 --- a/statistics/feedback.go +++ b/statistics/feedback.go @@ -1141,8 +1141,8 @@ func dumpFeedbackForIndex(h *Handle, q *QueryFeedback, t *Table) error { func (q *QueryFeedback) dumpRangeFeedback(h *Handle, ran *ranger.Range, rangeCount float64) error { lowIsNull := ran.LowVal[0].IsNull() + sc := &stmtctx.StatementContext{TimeZone: time.UTC} if q.tp == indexType { - sc := &stmtctx.StatementContext{TimeZone: time.UTC} lower, err := codec.EncodeKey(sc, nil, ran.LowVal[0]) if err != nil { return errors.Trace(err) @@ -1164,7 +1164,7 @@ func (q *QueryFeedback) dumpRangeFeedback(h *Handle, ran *ranger.Range, rangeCou ran.HighVal[0] = getMaxValue(q.hist.tp) } } - ranges := q.hist.SplitRange([]*ranger.Range{ran}) + ranges := q.hist.SplitRange(sc, []*ranger.Range{ran}) counts := make([]float64, 0, len(ranges)) sum := 0.0 for i, r := range ranges { diff --git a/statistics/histogram.go b/statistics/histogram.go index 940868a74fe1b..5be059fe3f622 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -503,12 +503,20 @@ func (hg *Histogram) getIncreaseFactor(totalCount int64) float64 { // validRange checks if the range is valid, it is used by `SplitRange` to remove the invalid range, // the possible types of range are index key range and handle key range. -func validRange(ran *ranger.Range) bool { +func validRange(sc *stmtctx.StatementContext, ran *ranger.Range) bool { var low, high []byte if ran.LowVal[0].Kind() == types.KindBytes { low, high = ran.LowVal[0].GetBytes(), ran.HighVal[0].GetBytes() } else { - low, high = codec.EncodeInt(nil, ran.LowVal[0].GetInt64()), codec.EncodeInt(nil, ran.HighVal[0].GetInt64()) + var err error + low, err = codec.EncodeKey(sc, nil, ran.LowVal[0]) + if err != nil { + return false + } + high, err = codec.EncodeKey(sc, nil, ran.HighVal[0]) + if err != nil { + return false + } } if ran.LowExclude { low = kv.Key(low).PrefixNext() @@ -522,7 +530,7 @@ func validRange(ran *ranger.Range) bool { // SplitRange splits the range according to the histogram upper bound. Note that we treat last bucket's upper bound // as inf, so all the split ranges will totally fall in one of the (-inf, u(0)], (u(0), u(1)],...(u(n-3), u(n-2)], // (u(n-2), +inf), where n is the number of buckets, u(i) is the i-th bucket's upper bound. -func (hg *Histogram) SplitRange(ranges []*ranger.Range) []*ranger.Range { +func (hg *Histogram) SplitRange(sc *stmtctx.StatementContext, ranges []*ranger.Range) []*ranger.Range { split := make([]*ranger.Range, 0, len(ranges)) for len(ranges) > 0 { // Find the last bound that greater or equal to the LowVal. @@ -566,7 +574,7 @@ func (hg *Histogram) SplitRange(ranges []*ranger.Range) []*ranger.Range { HighExclude: false}) ranges[0].LowVal[0] = upper ranges[0].LowExclude = true - if !validRange(ranges[0]) { + if !validRange(sc, ranges[0]) { ranges = ranges[1:] } } diff --git a/statistics/update_test.go b/statistics/update_test.go index 1977993a97757..816aef606d7f0 100644 --- a/statistics/update_test.go +++ b/statistics/update_test.go @@ -604,7 +604,7 @@ func (s *testStatsUpdateSuite) TestSplitRange(c *C) { HighExclude: t.exclude[i+1], }) } - ranges = h.SplitRange(ranges) + ranges = h.SplitRange(nil, ranges) var ranStrs []string for _, ran := range ranges { ranStrs = append(ranStrs, ran.String()) @@ -1344,7 +1344,7 @@ func (s *testStatsUpdateSuite) TestUnsignedFeedbackRanges(c *C) { statistics.FeedbackProbability = 1 testKit.MustExec("use test") - testKit.MustExec("create table t (a tinyint unsigned, primary key(a))") + testKit.MustExec("create table t (a bigint unsigned, primary key(a))") for i := 0; i < 20; i++ { testKit.MustExec(fmt.Sprintf("insert into t values (%d)", i)) } @@ -1371,7 +1371,7 @@ func (s *testStatsUpdateSuite) TestUnsignedFeedbackRanges(c *C) { hist: "column:1 ndv:30 totColSize:0\n" + "num: 8 lower_bound: 0 upper_bound: 7 repeats: 0\n" + "num: 8 lower_bound: 8 upper_bound: 15 repeats: 0\n" + - "num: 14 lower_bound: 16 upper_bound: 255 repeats: 0", + "num: 14 lower_bound: 16 upper_bound: 18446744073709551615 repeats: 0", }, } is := s.do.InfoSchema()