Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: replace compareDatum by compare, range part #30710

Merged
merged 3 commits into from
Dec 16, 2021
Merged

*: replace compareDatum by compare, range part #30710

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2fd3b08
done
wjhuang2016 Dec 13, 2021
a659379
Merge branch 'master' into replace_datumcompare_13
wjhuang2016 Dec 16, 2021
39e0fda
Merge branch 'master' into replace_datumcompare_13
wjhuang2016 Dec 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions distsql/request_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/ranger"
"github.com/pingcap/tipb/go-tipb"
Expand Down Expand Up @@ -382,7 +383,7 @@ func tablesRangesToKVRanges(tids []int64, ranges []*ranger.Range, fb *statistics
// since we need to guarantee each range falls inside the exactly one bucket, `PrefixNext` will make the
// high value greater than upper bound, so we store the range here.
r := &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}}
HighVal: []types.Datum{types.NewBytesDatum(high)}, Collators: collate.GetBinaryCollatorSlice(1)}
feedbackRanges = append(feedbackRanges, r)

if !ran.HighExclude {
Expand Down Expand Up @@ -465,13 +466,15 @@ func SplitRangesAcrossInt64Boundary(ranges []*ranger.Range, keepOrder bool, desc
LowVal: ranges[idx].LowVal,
LowExclude: ranges[idx].LowExclude,
HighVal: []types.Datum{types.NewUintDatum(math.MaxInt64)},
Collators: ranges[idx].Collators,
})
}
if !(ranges[idx].HighVal[0].GetUint64() == math.MaxInt64+1 && ranges[idx].HighExclude) {
unsignedRanges = append(unsignedRanges, &ranger.Range{
LowVal: []types.Datum{types.NewUintDatum(math.MaxInt64 + 1)},
HighVal: ranges[idx].HighVal,
HighExclude: ranges[idx].HighExclude,
Collators: ranges[idx].Collators,
})
}
if idx < len(ranges) {
Expand Down Expand Up @@ -585,7 +588,7 @@ func indexRangesToKVRangesForTablesWithInterruptSignal(sc *stmtctx.StatementCont
return nil, err
}
feedbackRanges = append(feedbackRanges, &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true})
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true, Collators: collate.GetBinaryCollatorSlice(1)})
}
feedbackRanges, ok := fb.Hist.SplitRange(sc, feedbackRanges, true)
if !ok {
Expand Down Expand Up @@ -624,7 +627,7 @@ func CommonHandleRangesToKVRanges(sc *stmtctx.StatementContext, tids []int64, ra
return nil, err
}
rans = append(rans, &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true})
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true, Collators: collate.GetBinaryCollatorSlice(1)})
}
krs := make([]kv.KeyRange, 0, len(rans))
for _, ran := range rans {
Expand Down
47 changes: 35 additions & 12 deletions distsql/request_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/ranger"
"github.com/pingcap/tipb/go-tipb"
Expand Down Expand Up @@ -72,29 +73,34 @@ func TestTableHandlesToKVRanges(t *testing.T) {
func TestTableRangesToKVRanges(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(2)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowExclude: true,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(4)},
HighVal: []types.Datum{types.NewIntDatum(19)},
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(19)},
HighVal: []types.Datum{types.NewIntDatum(32)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(34)},
HighVal: []types.Datum{types.NewIntDatum(34)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
}

Expand Down Expand Up @@ -129,29 +135,34 @@ func TestTableRangesToKVRanges(t *testing.T) {
func TestIndexRangesToKVRanges(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(2)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowExclude: true,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(4)},
HighVal: []types.Datum{types.NewIntDatum(19)},
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(19)},
HighVal: []types.Datum{types.NewIntDatum(32)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(34)},
HighVal: []types.Datum{types.NewIntDatum(34)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
}

Expand Down Expand Up @@ -188,29 +199,34 @@ func TestIndexRangesToKVRanges(t *testing.T) {
func TestRequestBuilder1(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(2)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowExclude: true,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(4)},
HighVal: []types.Datum{types.NewIntDatum(19)},
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(19)},
HighVal: []types.Datum{types.NewIntDatum(32)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(34)},
HighVal: []types.Datum{types.NewIntDatum(34)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
}

Expand Down Expand Up @@ -265,29 +281,34 @@ func TestRequestBuilder1(t *testing.T) {
func TestRequestBuilder2(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(2)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowExclude: true,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(4)},
HighVal: []types.Datum{types.NewIntDatum(19)},
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(19)},
HighVal: []types.Datum{types.NewIntDatum(32)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
{
LowVal: []types.Datum{types.NewIntDatum(34)},
HighVal: []types.Datum{types.NewIntDatum(34)},
LowExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
},
}

Expand Down Expand Up @@ -573,8 +594,9 @@ func TestRequestBuilder8(t *testing.T) {
func TestTableRangesToKVRangesWithFbs(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(4)},
Collators: collate.GetBinaryCollatorSlice(1),
},
}
fb := newTestFb()
Expand All @@ -594,8 +616,9 @@ func TestTableRangesToKVRangesWithFbs(t *testing.T) {
func TestIndexRangesToKVRangesWithFbs(t *testing.T) {
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(4)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(4)},
Collators: collate.GetBinaryCollatorSlice(1),
},
}
fb := newTestFb()
Expand Down
4 changes: 2 additions & 2 deletions executor/analyze.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2117,7 +2117,7 @@ func analyzeIndexIncremental(idxExec *analyzeIndexIncrementalExec) *statistics.A
if err != nil {
return &statistics.AnalyzeResults{Err: err, Job: idxExec.job}
}
ran := ranger.Range{LowVal: values, HighVal: []types.Datum{types.MaxValueDatum()}}
ran := ranger.Range{LowVal: values, HighVal: []types.Datum{types.MaxValueDatum()}, Collators: collate.GetBinaryCollatorSlice(1)}
hist, cms, fms, topN, err := idxExec.buildStats([]*ranger.Range{&ran}, false)
if err != nil {
return &statistics.AnalyzeResults{Err: err, Job: idxExec.job}
Expand Down Expand Up @@ -2172,7 +2172,7 @@ func analyzePKIncremental(colExec *analyzePKIncrementalExec) *statistics.Analyze
maxVal = types.NewIntDatum(math.MaxInt64)
}
startPos := *colExec.oldHist.GetUpper(colExec.oldHist.Len() - 1)
ran := ranger.Range{LowVal: []types.Datum{startPos}, LowExclude: true, HighVal: []types.Datum{maxVal}}
ran := ranger.Range{LowVal: []types.Datum{startPos}, LowExclude: true, HighVal: []types.Datum{maxVal}, Collators: collate.GetBinaryCollatorSlice(1)}
hists, _, _, _, _, err := colExec.buildStats([]*ranger.Range{&ran}, false)
if err != nil {
return &statistics.AnalyzeResults{Err: err, Job: colExec.job}
Expand Down
2 changes: 2 additions & 0 deletions executor/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4093,6 +4093,7 @@ func buildRangesForIndexJoin(ctx sessionctx.Context, lookUpContents []*indexJoin
ran.HighVal[lastPos] = nextColRan.HighVal[0]
ran.LowExclude = nextColRan.LowExclude
ran.HighExclude = nextColRan.HighExclude
ran.Collators = nextColRan.Collators
tmpDatumRanges = append(tmpDatumRanges, ran.Clone())
}
}
Expand Down Expand Up @@ -4144,6 +4145,7 @@ func buildKvRangesForIndexJoin(ctx sessionctx.Context, tableID, indexID int64, l
ran.HighVal[lastPos] = nextColRan.HighVal[0]
ran.LowExclude = nextColRan.LowExclude
ran.HighExclude = nextColRan.HighExclude
ran.Collators = nextColRan.Collators
tmpDatumRanges = append(tmpDatumRanges, ran.Clone())
}
}
Expand Down
3 changes: 2 additions & 1 deletion executor/executor_pkg_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util"
"github.com/pingcap/tidb/util/chunk"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tidb/util/mock"
"github.com/pingcap/tidb/util/ranger"
Expand Down Expand Up @@ -212,7 +213,7 @@ func generateIndexRange(vals ...int64) *ranger.Range {
lowDatums := generateDatumSlice(vals...)
highDatums := make([]types.Datum, len(vals))
copy(highDatums, lowDatums)
return &ranger.Range{LowVal: lowDatums, HighVal: highDatums}
return &ranger.Range{LowVal: lowDatums, HighVal: highDatums, Collators: collate.GetBinaryCollatorSlice(len(lowDatums))}
}

func generateDatumSlice(vals ...int64) []types.Datum {
Expand Down
13 changes: 9 additions & 4 deletions planner/core/exhaust_physical_plans.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1538,6 +1538,7 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
HighVal: make([]types.Datum, pointLength, pointLength+1),
LowExclude: colRan.LowExclude,
HighExclude: colRan.HighExclude,
Collators: make([]collate.Collator, pointLength, pointLength+1),
}
ran.LowVal = append(ran.LowVal, colRan.LowVal[0])
ran.HighVal = append(ran.HighVal, colRan.HighVal[0])
Expand All @@ -1546,13 +1547,15 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
} else if haveExtraCol {
// Reserve a position for the last col.
ranges = append(ranges, &ranger.Range{
LowVal: make([]types.Datum, pointLength+1),
HighVal: make([]types.Datum, pointLength+1),
LowVal: make([]types.Datum, pointLength+1),
HighVal: make([]types.Datum, pointLength+1),
Collators: make([]collate.Collator, pointLength+1),
})
} else {
ranges = append(ranges, &ranger.Range{
LowVal: make([]types.Datum, pointLength),
HighVal: make([]types.Datum, pointLength),
LowVal: make([]types.Datum, pointLength),
HighVal: make([]types.Datum, pointLength),
Collators: make([]collate.Collator, pointLength),
})
}
sc := ijHelper.join.ctx.GetSessionVars().StmtCtx
Expand All @@ -1575,6 +1578,7 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
for _, ran := range ranges {
ran.LowVal[i] = oneColumnRan[0].LowVal[0]
ran.HighVal[i] = oneColumnRan[0].HighVal[0]
ran.Collators[i] = oneColumnRan[0].Collators[0]
}
curRangeLen := len(ranges)
for ranIdx := 1; ranIdx < len(oneColumnRan); ranIdx++ {
Expand All @@ -1583,6 +1587,7 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
newRange := ranges[oldRangeIdx].Clone()
newRange.LowVal[i] = oneColumnRan[ranIdx].LowVal[0]
newRange.HighVal[i] = oneColumnRan[ranIdx].HighVal[0]
newRange.Collators[i] = oneColumnRan[0].Collators[0]
newRanges = append(newRanges, newRange)
}
if sc.MemTracker != nil && len(newRanges) != 0 {
Expand Down
4 changes: 2 additions & 2 deletions planner/core/find_best_task.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1520,7 +1520,7 @@ func convertRangeFromExpectedCnt(ranges []*ranger.Range, rangeCounts []float64,
if i < 0 {
return nil, 0, true
}
convertedRanges = []*ranger.Range{{LowVal: ranges[i].HighVal, HighVal: []types.Datum{types.MaxValueDatum()}, LowExclude: !ranges[i].HighExclude}}
convertedRanges = []*ranger.Range{{LowVal: ranges[i].HighVal, HighVal: []types.Datum{types.MaxValueDatum()}, LowExclude: !ranges[i].HighExclude, Collators: ranges[i].Collators}}
} else {
for i = 0; i < len(ranges); i++ {
if count+rangeCounts[i] >= expectedCnt {
Expand All @@ -1531,7 +1531,7 @@ func convertRangeFromExpectedCnt(ranges []*ranger.Range, rangeCounts []float64,
if i == len(ranges) {
return nil, 0, true
}
convertedRanges = []*ranger.Range{{LowVal: []types.Datum{{}}, HighVal: ranges[i].LowVal, HighExclude: !ranges[i].LowExclude}}
convertedRanges = []*ranger.Range{{LowVal: []types.Datum{{}}, HighVal: ranges[i].LowVal, HighExclude: !ranges[i].LowExclude, Collators: ranges[i].Collators}}
}
return convertedRanges, count, false
}
Expand Down
16 changes: 10 additions & 6 deletions planner/util/path_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/pingcap/tidb/planner/core"
"github.com/pingcap/tidb/planner/util"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/ranger"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -81,16 +82,19 @@ func TestOnlyPointRange(t *testing.T) {
nullDatum := types.MinNotNullDatum()
nullDatum.SetNull()
nullPointRange := ranger.Range{
LowVal: []types.Datum{*nullDatum.Clone()},
HighVal: []types.Datum{*nullDatum.Clone()},
LowVal: []types.Datum{*nullDatum.Clone()},
HighVal: []types.Datum{*nullDatum.Clone()},
Collators: collate.GetBinaryCollatorSlice(1),
}
onePointRange := ranger.Range{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(1)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(1)},
Collators: collate.GetBinaryCollatorSlice(1),
}
one2TwoRange := ranger.Range{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Collators: collate.GetBinaryCollatorSlice(1),
}

intHandlePath := &util.AccessPath{IsIntHandlePath: true}
Expand Down
1 change: 1 addition & 0 deletions statistics/feedback.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ func (q *QueryFeedback) DecodeToRanges(isIndex bool) ([]*ranger.Range, error) {
LowVal: lowVal,
HighVal: highVal,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(len(lowVal)),
}))
}
return ranges, nil
Expand Down
1 change: 1 addition & 0 deletions statistics/handle/handle_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ func (s *testStatsSuite) TestColumnIDs(c *C) {
HighVal: []types.Datum{types.NewIntDatum(2)},
LowExclude: false,
HighExclude: true,
Collators: collate.GetBinaryCollatorSlice(1),
}
count, err := statsTbl.GetRowCountByColumnRanges(sctx, tableInfo.Columns[0].ID, []*ranger.Range{ran})
c.Assert(err, IsNil)
Expand Down
Loading