Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

evalengine: Fix handling of datetime and numeric comparisons #12789

Merged
merged 2 commits into from
Mar 31, 2023
Merged

evalengine: Fix handling of datetime and numeric comparisons #12789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2c2aab6
evalengine: Fix handling of datetime and numeric comparisons
dbussink Mar 29, 2023
35f992d
Fix tests
dbussink Mar 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 101 additions & 19 deletions go/mysql/datetime/parse.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package datetime

import (
"fmt"
"math"
"strconv"
"strings"
"time"
Expand All @@ -28,8 +29,11 @@ import (

var dateFormats = []string{"2006-01-02", "06-01-02", "20060102", "060102"}
var datetimeFormats = []string{"2006-01-02 15:04:05.9", "06-01-02 15:04:05.9", "20060102150405.9", "060102150405.9"}
var timeWithDayFormats = []string{"15:04:05.9", "15:04", "15"}
var timeWithoutDayFormats = []string{"15:04:05.9", "15:04", "150405.9", "0405", "05"}
var timeWithoutDayFormats = []string{"150405.9", "0405", "05"}

func validMessage(in string) string {
return strings.ToValidUTF8(strings.ReplaceAll(in, "\x00", ""), "?")
}

func ParseDate(in string) (t time.Time, err error) {
for _, f := range dateFormats {
Expand All @@ -38,15 +42,65 @@ func ParseDate(in string) (t time.Time, err error) {
return t, nil
}
}
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect DATE value: '%s'", in)
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect DATE value: '%s'", validMessage(in))
}

func parseHoursMinutesSeconds(in string) (hours, minutes, seconds, nanoseconds int, err error) {
parts := strings.Split(in, ":")
switch len(parts) {
case 3:
sub := strings.Split(parts[2], ".")
if len(sub) > 2 {
return 0, 0, 0, 0, fmt.Errorf("invalid time format: %s", in)
}
seconds, err = strconv.Atoi(sub[0])
if err != nil {
return 0, 0, 0, 0, err
}
if seconds > 59 {
return 0, 0, 0, 0, fmt.Errorf("invalid time format: %s", in)
}
if len(sub) == 2 {
nanoseconds, err = strconv.Atoi(sub[1])
if err != nil {
return 0, 0, 0, 0, err
}
nanoseconds = int(math.Round(float64(nanoseconds)*math.Pow10(9-len(sub[1]))/1000) * 1000)
}
fallthrough
case 2:
if len(parts[1]) > 2 {
return 0, 0, 0, 0, fmt.Errorf("invalid time format: %s", in)
}
minutes, err = strconv.Atoi(parts[1])
if err != nil {
return 0, 0, 0, 0, err
}
if minutes > 59 {
return 0, 0, 0, 0, fmt.Errorf("invalid time format: %s", in)
}
fallthrough
case 1:
hours, err = strconv.Atoi(parts[0])
if err != nil {
return 0, 0, 0, 0, err
}
default:
return 0, 0, 0, 0, fmt.Errorf("invalid time format: %s", in)
}

if hours < 0 || minutes < 0 || seconds < 0 || nanoseconds < 0 {
return 0, 0, 0, 0, fmt.Errorf("invalid time format: %s", in)
}
return
}

func ParseTime(in string) (t time.Time, err error) {
func ParseTime(in string) (t time.Time, out string, err error) {
// ParseTime is right now only excepting on specific
// time format and doesn't accept all formats MySQL accepts.
// Can be improved in the future as needed.
if in == "" {
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
return t, "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", validMessage(in))
}
start := 0
neg := in[start] == '-'
Expand All @@ -56,46 +110,59 @@ func ParseTime(in string) (t time.Time, err error) {

parts := strings.Split(in[start:], " ")
if len(parts) > 2 {
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
return t, "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", validMessage(in))
}
days := 0
var days, hours, minutes, seconds, nanoseconds int
hourMinuteSeconds := parts[0]
if len(parts) == 2 {
days, err = strconv.Atoi(parts[0])
if err != nil {
fmt.Printf("atoi failed: %+v\n", err)
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
return t, "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", validMessage(in))
}
if days < 0 {
// Double negative which is not allowed
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
return t, "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", validMessage(in))
}
if days > 34 {
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
return t, "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", validMessage(in))
}
for _, f := range timeWithDayFormats {
t, err = time.Parse(f, parts[1])
if err == nil {
break
}
hours, minutes, seconds, nanoseconds, err = parseHoursMinutesSeconds(parts[1])
if err != nil {
return t, "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", validMessage(in))
}
days = days + hours/24
hours = hours % 24
t = time.Date(0, 1, 1, hours, minutes, seconds, nanoseconds, time.UTC)
} else {
for _, f := range timeWithoutDayFormats {
t, err = time.Parse(f, hourMinuteSeconds)
if err == nil {
break
}
}
if err != nil {
hours, minutes, seconds, nanoseconds, err = parseHoursMinutesSeconds(parts[0])
if err != nil {
return t, "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", validMessage(in))
}
days = hours / 24
hours = hours % 24
t = time.Date(0, 1, 1, hours, minutes, seconds, nanoseconds, time.UTC)
err = nil
}
}

if err != nil {
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", in)
return t, "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", validMessage(in))
}

// setting the date to today's date, because t is "0000-01-01 xx:xx:xx"
now := time.Now()
year, month, day := now.Date()
b := strings.Builder{}

if neg {
b.WriteString("-")
// If we have a negative time, we start with the start of today
// and substract the total duration of the parsed time.
today := time.Date(year, month, day, 0, 0, 0, 0, t.Location())
Expand All @@ -104,13 +171,28 @@ func ParseTime(in string) (t time.Time, err error) {
time.Duration(t.Minute())*time.Minute +
time.Duration(t.Second())*time.Second +
time.Duration(t.Nanosecond())*time.Nanosecond
fmt.Fprintf(&b, "%02d", days*24+t.Hour())
fmt.Fprintf(&b, ":%02d", t.Minute())
fmt.Fprintf(&b, ":%02d", t.Second())
if t.Nanosecond() > 0 {
fmt.Fprintf(&b, ".%09d", t.Nanosecond())
}

t = today.Add(-duration)
} else {
fmt.Fprintf(&b, "%02d", days*24+t.Hour())
fmt.Fprintf(&b, ":%02d", t.Minute())
fmt.Fprintf(&b, ":%02d", t.Second())
if t.Nanosecond() > 0 {
fmt.Fprintf(&b, ".%09d", t.Nanosecond())
}

// In case of a positive time, we can take a quicker
// shortcut and add the date of today.
t = t.AddDate(year, int(month-1), day-1+days)
}
return t, nil

return t, b.String(), nil
}

func ParseDateTime(in string) (t time.Time, err error) {
Expand All @@ -120,5 +202,5 @@ func ParseDateTime(in string) (t time.Time, err error) {
return t, nil
}
}
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect DATETIME value: '%s'", in)
return t, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect DATETIME value: '%s'", validMessage(in))
}
29 changes: 26 additions & 3 deletions go/mysql/datetime/parse_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,60 +77,82 @@ func TestParseTime(t *testing.T) {
tests := []struct {
input string
output testTime
norm string
err bool
}{{
input: "11:12:13",
norm: "11:12:13",
output: testTime{11, 12, 13, 0},
}, {
input: "11:12:13.123456",
norm: "11:12:13.123456000",
output: testTime{11, 12, 13, 123456000},
}, {
input: "3 11:12:13",
norm: "83:12:13",
output: testTime{3*24 + 11, 12, 13, 0},
}, {
input: "3 41:12:13",
norm: "113:12:13",
output: testTime{3*24 + 41, 12, 13, 0},
}, {
input: "35 11:12:13",
err: true,
}, {
input: "11:12",
norm: "11:12:00",
output: testTime{11, 12, 0, 0},
}, {
input: "5 11:12",
norm: "131:12:00",
output: testTime{5*24 + 11, 12, 0, 0},
}, {
input: "-2 11:12",
norm: "-59:12:00",
output: testTime{-2*24 - 11, -12, 0, 0},
}, {
input: "--2 11:12",
err: true,
}, {
input: "2 11",
norm: "59:00:00",
output: testTime{2*24 + 11, 0, 0, 0},
}, {
input: "2 -11",
err: true,
}, {
input: "13",
norm: "00:00:13",
output: testTime{0, 0, 13, 0},
}, {
input: "111213",
norm: "11:12:13",
output: testTime{11, 12, 13, 0},
}, {
input: "111213.123456",
norm: "11:12:13.123456000",
output: testTime{11, 12, 13, 123456000},
}, {
input: "-111213",
norm: "-11:12:13",
output: testTime{-11, -12, -13, 0},
}, {
input: "1213",
norm: "00:12:13",
output: testTime{0, 12, 13, 0},
}, {
input: "25:12:13",
err: true,
input: "25:12:13",
norm: "25:12:13",
output: testTime{25, 12, 13, 0},
}, {
input: "32:35",
norm: "32:35:00",
output: testTime{32, 35, 0, 0},
}}

for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
got, err := ParseTime(test.input)
got, norm, err := ParseTime(test.input)
if test.err {
assert.Errorf(t, err, "got: %s", got)
return
Expand All @@ -145,6 +167,7 @@ func TestParseTime(t *testing.T) {
time.Duration(test.output.nanosecond)*time.Nanosecond)

assert.Equal(t, expected, got)
assert.Equal(t, test.norm, norm)
})
}
}
Expand Down
2 changes: 1 addition & 1 deletion go/mysql/json/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -846,7 +846,7 @@ func (v *Value) Time() (time.Time, bool) {
if v.t != TypeTime {
return time.Time{}, false
}
t, err := datetime.ParseTime(v.s)
t, _, err := datetime.ParseTime(v.s)
if err != nil {
return time.Time{}, false
}
Expand Down
9 changes: 7 additions & 2 deletions go/sqltypes/type.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,16 @@ func IsNumber(t querypb.Type) bool {
return IsIntegral(t) || IsFloat(t) || t == Decimal
}

// IsDate returns true if the type represents a date and/or time.
func IsDate(t querypb.Type) bool {
// IsDateOrTime returns true if the type represents a date and/or time.
func IsDateOrTime(t querypb.Type) bool {
return t == Datetime || t == Date || t == Timestamp || t == Time
}

// IsDate returns true if the type has a date component
func IsDate(t querypb.Type) bool {
return t == Datetime || t == Date || t == Timestamp
}

// IsNull returns true if the type is NULL type
func IsNull(t querypb.Type) bool {
return t == Null
Expand Down
2 changes: 1 addition & 1 deletion go/vt/sqlparser/normalizer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func validateLiteral(node *Literal) (err error) {
case DateVal:
_, err = datetime.ParseDate(node.Val)
case TimeVal:
_, err = datetime.ParseTime(node.Val)
_, _, err = datetime.ParseTime(node.Val)
case TimestampVal:
_, err = datetime.ParseDateTime(node.Val)
}
Expand Down
6 changes: 3 additions & 3 deletions go/vt/sqlparser/normalizer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,13 +363,13 @@ func TestNormalizeInvalidDates(t *testing.T) {
err error
}{{
in: "select date'foo'",
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect DATE value: '%s'", "foo"),
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect DATE value: '%s'", "foo"),
}, {
in: "select time'foo'",
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect TIME value: '%s'", "foo"),
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect TIME value: '%s'", "foo"),
}, {
in: "select timestamp'foo'",
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "incorrect DATETIME value: '%s'", "foo"),
err: vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongValue, "Incorrect DATETIME value: '%s'", "foo"),
}}
for _, tc := range testcases {
t.Run(tc.in, func(t *testing.T) {
Expand Down
14 changes: 7 additions & 7 deletions go/vt/vtgate/evalengine/api_compare_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -871,43 +871,43 @@ func TestCompareDates(t *testing.T) {
},
{
name: "string equal datetime",
v1: NewColumn(0), v2: NewColumn(1),
v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1),
out: &T, op: sqlparser.EqualOp,
row: []sqltypes.Value{sqltypes.NewVarChar("2021-10-22"), sqltypes.NewDatetime("2021-10-22 00:00:00")},
},
{
name: "string equal timestamp",
v1: NewColumn(0), v2: NewColumn(1),
v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1),
out: &T, op: sqlparser.EqualOp,
row: []sqltypes.Value{sqltypes.NewVarChar("2021-10-22 00:00:00"), sqltypes.NewTimestamp("2021-10-22 00:00:00")},
},
{
name: "string not equal timestamp",
v1: NewColumn(0), v2: NewColumn(1),
v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1),
out: &T, op: sqlparser.NotEqualOp,
row: []sqltypes.Value{sqltypes.NewVarChar("2021-10-22 06:00:30"), sqltypes.NewTimestamp("2021-10-20 15:02:10")},
},
{
name: "string equal time",
v1: NewColumn(0), v2: NewColumn(1),
v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1),
out: &T, op: sqlparser.EqualOp,
row: []sqltypes.Value{sqltypes.NewVarChar("00:05:12"), sqltypes.NewTime("00:05:12")},
},
{
name: "string equal date",
v1: NewColumn(0), v2: NewColumn(1),
v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1),
out: &T, op: sqlparser.EqualOp,
row: []sqltypes.Value{sqltypes.NewVarChar("2021-02-22"), sqltypes.NewDate("2021-02-22")},
},
{
name: "string not equal date (1, date on the RHS)",
v1: NewColumn(0), v2: NewColumn(1),
v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1),
out: &T, op: sqlparser.NotEqualOp,
row: []sqltypes.Value{sqltypes.NewVarChar("2021-02-20"), sqltypes.NewDate("2021-03-30")},
},
{
name: "string not equal date (2, date on the LHS)",
v1: NewColumn(0), v2: NewColumn(1),
v1: NewColumn(0), v2: NewColumnWithCollation(1, defaultCollation()),
out: &T, op: sqlparser.NotEqualOp,
row: []sqltypes.Value{sqltypes.NewDate("2021-03-30"), sqltypes.NewVarChar("2021-02-20")},
},
Expand Down
Loading