From 087af5faba2e382fd01150523b1c20b038a8ac42 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 2 Jan 2025 12:07:02 +0200 Subject: [PATCH] support weekly rotations, support YEARWEEK function, support 'mode' argument Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/schemadiff/partitioning_analysis.go | 142 +++++-- .../schemadiff/partitioning_analysis_test.go | 348 +++++++++++++++++- 2 files changed, 451 insertions(+), 39 deletions(-) diff --git a/go/vt/schemadiff/partitioning_analysis.go b/go/vt/schemadiff/partitioning_analysis.go index 3ac0caa1a7a..8f9c06eb458 100644 --- a/go/vt/schemadiff/partitioning_analysis.go +++ b/go/vt/schemadiff/partitioning_analysis.go @@ -19,6 +19,7 @@ package schemadiff import ( "errors" "fmt" + "math" "strconv" "strings" "time" @@ -28,6 +29,26 @@ import ( "vitess.io/vitess/go/vt/sqlparser" ) +const ( + ModeUndefined = math.MinInt +) + +// TemporalRangePartitioningAnalysis is the result of analyzing a table for temporal range partitioning. +type TemporalRangePartitioningAnalysis struct { + IsRangePartitioned bool // Is the table at all partitioned by RANGE? + IsTemporalRangePartitioned bool // Is the table range partitioned using temporal values? + IsRangeColumns bool // Is RANGE COLUMNS used? + MinimalInterval datetime.IntervalType // The minimal interval that the table is partitioned by (e.g. if partitioned by TO_DAYS, the minimal interval is 1 day) + Col *ColumnDefinitionEntity // The column used in the RANGE expression + FuncExpr *sqlparser.FuncExpr // The function used in the RANGE expression, if any + Mode int // The mode used in the WEEK function, if that's what's used + MaxvaluePartition *sqlparser.PartitionDefinition // The partition that has MAXVALUE, if any + HighestValueDateTime datetime.DateTime // The datetime value of the highest partition (excluding MAXVALUE) + HighestValueIntVal int64 // The integer value of the highest partition (excluding MAXVALUE) + Reason string // Why IsTemporalRangePartitioned is false + Error error // If there was an error during analysis +} + // IsRangePartitioned returns `true` when the given CREATE TABLE statement is partitioned by RANGE. func IsRangePartitioned(createTable *sqlparser.CreateTable) bool { if createTable.TableSpec.PartitionOption == nil { @@ -83,21 +104,6 @@ func AlterTableRotatesRangePartition(createTable *sqlparser.CreateTable, alterTa } } -// TemporalRangePartitioningAnalysis is the result of analyzing a table for temporal range partitioning. -type TemporalRangePartitioningAnalysis struct { - IsRangePartitioned bool // Is the table at all partitioned by RANGE? - IsTemporalRangePartitioned bool // Is the table range partitioned using temporal values? - IsRangeColumns bool // Is RANGE COLUMNS used? - MinimalInterval datetime.IntervalType // The minimal interval that the table is partitioned by (e.g. if partitioned by TO_DAYS, the minimal interval is 1 day) - Col *ColumnDefinitionEntity // The column used in the RANGE expression - FuncExpr *sqlparser.FuncExpr // The function used in the RANGE expression, if any - MaxvaluePartition *sqlparser.PartitionDefinition // The partition that has MAXVALUE, if any - HighestValueDateTime datetime.DateTime // The datetime value of the highest partition (excluding MAXVALUE) - HighestValueIntVal int64 // The integer value of the highest partition (excluding MAXVALUE) - Reason string // Why IsTemporalRangePartitioned is false - Error error // If there was an error during analysis -} - // supportedPartitioningScheme checks whether the given expression is supported for temporal range partitioning. // schemadiff only supports a subset of range partitioning expressions. func supportedPartitioningScheme(expr sqlparser.Expr, createTableEntity *CreateTableEntity, colName sqlparser.IdentifierCI, is84 bool) (matchFound bool, hasFunction bool, err error) { @@ -105,9 +111,15 @@ func supportedPartitioningScheme(expr sqlparser.Expr, createTableEntity *CreateT "create table %s (id int) PARTITION BY RANGE (%s)", "create table %s (id int) PARTITION BY RANGE (to_seconds(%s))", "create table %s (id int) PARTITION BY RANGE (to_days(%s))", - // "create table %s (id int) PARTITION BY RANGE (yearweek(%s))", - // "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 0))", - // "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 1))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 0))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 1))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 2))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 3))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 4))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 5))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 6))", + "create table %s (id int) PARTITION BY RANGE (yearweek(%s, 7))", "create table %s (id int) PARTITION BY RANGE (year(%s))", } if is84 { @@ -133,6 +145,32 @@ func supportedPartitioningScheme(expr sqlparser.Expr, createTableEntity *CreateT return false, false, nil } +// extractFuncMode extracts the mode argument from a WEEK/YEARWEEK function, if applicable. +// It returns a ModeUndefined when not applicable. +func extractFuncMode(funcExpr *sqlparser.FuncExpr) (int, error) { + switch funcExpr.Name.Lowered() { + case "week", "yearweek": + default: + return ModeUndefined, nil + } + if len(funcExpr.Exprs) <= 1 { + return 0, nil + } + // There is a `mode` argument in the YEARWEEK function. + literal, ok := funcExpr.Exprs[1].(*sqlparser.Literal) + if !ok { + return 0, fmt.Errorf("expected literal value in %v function", sqlparser.CanonicalString(funcExpr)) + } + if literal.Type != sqlparser.IntVal { + return 0, fmt.Errorf("expected integer literal argument in %v function", sqlparser.CanonicalString(funcExpr)) + } + intval, err := strconv.ParseInt(literal.Val, 0, 64) + if err != nil { + return 0, err + } + return int(intval), nil +} + // AnalyzeTemporalRangePartitioning analyzes a table for temporal range partitioning. func AnalyzeTemporalRangePartitioning(createTableEntity *CreateTableEntity) (*TemporalRangePartitioningAnalysis, error) { analysis := &TemporalRangePartitioningAnalysis{} @@ -162,6 +200,7 @@ func AnalyzeTemporalRangePartitioning(createTableEntity *CreateTableEntity) (*Te } analysis.IsRangeColumns = len(partitionOption.ColList) > 0 + analysis.Mode = ModeUndefined switch len(partitionOption.ColList) { case 0: // This is a PARTITION BY RANGE(expr), where "expr" can be just column name, or a complex expression. @@ -175,7 +214,7 @@ func AnalyzeTemporalRangePartitioning(createTableEntity *CreateTableEntity) (*Te // we create dummy statements with all supported variations, and check for equality. var col *ColumnDefinitionEntity expr := sqlparser.CloneExpr(partitionOption.Expr) - _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { + err = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { switch node := node.(type) { case *sqlparser.ColName: col = getColumn(node.Name.Lowered()) // known to be not-nil thanks to validate() @@ -183,9 +222,17 @@ func AnalyzeTemporalRangePartitioning(createTableEntity *CreateTableEntity) (*Te case *sqlparser.FuncExpr: analysis.FuncExpr = sqlparser.CloneRefOfFuncExpr(node) node.Name = sqlparser.NewIdentifierCI(node.Name.Lowered()) + mode, err := extractFuncMode(node) + if err != nil { + return false, err + } + analysis.Mode = mode } return true, nil }, expr) + if err != nil { + return nil, err + } matchFound, hasFunction, err := supportedPartitioningScheme(expr, createTableEntity, col.ColumnDefinition.Name, is84) if err != nil { return nil, err @@ -245,6 +292,8 @@ func AnalyzeTemporalRangePartitioning(createTableEntity *CreateTableEntity) (*Te analysis.MinimalInterval = datetime.IntervalSecond case "to_days": analysis.MinimalInterval = datetime.IntervalDay + case "yearweek": + analysis.MinimalInterval = datetime.IntervalWeek case "year": analysis.MinimalInterval = datetime.IntervalYear } @@ -263,7 +312,7 @@ func AnalyzeTemporalRangePartitioning(createTableEntity *CreateTableEntity) (*Te if err != nil { return analysis, err } - highestValueDateTime, err = truncateDateTime(highestValueDateTime, analysis.MinimalInterval) + highestValueDateTime, err = truncateDateTime(highestValueDateTime, analysis.MinimalInterval, analysis.Mode) if err != nil { return analysis, err } @@ -332,7 +381,9 @@ func computeDateTime(expr sqlparser.Expr, colType string, funcExpr *sqlparser.Fu } hasFuncExpr = true case *sqlparser.Literal: - literal = node + if literal == nil { + literal = node + } } return true, nil }, expr) @@ -373,6 +424,12 @@ func applyFuncExprToDateTime(dt datetime.DateTime, funcExpr *sqlparser.FuncExpr) intval = dt.ToSeconds() case "to_days": intval = int64(datetime.MysqlDayNumber(dt.Date.Year(), dt.Date.Month(), dt.Date.Day())) + case "yearweek": + mode, err := extractFuncMode(funcExpr) + if err != nil { + return 0, err + } + intval = int64(dt.Date.YearWeek(mode)) case "year": intval = int64(dt.Date.Year()) default: @@ -389,6 +446,7 @@ func temporalPartitionName(dt datetime.DateTime, resolution datetime.IntervalTyp switch resolution { case datetime.IntervalYear, datetime.IntervalMonth, + datetime.IntervalWeek, datetime.IntervalDay: return "p" + string(datetime.Date_YYYYMMDD.Format(dt, 0)), nil case datetime.IntervalHour, @@ -403,7 +461,8 @@ func temporalPartitionName(dt datetime.DateTime, resolution datetime.IntervalTyp // e.g. if resolution is IntervalDay, the time part is removed. // If resolution is IntervalMonth, the day part is set to 1. // etc. -func truncateDateTime(dt datetime.DateTime, interval datetime.IntervalType) (datetime.DateTime, error) { +// `mode` is used for WEEK calculations, see https://dev.mysql.com/doc/refman/8.4/en/date-and-time-functions.html#function_week +func truncateDateTime(dt datetime.DateTime, interval datetime.IntervalType, mode int) (datetime.DateTime, error) { if interval >= datetime.IntervalHour { // Remove the minutes, seconds, subseconds parts hourInterval := datetime.ParseIntervalInt64(int64(dt.Time.Hour()), datetime.IntervalHour, false) @@ -418,6 +477,25 @@ func truncateDateTime(dt datetime.DateTime, interval datetime.IntervalType) (dat // Remove the Time part: dt = datetime.DateTime{Date: dt.Date} } + if interval == datetime.IntervalWeek { + // IntervalWeek = IntervalDay | intervalMulti, which is larger than IntervalYear, so we interject here + // Get back to the first day of the week: + var startOfWeekInterval *datetime.Interval + switch mode { + case 0, 2, 4, 6: + startOfWeekInterval = datetime.ParseIntervalInt64(-int64(dt.Date.Weekday()), datetime.IntervalDay, false) + case 1, 3, 5, 7: + startOfWeekInterval = datetime.ParseIntervalInt64(-int64(dt.Date.Weekday()-1), datetime.IntervalDay, false) + default: + return dt, fmt.Errorf("invalid mode value %d for WEEK/YEARWEEK function", mode) + } + var ok bool + dt, _, ok = dt.AddInterval(startOfWeekInterval, 0, false) + if !ok { + return dt, fmt.Errorf("failed to add interval %v to reference time %v", startOfWeekInterval, dt.Format(0)) + } + return dt, nil + } if interval >= datetime.IntervalMonth { // Get back to the first day of the month: dayInterval := datetime.ParseIntervalInt64(int64(-(dt.Date.Day() - 1)), datetime.IntervalDay, false) @@ -446,7 +524,7 @@ func truncateDateTime(dt datetime.DateTime, interval datetime.IntervalType) (dat // e.g. "prepare 7 days ahead, starting from today". // The function computes values of existing partitions to determine how many new partitions are actually // required to satisfy the terms. -func TemporalRangePartitioningNextRotation(createTableEntity *CreateTableEntity, interval datetime.IntervalType, prepareAheadCount int, reference time.Time) (diffs []*AlterTableEntityDiff, err error) { +func TemporalRangePartitioningNextRotation(createTableEntity *CreateTableEntity, interval datetime.IntervalType, mode int, prepareAheadCount int, reference time.Time) (diffs []*AlterTableEntityDiff, err error) { analysis, err := AnalyzeTemporalRangePartitioning(createTableEntity) if err != nil { return nil, err @@ -454,10 +532,24 @@ func TemporalRangePartitioningNextRotation(createTableEntity *CreateTableEntity, if !analysis.IsTemporalRangePartitioned { return nil, errors.New(analysis.Reason) } - if interval < analysis.MinimalInterval { + intervalIsTooSmall := false + // IntervalWeek = IntervalDay | intervalMulti, which is larger all of the rest of normal intervals, + // so we need special handling for IntervalWeek comparisons + switch { + case analysis.MinimalInterval == datetime.IntervalWeek: + intervalIsTooSmall = (interval <= datetime.IntervalDay) + case interval == datetime.IntervalWeek: + intervalIsTooSmall = (analysis.MinimalInterval >= datetime.IntervalMonth) + default: + intervalIsTooSmall = (interval < analysis.MinimalInterval) + } + if intervalIsTooSmall { return nil, fmt.Errorf("interval %s is less than the minimal interval %s for table %s", interval.ToString(), analysis.MinimalInterval.ToString(), createTableEntity.Name()) } - referenceDatetime, err := truncateDateTime(datetime.NewDateTimeFromStd(reference), interval) + if analysis.Mode != ModeUndefined && mode != analysis.Mode { + return nil, fmt.Errorf("mode %d is different from the mode %d used in table %s", mode, analysis.Mode, createTableEntity.Name()) + } + referenceDatetime, err := truncateDateTime(datetime.NewDateTimeFromStd(reference), interval, mode) if err != nil { return nil, err } diff --git a/go/vt/schemadiff/partitioning_analysis_test.go b/go/vt/schemadiff/partitioning_analysis_test.go index 82fc6ed90e4..0ab1f9a3cde 100644 --- a/go/vt/schemadiff/partitioning_analysis_test.go +++ b/go/vt/schemadiff/partitioning_analysis_test.go @@ -155,6 +155,30 @@ func TestTemporalFunctions(t *testing.T) { yearweek := dt.Date.YearWeek(1) assert.EqualValues(t, 202451, yearweek) } + { + yearweek := dt.Date.YearWeek(2) + assert.EqualValues(t, 202450, yearweek) + } + { + yearweek := dt.Date.YearWeek(3) + assert.EqualValues(t, 202451, yearweek) + } + { + yearweek := dt.Date.YearWeek(4) + assert.EqualValues(t, 202451, yearweek) + } + { + yearweek := dt.Date.YearWeek(5) + assert.EqualValues(t, 202451, yearweek) + } + { + yearweek := dt.Date.YearWeek(6) + assert.EqualValues(t, 202451, yearweek) + } + { + yearweek := dt.Date.YearWeek(7) + assert.EqualValues(t, 202451, yearweek) + } } func TestTruncateDateTime(t *testing.T) { @@ -163,8 +187,10 @@ func TestTruncateDateTime(t *testing.T) { dt := datetime.NewDateTimeFromStd(tm) tcases := []struct { - interval datetime.IntervalType - expect string + interval datetime.IntervalType + mode int + expect string + expectErr error }{ { interval: datetime.IntervalYear, @@ -174,6 +200,49 @@ func TestTruncateDateTime(t *testing.T) { interval: datetime.IntervalMonth, expect: "2024-12-01 00:00:00", }, + { + interval: datetime.IntervalWeek, + expect: "2024-12-15 00:00:00", + }, + { + interval: datetime.IntervalWeek, + mode: 1, + expect: "2024-12-16 00:00:00", + }, + { + interval: datetime.IntervalWeek, + mode: 2, + expect: "2024-12-15 00:00:00", + }, + { + interval: datetime.IntervalWeek, + mode: 3, + expect: "2024-12-16 00:00:00", + }, + { + interval: datetime.IntervalWeek, + mode: 4, + expect: "2024-12-15 00:00:00", + }, + { + interval: datetime.IntervalWeek, + mode: 5, + expect: "2024-12-16 00:00:00", + }, + { + interval: datetime.IntervalWeek, + mode: 6, + expect: "2024-12-15 00:00:00", + }, + { + interval: datetime.IntervalWeek, + mode: 7, + expect: "2024-12-16 00:00:00", + }, { + interval: datetime.IntervalWeek, + mode: 8, + expectErr: fmt.Errorf("invalid mode value 8 for WEEK/YEARWEEK function"), + }, { interval: datetime.IntervalDay, expect: "2024-12-19 00:00:00", @@ -195,7 +264,12 @@ func TestTruncateDateTime(t *testing.T) { } for _, tcase := range tcases { t.Run(tcase.interval.ToString(), func(t *testing.T) { - truncated, err := truncateDateTime(dt, tcase.interval) + truncated, err := truncateDateTime(dt, tcase.interval, tcase.mode) + if tcase.expectErr != nil { + require.Error(t, err) + assert.EqualError(t, err, tcase.expectErr.Error()) + return + } require.NoError(t, err) assert.Equal(t, tcase.expect, string(truncated.Format(0))) }) @@ -430,19 +504,158 @@ func TestAnalyzeTemporalRangePartitioning(t *testing.T) { }, }, { - name: "unsupported function expression", - create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", - expectErr: fmt.Errorf("expression: YEARWEEK(`created_at`) is unsupported in temporal range partitioning analysis in table t"), + name: "range by YEARWEEK(DATETIME)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-15 00:00:00"), + }, }, { - name: "unsupported function expression", - create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 0)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", - expectErr: fmt.Errorf("expression: YEARWEEK(`created_at`, 0) is unsupported in temporal range partitioning analysis in table t"), + name: "range by YEARWEEK(DATETIME, 0)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 0)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-15 00:00:00"), + }, }, { - name: "unsupported function expression", - create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 7)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", - expectErr: fmt.Errorf("expression: YEARWEEK(`created_at`, 7) is unsupported in temporal range partitioning analysis in table t"), + name: "range by YEARWEEK(DATETIME, 1)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 1)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-16 00:00:00"), + }, + }, + { + name: "range by YEARWEEK(DATETIME, 2)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 2)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-15 00:00:00"), + }, + }, + { + name: "range by YEARWEEK(DATETIME, 3)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 3)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-16 00:00:00"), + }, + }, + { + name: "range by YEARWEEK(DATETIME, 4)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 4)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-15 00:00:00"), + }, + }, + { + name: "range by YEARWEEK(DATETIME, 5)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 5)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-16 00:00:00"), + }, + }, + { + name: "range by YEARWEEK(DATETIME, 6)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 6)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-15 00:00:00"), + }, + }, + { + name: "range by YEARWEEK(DATETIME, 7)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 7)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expect: &TemporalRangePartitioningAnalysis{ + IsRangePartitioned: true, + IsTemporalRangePartitioned: true, + MinimalInterval: datetime.IntervalWeek, + Col: &ColumnDefinitionEntity{ + ColumnDefinition: &sqlparser.ColumnDefinition{Name: sqlparser.NewIdentifierCI("created_at")}, + }, + FuncExpr: &sqlparser.FuncExpr{ + Name: sqlparser.NewIdentifierCI("YEARWEEK"), + }, + HighestValueDateTime: parseDateTime("2024-12-16 00:00:00"), + }, + }, + { + name: "unsupported YEARWEEK(DATETIME, 8)", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 8)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expectErr: fmt.Errorf("expression: YEARWEEK(`created_at`, 8) is unsupported in temporal range partitioning analysis in table t"), + }, + { + name: "unsupported YEARWEEK(DATETIME, 'x')", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 'x')) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + expectErr: fmt.Errorf("expected integer literal argument in yearweek(`created_at`, 'x') function"), }, { name: "unsupported function expression", @@ -493,6 +706,8 @@ func TestAnalyzeTemporalRangePartitioning(t *testing.T) { assert.EqualError(t, err, tcase.expectErr.Error()) return } + assert.NoError(t, err) + require.NotNil(t, result) assert.NoError(t, result.Error) require.NotNil(t, tcase.expect) assert.Equal(t, tcase.expect.Reason, result.Reason) @@ -531,6 +746,7 @@ func TestTemporalRangePartitioningNextRotation(t *testing.T) { name string create string interval datetime.IntervalType + mode int prepareAheadCount int expactMaxValue bool expectStatements []string @@ -544,12 +760,40 @@ func TestTemporalRangePartitioningNextRotation(t *testing.T) { expectErr: fmt.Errorf("Table does not use PARTITION BY RANGE"), }, { - name: "interval too short", + name: "interval too short: hour vs day", create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19 09:56:32')))", interval: datetime.IntervalHour, prepareAheadCount: 7, expectErr: fmt.Errorf("interval hour is less than the minimal interval day for table t"), }, + { + name: "interval too short: hour vs week", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:56:32')))", + interval: datetime.IntervalHour, + prepareAheadCount: 7, + expectErr: fmt.Errorf("interval hour is less than the minimal interval week for table t"), + }, + { + name: "interval too short: hour vs year", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEAR(created_at)) (PARTITION p0 VALUES LESS THAN (YEAR('2024-12-19 09:56:32')))", + interval: datetime.IntervalHour, + prepareAheadCount: 7, + expectErr: fmt.Errorf("interval hour is less than the minimal interval year for table t"), + }, + { + name: "interval too short: day vs year", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEAR(created_at)) (PARTITION p0 VALUES LESS THAN (YEAR('2024-12-19 09:56:32')))", + interval: datetime.IntervalDay, + prepareAheadCount: 7, + expectErr: fmt.Errorf("interval day is less than the minimal interval year for table t"), + }, + { + name: "interval too short: week vs year", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEAR(created_at)) (PARTITION p0 VALUES LESS THAN (YEAR('2024-12-19 09:56:32')))", + interval: datetime.IntervalWeek, + prepareAheadCount: 7, + expectErr: fmt.Errorf("interval week is less than the minimal interval year for table t"), + }, { name: "day interval with 7 days, DATE", create: "CREATE TABLE t (id int, created_at DATE, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (TO_DAYS(created_at)) (PARTITION p0 VALUES LESS THAN (TO_DAYS('2024-12-19')))", @@ -672,6 +916,82 @@ func TestTemporalRangePartitioningNextRotation(t *testing.T) { "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241219120000` VALUES LESS THAN ('2024-12-19 13:00:00'))", }, }, + { + name: "week(0) interval with 4 weeks", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'))", + interval: datetime.IntervalWeek, + mode: 0, + prepareAheadCount: 4, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241215` VALUES LESS THAN ('2024-12-22 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241222` VALUES LESS THAN ('2024-12-29 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241229` VALUES LESS THAN ('2025-01-05 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250105` VALUES LESS THAN ('2025-01-12 00:00:00'))", + }, + }, + { + name: "week(1) interval with 4 weeks", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'))", + interval: datetime.IntervalWeek, + mode: 1, + prepareAheadCount: 4, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241216` VALUES LESS THAN ('2024-12-23 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241223` VALUES LESS THAN ('2024-12-30 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241230` VALUES LESS THAN ('2025-01-06 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250106` VALUES LESS THAN ('2025-01-13 00:00:00'))", + }, + }, + { + name: "week(1) interval with 4 weeks, 2 of which are covered", + create: `CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) + PARTITION BY RANGE COLUMNS (created_at) ( + PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'), + PARTITION p20241216 VALUES LESS THAN ('2024-12-23 00:00:00'), + PARTITION p_somename VALUES LESS THAN ('2024-12-30 00:00:00') + )`, + interval: datetime.IntervalWeek, + mode: 1, + prepareAheadCount: 4, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241230` VALUES LESS THAN ('2025-01-06 00:00:00'))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250106` VALUES LESS THAN ('2025-01-13 00:00:00'))", + }, + }, + { + name: "yearweek(0) function with 4 weeks", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 0)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:00:00', 1)))", + interval: datetime.IntervalWeek, + mode: 0, + prepareAheadCount: 4, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241215` VALUES LESS THAN (202451))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241222` VALUES LESS THAN (202452))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241229` VALUES LESS THAN (202501))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250105` VALUES LESS THAN (202502))", + }, + }, + { + name: "yearweek(1) function with 4 weeks", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 1)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:00:00', 1)))", + interval: datetime.IntervalWeek, + mode: 1, + prepareAheadCount: 4, + expectStatements: []string{ + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241216` VALUES LESS THAN (202452))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241223` VALUES LESS THAN (202501))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20241230` VALUES LESS THAN (202502))", + "ALTER TABLE `t` ADD PARTITION (PARTITION `p20250106` VALUES LESS THAN (202503))", + }, + }, + { + name: "incompatible week mode", + create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE (YEARWEEK(created_at, 0)) (PARTITION p0 VALUES LESS THAN (YEARWEEK('2024-12-19 09:00:00', 1)))", + interval: datetime.IntervalWeek, + mode: 1, + prepareAheadCount: 4, + expectErr: fmt.Errorf("mode 1 is different from the mode 0 used in table t"), + }, { name: "month interval with 3 months", create: "CREATE TABLE t (id int, created_at DATETIME, PRIMARY KEY(id, created_at)) PARTITION BY RANGE COLUMNS (created_at) (PARTITION p0 VALUES LESS THAN ('2024-12-19 09:00:00'))", @@ -733,7 +1053,7 @@ func TestTemporalRangePartitioningNextRotation(t *testing.T) { entity, err := NewCreateTableEntityFromSQL(env, tcase.create) require.NoError(t, err) - diffs, err := TemporalRangePartitioningNextRotation(entity, tcase.interval, tcase.prepareAheadCount, reference) + diffs, err := TemporalRangePartitioningNextRotation(entity, tcase.interval, tcase.mode, tcase.prepareAheadCount, reference) if tcase.expectErr != nil { require.Error(t, err) assert.EqualError(t, err, tcase.expectErr.Error())