From 5012d331f0dd3fae51821118d5185bc125e041e1 Mon Sep 17 00:00:00 2001 From: James Cor Date: Fri, 15 Dec 2023 22:40:01 -0800 Subject: [PATCH] serialize `on update` expressions (#7115) --- go/cmd/dolt/commands/diff.go | 15 +++++++++- go/gen/fb/serial/schema.go | 30 +++++++++++++++++-- go/go.mod | 2 +- go/go.sum | 4 +-- .../doltcore/merge/merge_prolly_rows.go | 4 +-- .../doltcore/mvdata/engine_table_writer.go | 2 +- go/libraries/doltcore/schema/column.go | 3 ++ .../doltcore/schema/encoding/serialization.go | 30 +++++++++++++++++-- .../sqle/enginetest/dolt_engine_test.go | 6 ++++ .../doltcore/sqle/sqlfmt/schema_fmt.go | 8 +++-- go/libraries/doltcore/sqle/sqlutil/convert.go | 14 +++++++-- go/serial/schema.fbs | 6 ++++ 12 files changed, 107 insertions(+), 17 deletions(-) diff --git a/go/cmd/dolt/commands/diff.go b/go/cmd/dolt/commands/diff.go index 91cdf52d736..b21d22b0c81 100644 --- a/go/cmd/dolt/commands/diff.go +++ b/go/cmd/dolt/commands/diff.go @@ -886,7 +886,7 @@ func schemaFromCreateTableStmt(createTableStmt string) (schema.Schema, error) { } } - cols := []schema.Column{} + var cols []schema.Column for _, col := range create.TableSpec.Columns { internalTyp, err := types.ColumnTypeToType(&col.Type) typeInfo, err := typeinfo.FromSqlType(internalTyp) @@ -898,10 +898,22 @@ func schemaFromCreateTableStmt(createTableStmt string) (schema.Schema, error) { if col.Type.Default != nil { col.Type.Default.Format(defBuf) } + + genBuf := ast.NewTrackedBuffer(nil) + if col.Type.GeneratedExpr != nil { + col.Type.GeneratedExpr.Format(genBuf) + } + + onUpBuf := ast.NewTrackedBuffer(nil) + if col.Type.OnUpdate != nil { + col.Type.OnUpdate.Format(onUpBuf) + } + var comment string if col.Type.Comment != nil { comment = col.Type.Comment.String() } + sCol := schema.Column{ Name: col.Name.String(), Kind: typeInfo.NomsKind(), @@ -909,6 +921,7 @@ func schemaFromCreateTableStmt(createTableStmt string) (schema.Schema, error) { TypeInfo: typeInfo, Default: defBuf.String(), Generated: "", // TODO + OnUpdate: "", // TODO Virtual: false, // TODO AutoIncrement: col.Type.Autoincrement == true, Comment: comment, diff --git a/go/gen/fb/serial/schema.go b/go/gen/fb/serial/schema.go index fc60fc72893..67a903f8e5a 100644 --- a/go/gen/fb/serial/schema.go +++ b/go/gen/fb/serial/schema.go @@ -149,7 +149,19 @@ func (rcv *TableSchema) MutateCollation(n Collation) bool { return rcv._tab.MutateUint16Slot(12, uint16(n)) } -const TableSchemaNumFields = 5 +func (rcv *TableSchema) HasFeaturesAfterTryAccessors() bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + return rcv._tab.GetBool(o + rcv._tab.Pos) + } + return false +} + +func (rcv *TableSchema) MutateHasFeaturesAfterTryAccessors(n bool) bool { + return rcv._tab.MutateBoolSlot(14, n) +} + +const TableSchemaNumFields = 6 func TableSchemaStart(builder *flatbuffers.Builder) { builder.StartObject(TableSchemaNumFields) @@ -178,6 +190,9 @@ func TableSchemaStartChecksVector(builder *flatbuffers.Builder, numElems int) fl func TableSchemaAddCollation(builder *flatbuffers.Builder, collation Collation) { builder.PrependUint16Slot(4, uint16(collation), 0) } +func TableSchemaAddHasFeaturesAfterTryAccessors(builder *flatbuffers.Builder, hasFeaturesAfterTryAccessors bool) { + builder.PrependBoolSlot(5, hasFeaturesAfterTryAccessors, false) +} func TableSchemaEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } @@ -354,7 +369,15 @@ func (rcv *Column) MutateVirtual(n bool) bool { return rcv._tab.MutateBoolSlot(28, n) } -const ColumnNumFields = 13 +func (rcv *Column) OnUpdateValue() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(30)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +const ColumnNumFields = 14 func ColumnStart(builder *flatbuffers.Builder) { builder.StartObject(ColumnNumFields) @@ -398,6 +421,9 @@ func ColumnAddGenerated(builder *flatbuffers.Builder, generated bool) { func ColumnAddVirtual(builder *flatbuffers.Builder, virtual bool) { builder.PrependBoolSlot(12, virtual, false) } +func ColumnAddOnUpdateValue(builder *flatbuffers.Builder, onUpdateValue flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(onUpdateValue), 0) +} func ColumnEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } diff --git a/go/go.mod b/go/go.mod index 4bf7c7db349..9df69170b89 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20231214194603-39acc5e6f988 + github.com/dolthub/go-mysql-server v0.17.1-0.20231215225524-76f133444ea4 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/go-github/v57 v57.0.0 diff --git a/go/go.sum b/go/go.sum index 51a451a3c57..5aec6331004 100644 --- a/go/go.sum +++ b/go/go.sum @@ -181,8 +181,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.17.1-0.20231214194603-39acc5e6f988 h1:oYhHtAZFJujmu6NeHfjliwNaVrbaq5pYbl9Pu84Bq2I= -github.com/dolthub/go-mysql-server v0.17.1-0.20231214194603-39acc5e6f988/go.mod h1:zJCyPiYe9VZ9xIQTv7S1OFKwyoVQoeGxZXNtkFxTcOI= +github.com/dolthub/go-mysql-server v0.17.1-0.20231215225524-76f133444ea4 h1:s1YRhf3iwEO89iO7pfq+eIV6kQHl9h7XS6MlYKhPptw= +github.com/dolthub/go-mysql-server v0.17.1-0.20231215225524-76f133444ea4/go.mod h1:zJCyPiYe9VZ9xIQTv7S1OFKwyoVQoeGxZXNtkFxTcOI= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= diff --git a/go/libraries/doltcore/merge/merge_prolly_rows.go b/go/libraries/doltcore/merge/merge_prolly_rows.go index fb2779582d1..9b26bd7db2f 100644 --- a/go/libraries/doltcore/merge/merge_prolly_rows.go +++ b/go/libraries/doltcore/merge/merge_prolly_rows.go @@ -1181,7 +1181,7 @@ func resolveDefaults(ctx *sql.Context, tableName string, mergedSchema schema.Sch return false, nil } - if col.Default != "" || col.Generated != "" { + if col.Default != "" || col.Generated != "" || col.OnUpdate != "" { expr, err := index.ResolveDefaultExpression(ctx, tableName, mergedSchema, col) if err != nil { return true, err @@ -1443,7 +1443,7 @@ func remapTupleWithColumnDefaults( // If the column is a new column, then look up any default or generated value in a second pass, after the // non-default and non-generated fields have been established. Virtual columns have been excluded, so any // generated column is stored. - if col.Default != "" || col.Generated != "" { + if col.Default != "" || col.Generated != "" || col.OnUpdate != "" { secondPass = append(secondPass, to) } } else { diff --git a/go/libraries/doltcore/mvdata/engine_table_writer.go b/go/libraries/doltcore/mvdata/engine_table_writer.go index f786a372460..8d4ff9ec6fa 100644 --- a/go/libraries/doltcore/mvdata/engine_table_writer.go +++ b/go/libraries/doltcore/mvdata/engine_table_writer.go @@ -270,7 +270,7 @@ func (s *SqlEngineTableWriter) createTable() error { // upstream to make the dolt schema sqlCols := make([]string, len(s.tableSchema.Schema)) for i, c := range s.tableSchema.Schema { - sqlCols[i] = sql.GenerateCreateTableColumnDefinition(c, c.Default.String(), sql.Collation_Default) + sqlCols[i] = sql.GenerateCreateTableColumnDefinition(c, c.Default.String(), c.OnUpdate.String(), sql.Collation_Default) } var pks string var sep string diff --git a/go/libraries/doltcore/schema/column.go b/go/libraries/doltcore/schema/column.go index dcfe62962f6..10f03f7f73f 100644 --- a/go/libraries/doltcore/schema/column.go +++ b/go/libraries/doltcore/schema/column.go @@ -74,6 +74,9 @@ type Column struct { // Generated is the generated value of this column. This is the string representation of a sql.Expression. Generated string + // OnUpdate is the on update value of this column. This is the string representation of a sql.Expression. + OnUpdate string + // Virtual is true if this is a virtual column. Virtual bool diff --git a/go/libraries/doltcore/schema/encoding/serialization.go b/go/libraries/doltcore/schema/encoding/serialization.go index b07316e68c8..87d081a9743 100644 --- a/go/libraries/doltcore/schema/encoding/serialization.go +++ b/go/libraries/doltcore/schema/encoding/serialization.go @@ -57,12 +57,23 @@ func serializeSchemaAsFlatbuffer(sch schema.Schema) ([]byte, error) { indexes := serializeSecondaryIndexes(b, sch, sch.Indexes().AllIndexes()) checks := serializeChecks(b, sch.Checks().AllChecks()) + var hasFeaturesAfterTryAccessors bool + for _, col := range sch.GetAllCols().GetColumns() { + if col.OnUpdate != "" { + hasFeaturesAfterTryAccessors = true + break + } + } + serial.TableSchemaStart(b) serial.TableSchemaAddClusteredIndex(b, rows) serial.TableSchemaAddColumns(b, columns) serial.TableSchemaAddSecondaryIndexes(b, indexes) serial.TableSchemaAddChecks(b, checks) serial.TableSchemaAddCollation(b, serial.Collation(sch.GetCollation())) + if hasFeaturesAfterTryAccessors { + serial.TableSchemaAddHasFeaturesAfterTryAccessors(b, hasFeaturesAfterTryAccessors) + } root := serial.TableSchemaEnd(b) bs := serial.FinishMessage(b, root, []byte(serial.TableSchemaFileID)) return bs, nil @@ -215,15 +226,21 @@ func serializeSchemaColumns(b *fb.Builder, sch schema.Schema) fb.UOffsetT { // serialize columns in |cols| for i := len(cols) - 1; i >= 0; i-- { col := cols[i] - defVal := "" + var defVal, onUpdateVal string if col.Default != "" { defVal = col.Default } else { defVal = col.Generated } + if col.OnUpdate != "" { + onUpdateVal = col.OnUpdate + } + co := b.CreateString(col.Comment) do := b.CreateString(defVal) + ou := b.CreateString(onUpdateVal) + typeString := sqlTypeString(col.TypeInfo) to := b.CreateString(typeString) no := b.CreateString(col.Name) @@ -242,6 +259,9 @@ func serializeSchemaColumns(b *fb.Builder, sch schema.Schema) fb.UOffsetT { serial.ColumnAddNullable(b, col.IsNullable()) serial.ColumnAddGenerated(b, col.Generated != "") serial.ColumnAddVirtual(b, col.Virtual) + if onUpdateVal != "" { + serial.ColumnAddOnUpdateValue(b, ou) + } serial.ColumnAddHidden(b, false) offs[i] = serial.ColumnEnd(b) } @@ -319,8 +339,7 @@ func deserializeColumns(ctx context.Context, s *serial.TableSchema) ([]schema.Co return nil, err } - defVal := "" - generatedVal := "" + var defVal, generatedVal, onUpdateVal string if c.DefaultValue() != nil { if c.Generated() { generatedVal = string(c.DefaultValue()) @@ -329,6 +348,10 @@ func deserializeColumns(ctx context.Context, s *serial.TableSchema) ([]schema.Co } } + if c.OnUpdateValue() != nil { + onUpdateVal = string(c.OnUpdateValue()) + } + cols[i] = schema.Column{ Name: string(c.Name()), Tag: c.Tag(), @@ -337,6 +360,7 @@ func deserializeColumns(ctx context.Context, s *serial.TableSchema) ([]schema.Co TypeInfo: sqlType, Default: defVal, Generated: generatedVal, + OnUpdate: onUpdateVal, Virtual: c.Virtual(), AutoIncrement: c.AutoIncrement(), Comment: string(c.Comment()), diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 7ce0ef8d260..9d4f5fab56c 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -1234,6 +1234,12 @@ func TestColumnDefaults(t *testing.T) { enginetest.TestColumnDefaults(t, h) } +func TestOnUpdateExprScripts(t *testing.T) { + h := newDoltHarness(t) + defer h.Close() + enginetest.TestOnUpdateExprScripts(t, h) +} + func TestAlterTable(t *testing.T) { // This is a newly added test in GMS that dolt doesn't support yet h := newDoltHarness(t).WithSkippedQueries([]string{"ALTER TABLE t42 ADD COLUMN s varchar(20), drop check check1"}) diff --git a/go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go b/go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go index c26a1c0cb92..2ccdece1f21 100644 --- a/go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go +++ b/go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go @@ -32,13 +32,16 @@ func GenerateCreateTableColumnDefinition(col schema.Column, tableCollation sql.C // GenerateCreateTableIndentedColumnDefinition returns column definition for CREATE TABLE statement with no indentation func GenerateCreateTableIndentedColumnDefinition(col schema.Column, tableCollation sql.CollationID) string { - var defaultVal, genVal *sql.ColumnDefaultValue + var defaultVal, genVal, onUpdateVal *sql.ColumnDefaultValue if col.Default != "" { defaultVal = sql.NewUnresolvedColumnDefaultValue(col.Default) } if col.Generated != "" { genVal = sql.NewUnresolvedColumnDefaultValue(col.Generated) } + if col.OnUpdate != "" { + onUpdateVal = sql.NewUnresolvedColumnDefaultValue(col.OnUpdate) + } return sql.GenerateCreateTableColumnDefinition( &sql.Column{ @@ -50,7 +53,8 @@ func GenerateCreateTableIndentedColumnDefinition(col schema.Column, tableCollati Comment: col.Comment, Generated: genVal, Virtual: col.Virtual, - }, col.Default, tableCollation) + OnUpdate: onUpdateVal, + }, col.Default, col.OnUpdate, tableCollation) } // GenerateCreateTableIndexDefinition returns index definition for CREATE TABLE statement with indentation of 2 spaces diff --git a/go/libraries/doltcore/sqle/sqlutil/convert.go b/go/libraries/doltcore/sqle/sqlutil/convert.go index bf33398d505..af11a31dd2d 100644 --- a/go/libraries/doltcore/sqle/sqlutil/convert.go +++ b/go/libraries/doltcore/sqle/sqlutil/convert.go @@ -40,19 +40,23 @@ func FromDoltSchema(dbName, tableName string, sch schema.Schema) (sql.PrimaryKey extra = "auto_increment" } - var deflt, generated *sql.ColumnDefaultValue + var deflt, generated, onUpdate *sql.ColumnDefaultValue if col.Default != "" { deflt = sql.NewUnresolvedColumnDefaultValue(col.Default) } if col.Generated != "" { generated = sql.NewUnresolvedColumnDefaultValue(col.Generated) } + if col.OnUpdate != "" { + onUpdate = sql.NewUnresolvedColumnDefaultValue(col.OnUpdate) + } cols[i] = &sql.Column{ Name: col.Name, Type: sqlType, Default: deflt, Generated: generated, + OnUpdate: onUpdate, Nullable: col.IsNullable(), DatabaseSource: dbName, Source: tableName, @@ -141,14 +145,17 @@ func ToDoltCol(tag uint64, col *sql.Column) (schema.Column, error) { return schema.Column{}, err } - defaultVal := "" - generatedVal := "" + var defaultVal, generatedVal, onUpdateVal string if col.Default != nil { defaultVal = col.Default.String() } else { generatedVal = col.Generated.String() } + if col.OnUpdate != nil { + onUpdateVal = col.OnUpdate.String() + } + c := schema.Column{ Name: col.Name, Tag: tag, @@ -157,6 +164,7 @@ func ToDoltCol(tag uint64, col *sql.Column) (schema.Column, error) { TypeInfo: typeInfo, Default: defaultVal, Generated: generatedVal, + OnUpdate: onUpdateVal, Virtual: col.Virtual, AutoIncrement: col.AutoIncrement, Comment: col.Comment, diff --git a/go/serial/schema.fbs b/go/serial/schema.fbs index d51be4e0ded..19efe47ae4d 100644 --- a/go/serial/schema.fbs +++ b/go/serial/schema.fbs @@ -24,6 +24,9 @@ table TableSchema { secondary_indexes:[Index]; checks:[CheckConstraint]; collation:Collation; + + // this field is necessary because older dolt clients weren't using TryAccessor for Columns, but are in TableSchemas + has_features_after_try_accessors:bool; } table Column { @@ -55,6 +58,9 @@ table Column { hidden:bool; generated:bool; virtual:bool; + + // sql on update value + on_update_value:string; } table Index {