diff --git a/go.mod b/go.mod index 954a5c27d1f..3b69f9eb8d1 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/krishicks/yaml-patch v0.0.10 github.com/magiconair/properties v1.8.1 github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 + github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.0 // indirect github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 github.com/onsi/ginkgo v1.10.3 // indirect diff --git a/go.sum b/go.sum index 1c489ca43dc..3915dd2de43 100644 --- a/go.sum +++ b/go.sum @@ -378,6 +378,8 @@ github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1/go.mod h1:vuvdOZLJu github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.14.0 h1:/x0XQ6h+3U3nAyk1yx+bHPURrKa9sVVvYbuqZ7pIAtI= github.com/mitchellh/go-testing-interface v1.14.0/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= diff --git a/go/test/endtoend/messaging/message_test.go b/go/test/endtoend/messaging/message_test.go index 8de8c808186..0a2aa46e381 100644 --- a/go/test/endtoend/messaging/message_test.go +++ b/go/test/endtoend/messaging/message_test.go @@ -69,6 +69,8 @@ func TestMessage(t *testing.T) { exec(t, conn, fmt.Sprintf("use %s", lookupKeyspace)) exec(t, conn, createMessage) + clusterInstance.VtctlProcess.ExecuteCommand(fmt.Sprintf("ReloadSchemaKeyspace %s", lookupKeyspace)) + defer exec(t, conn, "drop table vitess_message") exec(t, streamConn, "set workload = 'olap'") diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index 3e3b47d8c96..d77328a0018 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -517,8 +517,8 @@ func CreateVReplicationTable() []string { } // AlterVReplicationTable adds new columns to vreplication table -func AlterVReplicationTable() []string { - return []string{"ALTER TABLE _vt.vreplication ADD COLUMN db_name VARBINARY(255) NOT NULL"} +var AlterVReplicationTable = []string{ + "ALTER TABLE _vt.vreplication ADD COLUMN db_name VARBINARY(255) NOT NULL", } // VRSettings contains the settings of a vreplication table. diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 564619addf2..b7fbb6901a7 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -86,6 +86,7 @@ const ( // GTIDs. VEventType_VGTID VEventType = 15 VEventType_JOURNAL VEventType = 16 + VEventType_VERSION VEventType = 17 ) var VEventType_name = map[int32]string{ @@ -106,6 +107,7 @@ var VEventType_name = map[int32]string{ 14: "HEARTBEAT", 15: "VGTID", 16: "JOURNAL", + 17: "VERSION", } var VEventType_value = map[string]int32{ @@ -126,6 +128,7 @@ var VEventType_value = map[string]int32{ "HEARTBEAT": 14, "VGTID": 15, "JOURNAL": 16, + "VERSION": 17, } func (x VEventType) String() string { @@ -1376,6 +1379,100 @@ func (m *VEvent) GetCurrentTime() int64 { return 0 } +type MinimalTable struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Fields []*query.Field `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty"` + PKColumns []int64 `protobuf:"varint,3,rep,packed,name=p_k_columns,json=pKColumns,proto3" json:"p_k_columns,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MinimalTable) Reset() { *m = MinimalTable{} } +func (m *MinimalTable) String() string { return proto.CompactTextString(m) } +func (*MinimalTable) ProtoMessage() {} +func (*MinimalTable) Descriptor() ([]byte, []int) { + return fileDescriptor_5fd02bcb2e350dad, []int{17} +} + +func (m *MinimalTable) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MinimalTable.Unmarshal(m, b) +} +func (m *MinimalTable) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MinimalTable.Marshal(b, m, deterministic) +} +func (m *MinimalTable) XXX_Merge(src proto.Message) { + xxx_messageInfo_MinimalTable.Merge(m, src) +} +func (m *MinimalTable) XXX_Size() int { + return xxx_messageInfo_MinimalTable.Size(m) +} +func (m *MinimalTable) XXX_DiscardUnknown() { + xxx_messageInfo_MinimalTable.DiscardUnknown(m) +} + +var xxx_messageInfo_MinimalTable proto.InternalMessageInfo + +func (m *MinimalTable) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *MinimalTable) GetFields() []*query.Field { + if m != nil { + return m.Fields + } + return nil +} + +func (m *MinimalTable) GetPKColumns() []int64 { + if m != nil { + return m.PKColumns + } + return nil +} + +type MinimalSchema struct { + Tables []*MinimalTable `protobuf:"bytes,1,rep,name=tables,proto3" json:"tables,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MinimalSchema) Reset() { *m = MinimalSchema{} } +func (m *MinimalSchema) String() string { return proto.CompactTextString(m) } +func (*MinimalSchema) ProtoMessage() {} +func (*MinimalSchema) Descriptor() ([]byte, []int) { + return fileDescriptor_5fd02bcb2e350dad, []int{18} +} + +func (m *MinimalSchema) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MinimalSchema.Unmarshal(m, b) +} +func (m *MinimalSchema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MinimalSchema.Marshal(b, m, deterministic) +} +func (m *MinimalSchema) XXX_Merge(src proto.Message) { + xxx_messageInfo_MinimalSchema.Merge(m, src) +} +func (m *MinimalSchema) XXX_Size() int { + return xxx_messageInfo_MinimalSchema.Size(m) +} +func (m *MinimalSchema) XXX_DiscardUnknown() { + xxx_messageInfo_MinimalSchema.DiscardUnknown(m) +} + +var xxx_messageInfo_MinimalSchema proto.InternalMessageInfo + +func (m *MinimalSchema) GetTables() []*MinimalTable { + if m != nil { + return m.Tables + } + return nil +} + // VStreamRequest is the payload for VStreamer type VStreamRequest struct { EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId,proto3" json:"effective_caller_id,omitempty"` @@ -1392,7 +1489,7 @@ func (m *VStreamRequest) Reset() { *m = VStreamRequest{} } func (m *VStreamRequest) String() string { return proto.CompactTextString(m) } func (*VStreamRequest) ProtoMessage() {} func (*VStreamRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5fd02bcb2e350dad, []int{17} + return fileDescriptor_5fd02bcb2e350dad, []int{19} } func (m *VStreamRequest) XXX_Unmarshal(b []byte) error { @@ -1460,7 +1557,7 @@ func (m *VStreamResponse) Reset() { *m = VStreamResponse{} } func (m *VStreamResponse) String() string { return proto.CompactTextString(m) } func (*VStreamResponse) ProtoMessage() {} func (*VStreamResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5fd02bcb2e350dad, []int{18} + return fileDescriptor_5fd02bcb2e350dad, []int{20} } func (m *VStreamResponse) XXX_Unmarshal(b []byte) error { @@ -1504,7 +1601,7 @@ func (m *VStreamRowsRequest) Reset() { *m = VStreamRowsRequest{} } func (m *VStreamRowsRequest) String() string { return proto.CompactTextString(m) } func (*VStreamRowsRequest) ProtoMessage() {} func (*VStreamRowsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5fd02bcb2e350dad, []int{19} + return fileDescriptor_5fd02bcb2e350dad, []int{21} } func (m *VStreamRowsRequest) XXX_Unmarshal(b []byte) error { @@ -1576,7 +1673,7 @@ func (m *VStreamRowsResponse) Reset() { *m = VStreamRowsResponse{} } func (m *VStreamRowsResponse) String() string { return proto.CompactTextString(m) } func (*VStreamRowsResponse) ProtoMessage() {} func (*VStreamRowsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5fd02bcb2e350dad, []int{20} + return fileDescriptor_5fd02bcb2e350dad, []int{22} } func (m *VStreamRowsResponse) XXX_Unmarshal(b []byte) error { @@ -1634,6 +1731,7 @@ func (m *VStreamRowsResponse) GetLastpk() *query.Row { // VStreamResultsRequest is the payload for VStreamResults // The ids match VStreamRows, in case we decide to merge the two. +// The ids match VStreamRows, in case we decide to merge the two. type VStreamResultsRequest struct { EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId,proto3" json:"effective_caller_id,omitempty"` ImmediateCallerId *query.VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId,proto3" json:"immediate_caller_id,omitempty"` @@ -1648,7 +1746,7 @@ func (m *VStreamResultsRequest) Reset() { *m = VStreamResultsRequest{} } func (m *VStreamResultsRequest) String() string { return proto.CompactTextString(m) } func (*VStreamResultsRequest) ProtoMessage() {} func (*VStreamResultsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_5fd02bcb2e350dad, []int{21} + return fileDescriptor_5fd02bcb2e350dad, []int{23} } func (m *VStreamResultsRequest) XXX_Unmarshal(b []byte) error { @@ -1712,7 +1810,7 @@ func (m *VStreamResultsResponse) Reset() { *m = VStreamResultsResponse{} func (m *VStreamResultsResponse) String() string { return proto.CompactTextString(m) } func (*VStreamResultsResponse) ProtoMessage() {} func (*VStreamResultsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5fd02bcb2e350dad, []int{22} + return fileDescriptor_5fd02bcb2e350dad, []int{24} } func (m *VStreamResultsResponse) XXX_Unmarshal(b []byte) error { @@ -1778,6 +1876,8 @@ func init() { proto.RegisterType((*KeyspaceShard)(nil), "binlogdata.KeyspaceShard") proto.RegisterType((*Journal)(nil), "binlogdata.Journal") proto.RegisterType((*VEvent)(nil), "binlogdata.VEvent") + proto.RegisterType((*MinimalTable)(nil), "binlogdata.MinimalTable") + proto.RegisterType((*MinimalSchema)(nil), "binlogdata.MinimalSchema") proto.RegisterType((*VStreamRequest)(nil), "binlogdata.VStreamRequest") proto.RegisterType((*VStreamResponse)(nil), "binlogdata.VStreamResponse") proto.RegisterType((*VStreamRowsRequest)(nil), "binlogdata.VStreamRowsRequest") @@ -1789,112 +1889,117 @@ func init() { func init() { proto.RegisterFile("binlogdata.proto", fileDescriptor_5fd02bcb2e350dad) } var fileDescriptor_5fd02bcb2e350dad = []byte{ - // 1709 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0x4b, 0x73, 0x23, 0x49, - 0x11, 0x9e, 0xd6, 0x5b, 0xd9, 0xb6, 0xdc, 0x2e, 0x3f, 0x10, 0x13, 0x2c, 0xe1, 0xed, 0x60, 0x76, - 0xbc, 0x8e, 0x40, 0x06, 0x01, 0xc3, 0x69, 0x59, 0xf4, 0x68, 0x7b, 0x34, 0xd3, 0x92, 0x3c, 0xa5, - 0x1e, 0x0f, 0xb1, 0x97, 0x8e, 0xb6, 0x54, 0xf6, 0x34, 0xee, 0xd7, 0x74, 0x97, 0xec, 0xd5, 0x0f, - 0x20, 0xf8, 0x01, 0xfc, 0x0a, 0xce, 0x5c, 0xe1, 0xca, 0x9d, 0x3b, 0x57, 0x4e, 0x9c, 0xf8, 0x07, - 0x44, 0x3d, 0xba, 0xd5, 0xad, 0x59, 0x76, 0x3c, 0x1b, 0xc1, 0x01, 0x2e, 0x8a, 0xac, 0xec, 0xcc, - 0xac, 0xcc, 0x2f, 0x1f, 0x55, 0x25, 0xd0, 0xae, 0xdc, 0xc0, 0x0b, 0x6f, 0x16, 0x0e, 0x75, 0x3a, - 0x51, 0x1c, 0xd2, 0x10, 0xc1, 0x9a, 0xf3, 0x58, 0xbd, 0xa3, 0x71, 0x34, 0x17, 0x1f, 0x1e, 0xab, - 0xef, 0x96, 0x24, 0x5e, 0xc9, 0x45, 0x8b, 0x86, 0x51, 0xb8, 0xd6, 0xd2, 0xc7, 0x50, 0x1f, 0xbc, - 0x75, 0xe2, 0x84, 0x50, 0x74, 0x08, 0xb5, 0xb9, 0xe7, 0x92, 0x80, 0xb6, 0x95, 0x23, 0xe5, 0xb8, - 0x8a, 0xe5, 0x0a, 0x21, 0xa8, 0xcc, 0xc3, 0x20, 0x68, 0x97, 0x38, 0x97, 0xd3, 0x4c, 0x36, 0x21, - 0xf1, 0x1d, 0x89, 0xdb, 0x65, 0x21, 0x2b, 0x56, 0xfa, 0x3f, 0xca, 0xb0, 0xdb, 0xe7, 0x7e, 0x58, - 0xb1, 0x13, 0x24, 0xce, 0x9c, 0xba, 0x61, 0x80, 0xce, 0x01, 0x12, 0xea, 0x50, 0xe2, 0x93, 0x80, - 0x26, 0x6d, 0xe5, 0xa8, 0x7c, 0xac, 0x76, 0x9f, 0x76, 0x72, 0x11, 0xbc, 0xa7, 0xd2, 0x99, 0xa5, - 0xf2, 0x38, 0xa7, 0x8a, 0xba, 0xa0, 0x92, 0x3b, 0x12, 0x50, 0x9b, 0x86, 0xb7, 0x24, 0x68, 0x57, - 0x8e, 0x94, 0x63, 0xb5, 0xbb, 0xdb, 0x11, 0x01, 0x1a, 0xec, 0x8b, 0xc5, 0x3e, 0x60, 0x20, 0x19, - 0xfd, 0xf8, 0xaf, 0x25, 0x68, 0x66, 0xd6, 0x90, 0x09, 0x8d, 0xb9, 0x43, 0xc9, 0x4d, 0x18, 0xaf, - 0x78, 0x98, 0xad, 0xee, 0x4f, 0x1e, 0xe8, 0x48, 0x67, 0x20, 0xf5, 0x70, 0x66, 0x01, 0xfd, 0x18, - 0xea, 0x73, 0x81, 0x1e, 0x47, 0x47, 0xed, 0xee, 0xe5, 0x8d, 0x49, 0x60, 0x71, 0x2a, 0x83, 0x34, - 0x28, 0x27, 0xef, 0x3c, 0x0e, 0xd9, 0x16, 0x66, 0xa4, 0xfe, 0x47, 0x05, 0x1a, 0xa9, 0x5d, 0xb4, - 0x07, 0x3b, 0x7d, 0xd3, 0x7e, 0x3d, 0xc1, 0xc6, 0x60, 0x7a, 0x3e, 0x19, 0x7d, 0x65, 0x0c, 0xb5, - 0x47, 0x68, 0x0b, 0x1a, 0x7d, 0xd3, 0xee, 0x1b, 0xe7, 0xa3, 0x89, 0xa6, 0xa0, 0x6d, 0x68, 0xf6, - 0x4d, 0x7b, 0x30, 0x1d, 0x8f, 0x47, 0x96, 0x56, 0x42, 0x3b, 0xa0, 0xf6, 0x4d, 0x1b, 0x4f, 0x4d, - 0xb3, 0xdf, 0x1b, 0xbc, 0xd4, 0xca, 0xe8, 0x00, 0x76, 0xfb, 0xa6, 0x3d, 0x1c, 0x9b, 0xf6, 0xd0, - 0xb8, 0xc0, 0xc6, 0xa0, 0x67, 0x19, 0x43, 0xad, 0x82, 0x00, 0x6a, 0x8c, 0x3d, 0x34, 0xb5, 0xaa, - 0xa4, 0x67, 0x86, 0xa5, 0xd5, 0xa4, 0xb9, 0xd1, 0x64, 0x66, 0x60, 0x4b, 0xab, 0xcb, 0xe5, 0xeb, - 0x8b, 0x61, 0xcf, 0x32, 0xb4, 0x86, 0x5c, 0x0e, 0x0d, 0xd3, 0xb0, 0x0c, 0xad, 0xf9, 0xa2, 0xd2, - 0x28, 0x69, 0xe5, 0x17, 0x95, 0x46, 0x59, 0xab, 0xe8, 0x7f, 0x50, 0xe0, 0x60, 0x46, 0x63, 0xe2, - 0xf8, 0x2f, 0xc9, 0x0a, 0x3b, 0xc1, 0x0d, 0xc1, 0xe4, 0xdd, 0x92, 0x24, 0x14, 0x3d, 0x86, 0x46, - 0x14, 0x26, 0x2e, 0xc3, 0x8e, 0x03, 0xdc, 0xc4, 0xd9, 0x1a, 0x9d, 0x42, 0xf3, 0x96, 0xac, 0xec, - 0x98, 0xc9, 0x4b, 0xc0, 0x50, 0x27, 0x2b, 0xc8, 0xcc, 0x52, 0xe3, 0x56, 0x52, 0x79, 0x7c, 0xcb, - 0x1f, 0xc6, 0x57, 0xbf, 0x86, 0xc3, 0x4d, 0xa7, 0x92, 0x28, 0x0c, 0x12, 0x82, 0x4c, 0x40, 0x42, - 0xd1, 0xa6, 0xeb, 0xdc, 0x72, 0xff, 0xd4, 0xee, 0x27, 0xdf, 0x5a, 0x00, 0x78, 0xf7, 0x6a, 0x93, - 0xa5, 0x7f, 0x0d, 0x7b, 0x62, 0x1f, 0xcb, 0xb9, 0xf2, 0x48, 0xf2, 0x90, 0xd0, 0x0f, 0xa1, 0x46, - 0xb9, 0x70, 0xbb, 0x74, 0x54, 0x3e, 0x6e, 0x62, 0xb9, 0xfa, 0xd8, 0x08, 0x17, 0xb0, 0x5f, 0xdc, - 0xf9, 0xbf, 0x12, 0xdf, 0xcf, 0xa1, 0x82, 0x97, 0x1e, 0x41, 0xfb, 0x50, 0xf5, 0x1d, 0x3a, 0x7f, - 0x2b, 0xa3, 0x11, 0x0b, 0x16, 0xca, 0xb5, 0xeb, 0x51, 0x12, 0xf3, 0x14, 0x36, 0xb1, 0x5c, 0xe9, - 0x7f, 0x52, 0xa0, 0x76, 0xc6, 0x49, 0xf4, 0x19, 0x54, 0xe3, 0x25, 0x0b, 0x56, 0xf4, 0xba, 0x96, - 0xf7, 0x80, 0x59, 0xc6, 0xe2, 0x33, 0x1a, 0x41, 0xeb, 0xda, 0x25, 0xde, 0x82, 0xb7, 0xee, 0x38, - 0x5c, 0x88, 0xaa, 0x68, 0x75, 0x3f, 0xcd, 0x2b, 0x08, 0x9b, 0x9d, 0xb3, 0x82, 0x20, 0xde, 0x50, - 0xd4, 0x9f, 0x41, 0xab, 0x28, 0xc1, 0xda, 0xc9, 0xc0, 0xd8, 0x9e, 0x4e, 0xec, 0xf1, 0x68, 0x36, - 0xee, 0x59, 0x83, 0xe7, 0xda, 0x23, 0xde, 0x31, 0xc6, 0xcc, 0xb2, 0x8d, 0xb3, 0xb3, 0x29, 0xb6, - 0x34, 0x45, 0xff, 0x67, 0x09, 0xb6, 0x04, 0x28, 0xb3, 0x70, 0x19, 0xcf, 0x09, 0xcb, 0xe2, 0x2d, - 0x59, 0x25, 0x91, 0x33, 0x27, 0x69, 0x16, 0xd3, 0x35, 0x03, 0x24, 0x79, 0xeb, 0xc4, 0x0b, 0x19, - 0xb9, 0x58, 0xa0, 0x5f, 0x80, 0xca, 0xb3, 0x49, 0x6d, 0xba, 0x8a, 0x08, 0xcf, 0x63, 0xab, 0xbb, - 0xbf, 0x2e, 0x6c, 0x9e, 0x2b, 0x6a, 0xad, 0x22, 0x82, 0x81, 0x66, 0x74, 0xb1, 0x1b, 0x2a, 0x0f, - 0xe8, 0x86, 0x75, 0x0d, 0x55, 0x0b, 0x35, 0x74, 0x92, 0x25, 0xa4, 0x26, 0xad, 0xbc, 0x87, 0x5e, - 0x9a, 0x24, 0xd4, 0x81, 0x5a, 0x18, 0xd8, 0x8b, 0x85, 0xd7, 0xae, 0x73, 0x37, 0xbf, 0x97, 0x97, - 0x9d, 0x06, 0xc3, 0xa1, 0xd9, 0x13, 0x65, 0x51, 0x0d, 0x83, 0xe1, 0xc2, 0x43, 0x4f, 0xa0, 0x45, - 0xbe, 0xa6, 0x24, 0x0e, 0x1c, 0xcf, 0xf6, 0x57, 0x6c, 0x7a, 0x35, 0x78, 0xe8, 0xdb, 0x29, 0x77, - 0xcc, 0x98, 0xe8, 0x33, 0xd8, 0x49, 0x68, 0x18, 0xd9, 0xce, 0x35, 0x25, 0xb1, 0x3d, 0x0f, 0xa3, - 0x55, 0xbb, 0x79, 0xa4, 0x1c, 0x37, 0xf0, 0x36, 0x63, 0xf7, 0x18, 0x77, 0x10, 0x46, 0x2b, 0xfd, - 0x15, 0x34, 0x71, 0x78, 0x3f, 0x78, 0xcb, 0xe3, 0xd1, 0xa1, 0x76, 0x45, 0xae, 0xc3, 0x98, 0xc8, - 0x42, 0x05, 0x39, 0xc8, 0x71, 0x78, 0x8f, 0xe5, 0x17, 0x74, 0x04, 0x55, 0x6e, 0x53, 0x8e, 0x8b, - 0xbc, 0x88, 0xf8, 0xa0, 0x3b, 0xd0, 0xc0, 0xe1, 0x3d, 0x4f, 0x3b, 0xfa, 0x04, 0x04, 0xc0, 0x76, - 0xe0, 0xf8, 0x69, 0xf6, 0x9a, 0x9c, 0x33, 0x71, 0x7c, 0x82, 0x9e, 0x81, 0x1a, 0x87, 0xf7, 0xf6, - 0x9c, 0x6f, 0x2f, 0x3a, 0x51, 0xed, 0x1e, 0x14, 0x8a, 0x33, 0x75, 0x0e, 0x43, 0x9c, 0x92, 0x89, - 0xfe, 0x0a, 0x60, 0x5d, 0x5b, 0x1f, 0xda, 0xe4, 0x47, 0x2c, 0x1b, 0xc4, 0x5b, 0xa4, 0xf6, 0xb7, - 0xa4, 0xcb, 0xdc, 0x02, 0x96, 0xdf, 0x18, 0x10, 0x33, 0x56, 0x3c, 0xe7, 0xd4, 0x5d, 0x7c, 0x87, - 0x92, 0x43, 0x50, 0xb9, 0xa1, 0xee, 0x82, 0xd7, 0x5a, 0x13, 0x73, 0x5a, 0xff, 0x12, 0xaa, 0x97, - 0xdc, 0xdc, 0x33, 0x50, 0xb9, 0x94, 0xcd, 0xd8, 0x69, 0x0f, 0x16, 0xc2, 0xcc, 0xb6, 0xc6, 0x90, - 0xa4, 0x64, 0xa2, 0xf7, 0x60, 0xfb, 0xa5, 0xdc, 0x96, 0x0b, 0x7c, 0xbc, 0x5f, 0xfa, 0x9f, 0x4b, - 0x50, 0x7f, 0x11, 0x2e, 0x59, 0x61, 0xa0, 0x16, 0x94, 0xdc, 0x05, 0xd7, 0x2b, 0xe3, 0x92, 0xbb, - 0x40, 0xbf, 0x86, 0x96, 0xef, 0xde, 0xc4, 0x0e, 0x2b, 0x2f, 0xd1, 0x29, 0xa2, 0xd9, 0xbf, 0x9f, - 0xf7, 0x6c, 0x9c, 0x4a, 0xf0, 0x76, 0xd9, 0xf6, 0xf3, 0xcb, 0x5c, 0x03, 0x94, 0x0b, 0x0d, 0xf0, - 0x04, 0x5a, 0x5e, 0x38, 0x77, 0x3c, 0x3b, 0x1b, 0xbf, 0x15, 0x51, 0xa4, 0x9c, 0x7b, 0x91, 0xce, - 0xe0, 0x0d, 0x5c, 0xaa, 0x0f, 0xc4, 0x05, 0x7d, 0x01, 0x5b, 0x91, 0x13, 0x53, 0x77, 0xee, 0x46, - 0x0e, 0xbb, 0xc0, 0xd4, 0xb8, 0x62, 0xc1, 0xed, 0x02, 0x6e, 0xb8, 0x20, 0x8e, 0x3e, 0x07, 0x2d, - 0xe1, 0xa3, 0xc5, 0xbe, 0x0f, 0xe3, 0xdb, 0x6b, 0x2f, 0xbc, 0x4f, 0xda, 0x75, 0xee, 0xff, 0x8e, - 0xe0, 0xbf, 0x49, 0xd9, 0xfa, 0xbf, 0x4a, 0x50, 0xbb, 0x14, 0x55, 0x76, 0x02, 0x15, 0x8e, 0x91, - 0xb8, 0xa4, 0x1c, 0xe6, 0x37, 0x13, 0x12, 0x1c, 0x20, 0x2e, 0x83, 0x7e, 0x00, 0x4d, 0xea, 0xfa, - 0x24, 0xa1, 0x8e, 0x1f, 0x71, 0x50, 0xcb, 0x78, 0xcd, 0xf8, 0xa6, 0x5a, 0x61, 0x37, 0x11, 0x36, - 0x03, 0x04, 0x4c, 0x8c, 0x44, 0x3f, 0x85, 0x26, 0xeb, 0x0d, 0x7e, 0x71, 0x6a, 0x57, 0x79, 0xb3, - 0xed, 0x6f, 0x74, 0x06, 0xdf, 0x16, 0x37, 0xe2, 0xb4, 0xdb, 0x7e, 0x09, 0x2a, 0xaf, 0x66, 0xa9, - 0x24, 0x86, 0xcf, 0x61, 0x71, 0xf8, 0xa4, 0x5d, 0x83, 0x61, 0x3d, 0xaf, 0xd1, 0x53, 0xa8, 0xde, - 0x71, 0x97, 0xea, 0xf2, 0x02, 0x97, 0x0f, 0x8e, 0xc3, 0x2f, 0xbe, 0xb3, 0xd3, 0xf1, 0xb7, 0xa2, - 0x9a, 0xf8, 0xd8, 0xd9, 0x38, 0x1d, 0x65, 0xa1, 0xe1, 0x54, 0x86, 0x47, 0xe5, 0x7b, 0x7c, 0xf2, - 0xb0, 0xa8, 0x7c, 0x0f, 0x7d, 0x0a, 0x5b, 0xf3, 0x65, 0x1c, 0xf3, 0x2b, 0xa3, 0xeb, 0x93, 0xf6, - 0x3e, 0x07, 0x47, 0x95, 0x3c, 0xcb, 0xf5, 0x89, 0xfe, 0xfb, 0x12, 0xb4, 0x2e, 0xc5, 0xa1, 0x9a, - 0x1e, 0xe4, 0x5f, 0xc2, 0x1e, 0xb9, 0xbe, 0x26, 0x73, 0xea, 0xde, 0x11, 0x7b, 0xee, 0x78, 0x1e, - 0x89, 0x6d, 0x59, 0xca, 0x6a, 0x77, 0xa7, 0x23, 0x2e, 0xd7, 0x03, 0xce, 0x1f, 0x0d, 0xf1, 0x6e, - 0x26, 0x2b, 0x59, 0x0b, 0x64, 0xc0, 0x9e, 0xeb, 0xfb, 0x64, 0xe1, 0x3a, 0x34, 0x6f, 0x40, 0xcc, - 0xb0, 0x03, 0x39, 0x10, 0x2e, 0xad, 0x73, 0x87, 0x92, 0xb5, 0x99, 0x4c, 0x23, 0x33, 0xf3, 0x84, - 0xd5, 0x7b, 0x7c, 0x93, 0xdd, 0x0d, 0xb6, 0xa5, 0xa6, 0xc5, 0x99, 0x58, 0x7e, 0x2c, 0xdc, 0x3b, - 0x2a, 0x1b, 0xf7, 0x8e, 0xf5, 0xd9, 0x50, 0xfd, 0xd0, 0xd9, 0xa0, 0x7f, 0x01, 0x3b, 0x19, 0x10, - 0xf2, 0x5e, 0x71, 0x02, 0x35, 0x9e, 0xdc, 0x74, 0x8a, 0xa0, 0xf7, 0xeb, 0x10, 0x4b, 0x09, 0xfd, - 0x77, 0x25, 0x40, 0xa9, 0x7e, 0x78, 0x9f, 0xfc, 0x8f, 0x82, 0xb9, 0x0f, 0x55, 0xce, 0x97, 0x48, - 0x8a, 0x05, 0xc3, 0xc1, 0x73, 0x12, 0x1a, 0xdd, 0x66, 0x30, 0x0a, 0xe5, 0x57, 0xec, 0x17, 0x93, - 0x64, 0xe9, 0x51, 0x2c, 0x25, 0xf4, 0xbf, 0x28, 0xb0, 0x57, 0xc0, 0x41, 0x62, 0xb9, 0x3e, 0x18, - 0x94, 0xff, 0x7c, 0x30, 0xa0, 0x63, 0x68, 0x44, 0xb7, 0xdf, 0x72, 0x80, 0x64, 0x5f, 0xbf, 0xb1, - 0xaf, 0x7f, 0x08, 0x95, 0x98, 0xcd, 0x97, 0x0a, 0xd7, 0xcc, 0x9f, 0x96, 0x9c, 0xcf, 0x8e, 0xdc, - 0x42, 0x1c, 0x85, 0x23, 0x57, 0xfa, 0xff, 0x77, 0x05, 0x0e, 0xd6, 0x75, 0xb0, 0xf4, 0xe8, 0xff, - 0x55, 0x2a, 0xf5, 0x18, 0x0e, 0x37, 0xa3, 0xfb, 0xa8, 0x04, 0x7d, 0x07, 0xd8, 0x4f, 0x7e, 0x05, - 0x6a, 0xee, 0x6e, 0xc5, 0x9e, 0x60, 0xa3, 0xf3, 0xc9, 0x14, 0x1b, 0xda, 0x23, 0xd4, 0x80, 0xca, - 0xcc, 0x9a, 0x5e, 0x68, 0x0a, 0xa3, 0x8c, 0xdf, 0x18, 0x03, 0xf1, 0xac, 0x63, 0x94, 0x2d, 0x85, - 0xca, 0x27, 0x7f, 0x53, 0x00, 0xd6, 0x53, 0x1f, 0xa9, 0x50, 0x7f, 0x3d, 0x79, 0x39, 0x99, 0xbe, - 0x99, 0x08, 0x03, 0xe7, 0xd6, 0x68, 0xa8, 0x29, 0xa8, 0x09, 0x55, 0xf1, 0x4e, 0x2c, 0xb1, 0x1d, - 0xe4, 0x23, 0xb1, 0xcc, 0x5e, 0x90, 0xd9, 0x0b, 0xb1, 0x82, 0xea, 0x50, 0xce, 0xde, 0x81, 0xf2, - 0xe1, 0x57, 0x63, 0x06, 0xb1, 0x71, 0x61, 0xf6, 0x06, 0x86, 0x56, 0x67, 0x1f, 0xb2, 0x27, 0x20, - 0x40, 0x2d, 0x7d, 0xff, 0x31, 0x4d, 0xf6, 0x6a, 0x04, 0xb6, 0xcf, 0xd4, 0x7a, 0x6e, 0x60, 0x4d, - 0x65, 0x3c, 0x3c, 0x7d, 0xa3, 0x6d, 0x31, 0xde, 0xd9, 0xc8, 0x30, 0x87, 0xda, 0x36, 0x7b, 0x36, - 0x3e, 0x37, 0x7a, 0xd8, 0xea, 0x1b, 0x3d, 0x4b, 0x6b, 0xb1, 0x2f, 0x97, 0xdc, 0xc1, 0x1d, 0xb6, - 0xcd, 0x8b, 0xe9, 0x6b, 0x3c, 0xe9, 0x99, 0x9a, 0x76, 0xf2, 0x14, 0xb6, 0x0b, 0x87, 0x3d, 0xdb, - 0xcb, 0xea, 0xf5, 0x4d, 0x63, 0xa6, 0x3d, 0x62, 0xf4, 0xec, 0x79, 0x0f, 0x0f, 0x67, 0x9a, 0xd2, - 0xff, 0xfc, 0xab, 0xa7, 0x77, 0x2e, 0x25, 0x49, 0xd2, 0x71, 0xc3, 0x53, 0x41, 0x9d, 0xde, 0x84, - 0xa7, 0x77, 0xf4, 0x94, 0xff, 0x85, 0x71, 0xba, 0x9e, 0x48, 0x57, 0x35, 0xce, 0xf9, 0xd9, 0xbf, - 0x03, 0x00, 0x00, 0xff, 0xff, 0x78, 0x33, 0x5b, 0xba, 0x1e, 0x11, 0x00, 0x00, + // 1779 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x72, 0xdb, 0xc8, + 0x11, 0x36, 0xff, 0xc9, 0x86, 0x44, 0x41, 0x23, 0x59, 0x61, 0x5c, 0xd9, 0x2d, 0x2d, 0x2a, 0x5e, + 0x6b, 0x55, 0x15, 0x6a, 0xc3, 0x24, 0xce, 0x69, 0xb3, 0xe1, 0x0f, 0x24, 0xd3, 0x02, 0x49, 0x79, + 0x08, 0xcb, 0xa9, 0xbd, 0xa0, 0x20, 0x70, 0x24, 0x21, 0xc2, 0x9f, 0x81, 0xa1, 0xb4, 0x7c, 0x80, + 0x54, 0x1e, 0x20, 0x4f, 0x91, 0x73, 0xae, 0xc9, 0x35, 0x4f, 0x91, 0xca, 0x2d, 0xa7, 0x9c, 0xf2, + 0x06, 0xa9, 0xf9, 0x01, 0x08, 0xc8, 0x5b, 0xb6, 0xbc, 0x55, 0x39, 0x64, 0x2f, 0xac, 0x9e, 0x9e, + 0xee, 0x9e, 0xee, 0x6f, 0xba, 0x1b, 0x3d, 0x04, 0xf5, 0xc2, 0x0d, 0xbc, 0xf0, 0x6a, 0x61, 0x53, + 0xbb, 0x1b, 0xc5, 0x21, 0x0d, 0x11, 0xac, 0x39, 0x4f, 0x94, 0x5b, 0x1a, 0x47, 0x8e, 0xd8, 0x78, + 0xa2, 0xbc, 0x5d, 0x92, 0x78, 0x25, 0x17, 0x6d, 0x1a, 0x46, 0xe1, 0x5a, 0x4b, 0x9b, 0x40, 0x63, + 0x78, 0x6d, 0xc7, 0x09, 0xa1, 0x68, 0x0f, 0xea, 0x8e, 0xe7, 0x92, 0x80, 0x76, 0x4a, 0xfb, 0xa5, + 0x83, 0x1a, 0x96, 0x2b, 0x84, 0xa0, 0xea, 0x84, 0x41, 0xd0, 0x29, 0x73, 0x2e, 0xa7, 0x99, 0x6c, + 0x42, 0xe2, 0x5b, 0x12, 0x77, 0x2a, 0x42, 0x56, 0xac, 0xb4, 0x7f, 0x55, 0x60, 0x7b, 0xc0, 0xfd, + 0x30, 0x63, 0x3b, 0x48, 0x6c, 0x87, 0xba, 0x61, 0x80, 0x4e, 0x00, 0x12, 0x6a, 0x53, 0xe2, 0x93, + 0x80, 0x26, 0x9d, 0xd2, 0x7e, 0xe5, 0x40, 0xe9, 0x3d, 0xeb, 0xe6, 0x22, 0x78, 0x47, 0xa5, 0x3b, + 0x4f, 0xe5, 0x71, 0x4e, 0x15, 0xf5, 0x40, 0x21, 0xb7, 0x24, 0xa0, 0x16, 0x0d, 0x6f, 0x48, 0xd0, + 0xa9, 0xee, 0x97, 0x0e, 0x94, 0xde, 0x76, 0x57, 0x04, 0xa8, 0xb3, 0x1d, 0x93, 0x6d, 0x60, 0x20, + 0x19, 0xfd, 0xe4, 0xef, 0x65, 0x68, 0x65, 0xd6, 0x90, 0x01, 0x4d, 0xc7, 0xa6, 0xe4, 0x2a, 0x8c, + 0x57, 0x3c, 0xcc, 0x76, 0xef, 0xcb, 0x07, 0x3a, 0xd2, 0x1d, 0x4a, 0x3d, 0x9c, 0x59, 0x40, 0x3f, + 0x83, 0x86, 0x23, 0xd0, 0xe3, 0xe8, 0x28, 0xbd, 0x9d, 0xbc, 0x31, 0x09, 0x2c, 0x4e, 0x65, 0x90, + 0x0a, 0x95, 0xe4, 0xad, 0xc7, 0x21, 0xdb, 0xc0, 0x8c, 0xd4, 0xfe, 0x5c, 0x82, 0x66, 0x6a, 0x17, + 0xed, 0xc0, 0xd6, 0xc0, 0xb0, 0x5e, 0x4f, 0xb1, 0x3e, 0x9c, 0x9d, 0x4c, 0xc7, 0xdf, 0xe8, 0x23, + 0xf5, 0x11, 0xda, 0x80, 0xe6, 0xc0, 0xb0, 0x06, 0xfa, 0xc9, 0x78, 0xaa, 0x96, 0xd0, 0x26, 0xb4, + 0x06, 0x86, 0x35, 0x9c, 0x4d, 0x26, 0x63, 0x53, 0x2d, 0xa3, 0x2d, 0x50, 0x06, 0x86, 0x85, 0x67, + 0x86, 0x31, 0xe8, 0x0f, 0x4f, 0xd5, 0x0a, 0x7a, 0x0c, 0xdb, 0x03, 0xc3, 0x1a, 0x4d, 0x0c, 0x6b, + 0xa4, 0x9f, 0x61, 0x7d, 0xd8, 0x37, 0xf5, 0x91, 0x5a, 0x45, 0x00, 0x75, 0xc6, 0x1e, 0x19, 0x6a, + 0x4d, 0xd2, 0x73, 0xdd, 0x54, 0xeb, 0xd2, 0xdc, 0x78, 0x3a, 0xd7, 0xb1, 0xa9, 0x36, 0xe4, 0xf2, + 0xf5, 0xd9, 0xa8, 0x6f, 0xea, 0x6a, 0x53, 0x2e, 0x47, 0xba, 0xa1, 0x9b, 0xba, 0xda, 0x7a, 0x59, + 0x6d, 0x96, 0xd5, 0xca, 0xcb, 0x6a, 0xb3, 0xa2, 0x56, 0xb5, 0x3f, 0x95, 0xe0, 0xf1, 0x9c, 0xc6, + 0xc4, 0xf6, 0x4f, 0xc9, 0x0a, 0xdb, 0xc1, 0x15, 0xc1, 0xe4, 0xed, 0x92, 0x24, 0x14, 0x3d, 0x81, + 0x66, 0x14, 0x26, 0x2e, 0xc3, 0x8e, 0x03, 0xdc, 0xc2, 0xd9, 0x1a, 0x1d, 0x41, 0xeb, 0x86, 0xac, + 0xac, 0x98, 0xc9, 0x4b, 0xc0, 0x50, 0x37, 0x4b, 0xc8, 0xcc, 0x52, 0xf3, 0x46, 0x52, 0x79, 0x7c, + 0x2b, 0x1f, 0xc6, 0x57, 0xbb, 0x84, 0xbd, 0xfb, 0x4e, 0x25, 0x51, 0x18, 0x24, 0x04, 0x19, 0x80, + 0x84, 0xa2, 0x45, 0xd7, 0x77, 0xcb, 0xfd, 0x53, 0x7a, 0x9f, 0xbc, 0x37, 0x01, 0xf0, 0xf6, 0xc5, + 0x7d, 0x96, 0xf6, 0x2d, 0xec, 0x88, 0x73, 0x4c, 0xfb, 0xc2, 0x23, 0xc9, 0x43, 0x42, 0xdf, 0x83, + 0x3a, 0xe5, 0xc2, 0x9d, 0xf2, 0x7e, 0xe5, 0xa0, 0x85, 0xe5, 0xea, 0x63, 0x23, 0x5c, 0xc0, 0x6e, + 0xf1, 0xe4, 0xff, 0x49, 0x7c, 0xbf, 0x84, 0x2a, 0x5e, 0x7a, 0x04, 0xed, 0x42, 0xcd, 0xb7, 0xa9, + 0x73, 0x2d, 0xa3, 0x11, 0x0b, 0x16, 0xca, 0xa5, 0xeb, 0x51, 0x12, 0xf3, 0x2b, 0x6c, 0x61, 0xb9, + 0xd2, 0xfe, 0x52, 0x82, 0xfa, 0x31, 0x27, 0xd1, 0xe7, 0x50, 0x8b, 0x97, 0x2c, 0x58, 0x51, 0xeb, + 0x6a, 0xde, 0x03, 0x66, 0x19, 0x8b, 0x6d, 0x34, 0x86, 0xf6, 0xa5, 0x4b, 0xbc, 0x05, 0x2f, 0xdd, + 0x49, 0xb8, 0x10, 0x59, 0xd1, 0xee, 0x7d, 0x96, 0x57, 0x10, 0x36, 0xbb, 0xc7, 0x05, 0x41, 0x7c, + 0x4f, 0x51, 0x7b, 0x0e, 0xed, 0xa2, 0x04, 0x2b, 0x27, 0x1d, 0x63, 0x6b, 0x36, 0xb5, 0x26, 0xe3, + 0xf9, 0xa4, 0x6f, 0x0e, 0x5f, 0xa8, 0x8f, 0x78, 0xc5, 0xe8, 0x73, 0xd3, 0xd2, 0x8f, 0x8f, 0x67, + 0xd8, 0x54, 0x4b, 0xda, 0xbf, 0xcb, 0xb0, 0x21, 0x40, 0x99, 0x87, 0xcb, 0xd8, 0x21, 0xec, 0x16, + 0x6f, 0xc8, 0x2a, 0x89, 0x6c, 0x87, 0xa4, 0xb7, 0x98, 0xae, 0x19, 0x20, 0xc9, 0xb5, 0x1d, 0x2f, + 0x64, 0xe4, 0x62, 0x81, 0x7e, 0x05, 0x0a, 0xbf, 0x4d, 0x6a, 0xd1, 0x55, 0x44, 0xf8, 0x3d, 0xb6, + 0x7b, 0xbb, 0xeb, 0xc4, 0xe6, 0x77, 0x45, 0xcd, 0x55, 0x44, 0x30, 0xd0, 0x8c, 0x2e, 0x56, 0x43, + 0xf5, 0x01, 0xd5, 0xb0, 0xce, 0xa1, 0x5a, 0x21, 0x87, 0x0e, 0xb3, 0x0b, 0xa9, 0x4b, 0x2b, 0xef, + 0xa0, 0x97, 0x5e, 0x12, 0xea, 0x42, 0x3d, 0x0c, 0xac, 0xc5, 0xc2, 0xeb, 0x34, 0xb8, 0x9b, 0x3f, + 0xca, 0xcb, 0xce, 0x82, 0xd1, 0xc8, 0xe8, 0x8b, 0xb4, 0xa8, 0x85, 0xc1, 0x68, 0xe1, 0xa1, 0xa7, + 0xd0, 0x26, 0xdf, 0x52, 0x12, 0x07, 0xb6, 0x67, 0xf9, 0x2b, 0xd6, 0xbd, 0x9a, 0x3c, 0xf4, 0xcd, + 0x94, 0x3b, 0x61, 0x4c, 0xf4, 0x39, 0x6c, 0x25, 0x34, 0x8c, 0x2c, 0xfb, 0x92, 0x92, 0xd8, 0x72, + 0xc2, 0x68, 0xd5, 0x69, 0xed, 0x97, 0x0e, 0x9a, 0x78, 0x93, 0xb1, 0xfb, 0x8c, 0x3b, 0x0c, 0xa3, + 0x95, 0xf6, 0x0a, 0x5a, 0x38, 0xbc, 0x1b, 0x5e, 0xf3, 0x78, 0x34, 0xa8, 0x5f, 0x90, 0xcb, 0x30, + 0x26, 0x32, 0x51, 0x41, 0x36, 0x72, 0x1c, 0xde, 0x61, 0xb9, 0x83, 0xf6, 0xa1, 0xc6, 0x6d, 0xca, + 0x76, 0x91, 0x17, 0x11, 0x1b, 0x9a, 0x0d, 0x4d, 0x1c, 0xde, 0xf1, 0x6b, 0x47, 0x9f, 0x80, 0x00, + 0xd8, 0x0a, 0x6c, 0x3f, 0xbd, 0xbd, 0x16, 0xe7, 0x4c, 0x6d, 0x9f, 0xa0, 0xe7, 0xa0, 0xc4, 0xe1, + 0x9d, 0xe5, 0xf0, 0xe3, 0x45, 0x25, 0x2a, 0xbd, 0xc7, 0x85, 0xe4, 0x4c, 0x9d, 0xc3, 0x10, 0xa7, + 0x64, 0xa2, 0xbd, 0x02, 0x58, 0xe7, 0xd6, 0x87, 0x0e, 0xf9, 0x29, 0xbb, 0x0d, 0xe2, 0x2d, 0x52, + 0xfb, 0x1b, 0xd2, 0x65, 0x6e, 0x01, 0xcb, 0x3d, 0x06, 0xc4, 0x9c, 0x25, 0xcf, 0x09, 0x75, 0x17, + 0xdf, 0x23, 0xe5, 0x10, 0x54, 0xaf, 0xa8, 0xbb, 0xe0, 0xb9, 0xd6, 0xc2, 0x9c, 0xd6, 0xbe, 0x86, + 0xda, 0x39, 0x37, 0xf7, 0x1c, 0x14, 0x2e, 0x65, 0x31, 0x76, 0x5a, 0x83, 0x85, 0x30, 0xb3, 0xa3, + 0x31, 0x24, 0x29, 0x99, 0x68, 0x7d, 0xd8, 0x3c, 0x95, 0xc7, 0x72, 0x81, 0x8f, 0xf7, 0x4b, 0xfb, + 0x6b, 0x19, 0x1a, 0x2f, 0xc3, 0x25, 0x4b, 0x0c, 0xd4, 0x86, 0xb2, 0xbb, 0xe0, 0x7a, 0x15, 0x5c, + 0x76, 0x17, 0xe8, 0xb7, 0xd0, 0xf6, 0xdd, 0xab, 0xd8, 0x66, 0xe9, 0x25, 0x2a, 0x45, 0x14, 0xfb, + 0x8f, 0xf3, 0x9e, 0x4d, 0x52, 0x09, 0x5e, 0x2e, 0x9b, 0x7e, 0x7e, 0x99, 0x2b, 0x80, 0x4a, 0xa1, + 0x00, 0x9e, 0x42, 0xdb, 0x0b, 0x1d, 0xdb, 0xb3, 0xb2, 0xf6, 0x5b, 0x15, 0x49, 0xca, 0xb9, 0x67, + 0x69, 0x0f, 0xbe, 0x87, 0x4b, 0xed, 0x81, 0xb8, 0xa0, 0xaf, 0x60, 0x23, 0xb2, 0x63, 0xea, 0x3a, + 0x6e, 0x64, 0xb3, 0x01, 0xa6, 0xce, 0x15, 0x0b, 0x6e, 0x17, 0x70, 0xc3, 0x05, 0x71, 0xf4, 0x05, + 0xa8, 0x09, 0x6f, 0x2d, 0xd6, 0x5d, 0x18, 0xdf, 0x5c, 0x7a, 0xe1, 0x5d, 0xd2, 0x69, 0x70, 0xff, + 0xb7, 0x04, 0xff, 0x4d, 0xca, 0xd6, 0xfe, 0x53, 0x86, 0xfa, 0xb9, 0xc8, 0xb2, 0x43, 0xa8, 0x72, + 0x8c, 0xc4, 0x90, 0xb2, 0x97, 0x3f, 0x4c, 0x48, 0x70, 0x80, 0xb8, 0x0c, 0xfa, 0x09, 0xb4, 0xa8, + 0xeb, 0x93, 0x84, 0xda, 0x7e, 0xc4, 0x41, 0xad, 0xe0, 0x35, 0xe3, 0xbb, 0x72, 0x85, 0x4d, 0x22, + 0xac, 0x07, 0x08, 0x98, 0x18, 0x89, 0x7e, 0x0e, 0x2d, 0x56, 0x1b, 0x7c, 0x70, 0xea, 0xd4, 0x78, + 0xb1, 0xed, 0xde, 0xab, 0x0c, 0x7e, 0x2c, 0x6e, 0xc6, 0x69, 0xb5, 0xfd, 0x1a, 0x14, 0x9e, 0xcd, + 0x52, 0x49, 0x34, 0x9f, 0xbd, 0x62, 0xf3, 0x49, 0xab, 0x06, 0xc3, 0xba, 0x5f, 0xa3, 0x67, 0x50, + 0xbb, 0xe5, 0x2e, 0x35, 0xe4, 0x00, 0x97, 0x0f, 0x8e, 0xc3, 0x2f, 0xf6, 0xd9, 0xd7, 0xf1, 0xf7, + 0x22, 0x9b, 0x78, 0xdb, 0xb9, 0xf7, 0x75, 0x94, 0x89, 0x86, 0x53, 0x19, 0x1e, 0x95, 0xef, 0xf1, + 0xce, 0xc3, 0xa2, 0xf2, 0x3d, 0xf4, 0x19, 0x6c, 0x38, 0xcb, 0x38, 0xe6, 0x23, 0xa3, 0xeb, 0x93, + 0xce, 0x2e, 0x07, 0x47, 0x91, 0x3c, 0xd3, 0xf5, 0x89, 0x76, 0x0d, 0x1b, 0x13, 0x37, 0x70, 0x7d, + 0xdb, 0xe3, 0x7d, 0x9a, 0xc1, 0x95, 0x2b, 0x6c, 0x4e, 0x3f, 0xac, 0xa6, 0xd1, 0xa7, 0xa0, 0x44, + 0xd6, 0x8d, 0xe5, 0x84, 0xde, 0xd2, 0x0f, 0x44, 0x8e, 0x56, 0x70, 0x2b, 0x3a, 0x1d, 0x0a, 0x06, + 0xab, 0x2f, 0x79, 0xd2, 0xdc, 0xb9, 0x26, 0xbe, 0x8d, 0xbe, 0xcc, 0xf2, 0x59, 0xd4, 0x68, 0xa7, + 0x58, 0x09, 0x6b, 0xa7, 0xd2, 0x4c, 0xd7, 0xfe, 0x58, 0x86, 0xf6, 0xb9, 0x98, 0x00, 0xd2, 0xa9, + 0xe3, 0x6b, 0xd8, 0x21, 0x97, 0x97, 0xc4, 0xa1, 0xee, 0x2d, 0xb1, 0x1c, 0xdb, 0xf3, 0x48, 0x6c, + 0xc9, 0xba, 0x53, 0x7a, 0x5b, 0x5d, 0xf1, 0x12, 0x18, 0x72, 0xfe, 0x78, 0x84, 0xb7, 0x33, 0x59, + 0xc9, 0x5a, 0x20, 0x1d, 0x76, 0x5c, 0xdf, 0x27, 0x0b, 0xd7, 0xa6, 0x79, 0x03, 0xa2, 0xe1, 0x3e, + 0x96, 0x91, 0x9e, 0x9b, 0x27, 0x36, 0x25, 0x6b, 0x33, 0x99, 0x46, 0x66, 0xe6, 0x29, 0x0b, 0x26, + 0xbe, 0xca, 0x06, 0x99, 0x4d, 0xa9, 0x69, 0x72, 0x26, 0x96, 0x9b, 0x85, 0x21, 0xa9, 0x7a, 0x6f, + 0x48, 0x5a, 0x7f, 0xc8, 0x6a, 0x1f, 0xfa, 0x90, 0x69, 0x5f, 0xc1, 0x56, 0x06, 0x84, 0x1c, 0x82, + 0x0e, 0xa1, 0xce, 0x33, 0x31, 0x85, 0x13, 0xbd, 0x5b, 0x34, 0x58, 0x4a, 0x68, 0x7f, 0x28, 0x03, + 0x4a, 0xf5, 0xc3, 0xbb, 0xe4, 0xff, 0x14, 0xcc, 0x5d, 0xa8, 0x71, 0xbe, 0x44, 0x52, 0x2c, 0x18, + 0x0e, 0x9e, 0x9d, 0xd0, 0xe8, 0x26, 0x83, 0x51, 0x28, 0xbf, 0x62, 0xbf, 0x98, 0x24, 0x4b, 0x8f, + 0x62, 0x29, 0xa1, 0xfd, 0xad, 0x04, 0x3b, 0x05, 0x1c, 0x24, 0x96, 0xeb, 0x8c, 0x2f, 0xbd, 0x27, + 0xe3, 0x0f, 0xa0, 0x19, 0xdd, 0xbc, 0xa7, 0x32, 0xb2, 0xdd, 0xef, 0x6c, 0x42, 0x9f, 0x42, 0x35, + 0x66, 0xcd, 0xb0, 0xca, 0x35, 0xf3, 0x9f, 0x76, 0xce, 0x67, 0xf3, 0x41, 0x21, 0x8e, 0xc2, 0x7c, + 0x20, 0xfd, 0xff, 0x47, 0x09, 0x1e, 0xaf, 0xf3, 0x60, 0xe9, 0xd1, 0x1f, 0xd4, 0x55, 0x6a, 0x31, + 0xec, 0xdd, 0x8f, 0xee, 0xa3, 0x2e, 0xe8, 0x7b, 0xc0, 0x7e, 0xf8, 0x1b, 0x50, 0x72, 0x83, 0x20, + 0x7b, 0x2f, 0x8e, 0x4f, 0xa6, 0x33, 0xac, 0xab, 0x8f, 0x50, 0x13, 0xaa, 0x73, 0x73, 0x76, 0xa6, + 0x96, 0x18, 0xa5, 0xff, 0x4e, 0x1f, 0x8a, 0x37, 0x28, 0xa3, 0x2c, 0x29, 0x54, 0x39, 0xfc, 0x67, + 0x09, 0x60, 0xfd, 0x89, 0x42, 0x0a, 0x34, 0x5e, 0x4f, 0x4f, 0xa7, 0xb3, 0x37, 0x53, 0x61, 0xe0, + 0xc4, 0x1c, 0x8f, 0xd4, 0x12, 0x6a, 0x41, 0x4d, 0x3c, 0x6a, 0xcb, 0xec, 0x04, 0xf9, 0xa2, 0xad, + 0xb0, 0xe7, 0x6e, 0xf6, 0x9c, 0xad, 0xa2, 0x06, 0x54, 0xb2, 0x47, 0xab, 0x7c, 0xa5, 0xd6, 0x99, + 0x41, 0xac, 0x9f, 0x19, 0xfd, 0xa1, 0xae, 0x36, 0xd8, 0x46, 0xf6, 0x5e, 0x05, 0xa8, 0xa7, 0x8f, + 0x55, 0xa6, 0xc9, 0x9e, 0xb8, 0xc0, 0xce, 0x99, 0x99, 0x2f, 0x74, 0xac, 0x2a, 0x8c, 0x87, 0x67, + 0x6f, 0xd4, 0x0d, 0xc6, 0x3b, 0x1e, 0xeb, 0xc6, 0x48, 0xdd, 0x64, 0x6f, 0xdc, 0x17, 0x7a, 0x1f, + 0x9b, 0x03, 0xbd, 0x6f, 0xaa, 0x6d, 0xb6, 0x73, 0xce, 0x1d, 0xdc, 0x62, 0xc7, 0xbc, 0x9c, 0xbd, + 0xc6, 0xd3, 0xbe, 0xa1, 0xaa, 0x6c, 0x71, 0xae, 0xe3, 0xf9, 0x78, 0x36, 0x55, 0xb7, 0x0f, 0x9f, + 0xb1, 0x3e, 0x9e, 0x9f, 0x4b, 0x00, 0xea, 0x66, 0x7f, 0x60, 0xe8, 0x73, 0xf5, 0x11, 0xa3, 0xe7, + 0x2f, 0xfa, 0x78, 0x34, 0x57, 0x4b, 0x83, 0x2f, 0xbe, 0x79, 0x76, 0xeb, 0x52, 0x92, 0x24, 0x5d, + 0x37, 0x3c, 0x12, 0xd4, 0xd1, 0x55, 0x78, 0x74, 0x4b, 0x8f, 0xf8, 0x9f, 0x2f, 0x47, 0xeb, 0xf6, + 0x74, 0x51, 0xe7, 0x9c, 0x5f, 0xfc, 0x37, 0x00, 0x00, 0xff, 0xff, 0x1b, 0xb2, 0xd4, 0x99, 0xd8, + 0x11, 0x00, 0x00, } diff --git a/go/vt/tableacl/tableacl.go b/go/vt/tableacl/tableacl.go index 3d8305868a7..9a21dcd3d6a 100644 --- a/go/vt/tableacl/tableacl.go +++ b/go/vt/tableacl/tableacl.go @@ -117,7 +117,7 @@ func (tacl *tableACL) init(configFile string, aclCB func()) error { } config := &tableaclpb.Config{} if err := proto.Unmarshal(data, config); err != nil { - log.Infof("unable to parse tableACL config file as a protobuf file: %v", err) + log.Infof("unable to parse tableACL config file as a protobuf file: %v, will try as json", err) // try to parse tableacl as json file if jsonErr := json2.Unmarshal(data, config); jsonErr != nil { log.Infof("unable to parse tableACL config file as a json file: %v", jsonErr) diff --git a/go/vt/vttablet/endtoend/framework/server.go b/go/vt/vttablet/endtoend/framework/server.go index ec2d593318c..1383b0a0d1e 100644 --- a/go/vt/vttablet/endtoend/framework/server.go +++ b/go/vt/vttablet/endtoend/framework/server.go @@ -22,6 +22,8 @@ import ( "net/http" "time" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vterrors" @@ -47,6 +49,8 @@ var ( ServerAddress string // ResolveChan is the channel that sends dtids that are to be resolved. ResolveChan = make(chan string, 1) + // TopoServer is the topology for the server + TopoServer *topo.Server ) // StartServer starts the server and initializes @@ -76,8 +80,9 @@ func StartServer(connParams, connAppDebugParams mysql.ConnParams, dbName string) Shard: "0", TabletType: topodatapb.TabletType_MASTER, } + TopoServer = memorytopo.NewServer("") - Server = tabletserver.NewTabletServer("", config, memorytopo.NewServer(""), topodatapb.TabletAlias{}) + Server = tabletserver.NewTabletServer("", config, TopoServer, topodatapb.TabletAlias{}) Server.Register() err := Server.StartService(Target, dbcfgs) if err != nil { diff --git a/go/vt/vttablet/endtoend/main_test.go b/go/vt/vttablet/endtoend/main_test.go index 46375ae2549..43de9592f12 100644 --- a/go/vt/vttablet/endtoend/main_test.go +++ b/go/vt/vttablet/endtoend/main_test.go @@ -268,6 +268,27 @@ var tableACLConfig = `{ "table_names_or_prefixes": ["vitess_test_debuguser"], "readers": ["dev", "vt_appdebug"], "writers": ["dev", "vt_appdebug"] + }, + { + "name": "version", + "table_names_or_prefixes": ["vitess_version"], + "readers": ["dev"], + "writers": ["dev"], + "admins": ["dev"] + }, + { + "name": "schema_version", + "table_names_or_prefixes": ["schema_version"], + "readers": ["dev"], + "writers": ["dev"], + "admins": ["dev"] + }, + { + "name": "historian_test1", + "table_names_or_prefixes": ["historian_test1"], + "readers": ["dev"], + "writers": ["dev"], + "admins": ["dev"] } ] }` diff --git a/go/vt/vttablet/endtoend/vstreamer_test.go b/go/vt/vttablet/endtoend/vstreamer_test.go new file mode 100644 index 00000000000..9528a4018de --- /dev/null +++ b/go/vt/vttablet/endtoend/vstreamer_test.go @@ -0,0 +1,499 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endtoend + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/sqlparser" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/log" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + tabletpb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/vttablet/endtoend/framework" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" + "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer" +) + +type test struct { + query string + output []string +} + +func TestHistorianSchemaUpdate(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tsv := framework.Server + historian := tsv.Historian() + srvTopo := srvtopo.NewResilientServer(framework.TopoServer, "SchemaVersionE2ETestTopo") + + vstreamer.NewEngine(tabletenv.NewEnv(tsv.Config(), "SchemaVersionE2ETest"), srvTopo, tsv.SchemaEngine(), historian) + target := &querypb.Target{ + Keyspace: "vttest", + Shard: "0", + TabletType: tabletpb.TabletType_MASTER, + Cell: "", + } + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*/", + }}, + } + var createTableSQL = "create table historian_test1(id1 int)" + var mu sync.Mutex + mu.Lock() + send := func(events []*binlogdatapb.VEvent) error { + for _, ev := range events { + if ev.Type == binlogdatapb.VEventType_DDL && ev.Ddl == createTableSQL { + log.Info("Found DDL for table historian_test1") + mu.Unlock() + } + } + return nil + } + go func() { + if err := tsv.VStream(ctx, target, "current", filter, send); err != nil { + fmt.Printf("Error in tsv.VStream: %v", err) + t.Error(err) + } + }() + + require.Nil(t, historian.GetTableForPos(sqlparser.NewTableIdent("historian_test1"), "")) + require.NotNil(t, historian.GetTableForPos(sqlparser.NewTableIdent("vitess_test"), "")) + client := framework.NewClient() + client.Execute(createTableSQL, nil) + + mu.Lock() + minSchema := historian.GetTableForPos(sqlparser.NewTableIdent("historian_test1"), "") + want := `name:"historian_test1" fields: ` + require.Equal(t, fmt.Sprintf("%v", minSchema), want) + +} + +func TestSchemaVersioning(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tsv := framework.Server + tsv.Historian().SetTrackSchemaVersions(true) + tsv.StartTracker() + srvTopo := srvtopo.NewResilientServer(framework.TopoServer, "SchemaVersionE2ETestTopo") + + vstreamer.NewEngine(tabletenv.NewEnv(tsv.Config(), "SchemaVersionE2ETest"), srvTopo, tsv.SchemaEngine(), tsv.Historian()) + target := &querypb.Target{ + Keyspace: "vttest", + Shard: "0", + TabletType: tabletpb.TabletType_MASTER, + Cell: "", + } + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*/", + }}, + } + + var cases = []test{ + { + query: "create table vitess_version (id1 int, id2 int)", + output: []string{ + `gtid`, //gtid+other => vstream current pos + `other`, + `gtid`, //gtid+ddl => actual query + `type:DDL ddl:"create table vitess_version (id1 int, id2 int)" `, + `gtid`, //gtid+other => insert into schema_version resulting in version+other + `other`, + `version`, + `gtid`, + }, + }, + { + query: "insert into vitess_version values(1, 10)", + output: []string{ + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + }, + }, { + query: "alter table vitess_version add column id3 int", + output: []string{ + `gtid`, + `type:DDL ddl:"alter table vitess_version add column id3 int" `, + `gtid`, //gtid+other => insert into schema_version resulting in version+other + `other`, + `version`, + `gtid`, + }, + }, { + query: "insert into vitess_version values(2, 20, 200)", + output: []string{ + `type:FIELD field_event: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + }, + }, { + query: "alter table vitess_version modify column id3 varbinary(16)", + output: []string{ + `gtid`, + `type:DDL ddl:"alter table vitess_version modify column id3 varbinary(16)" `, + `gtid`, //gtid+other => insert into schema_version resulting in version+other + `other`, + `version`, + `gtid`, + }, + }, { + query: "insert into vitess_version values(3, 30, 'TTT')", + output: []string{ + `type:FIELD field_event: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + }, + }, + } + eventCh := make(chan []*binlogdatapb.VEvent) + var startPos string + send := func(events []*binlogdatapb.VEvent) error { + var evs []*binlogdatapb.VEvent + for _, event := range events { + if event.Type == binlogdatapb.VEventType_GTID { + if startPos == "" { + startPos = event.Gtid + } + } + if event.Type == binlogdatapb.VEventType_HEARTBEAT { + continue + } + log.Infof("Received event %v", event) + evs = append(evs, event) + } + select { + case eventCh <- evs: + case <-ctx.Done(): + t.Fatal("Context Done() in send") + } + return nil + } + go func() { + defer close(eventCh) + if err := tsv.VStream(ctx, target, "current", filter, send); err != nil { + fmt.Printf("Error in tsv.VStream: %v", err) + t.Error(err) + } + }() + log.Infof("\n\n\n=============================================== CURRENT EVENTS START HERE ======================\n\n\n") + runCases(ctx, t, cases, eventCh) + + tsv.StopTracker() + cases = []test{ + { + //comment prefix required so we don't look for ddl in schema_version + query: "/**/alter table vitess_version add column id4 varbinary(16)", + output: []string{ + `gtid`, //no tracker, so no insert into schema_version or version event + `type:DDL ddl:"alter table vitess_version add column id4 varbinary(16)" `, + }, + }, { + query: "insert into vitess_version values(4, 40, 'FFF', 'GGGG' )", + output: []string{ + `type:FIELD field_event: fields: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + }, + }, + } + runCases(ctx, t, cases, eventCh) + cancel() + log.Infof("\n\n\n=============================================== PAST EVENTS WITH TRACK VERSIONS START HERE ======================\n\n\n") + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + eventCh = make(chan []*binlogdatapb.VEvent) + send = func(events []*binlogdatapb.VEvent) error { + var evs []*binlogdatapb.VEvent + for _, event := range events { + if event.Type == binlogdatapb.VEventType_HEARTBEAT { + continue + } + log.Infof("Received event %v", event) + evs = append(evs, event) + } + select { + case eventCh <- evs: + case <-ctx.Done(): + t.Fatal("Context Done() in send") + } + return nil + } + go func() { + defer close(eventCh) + if err := tsv.VStream(ctx, target, startPos, filter, send); err != nil { + fmt.Printf("Error in tsv.VStream: %v", err) + t.Error(err) + } + }() + + // playing events from the past: same events as original since historian is providing the latest schema + output := []string{ + `gtid`, + `type:DDL ddl:"create table vitess_version (id1 int, id2 int)" `, + `gtid`, + `other`, + `version`, + `gtid`, + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + `gtid`, + `type:DDL ddl:"alter table vitess_version add column id3 int" `, + `gtid`, + `other`, + `version`, + `gtid`, + `type:FIELD field_event: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + `gtid`, + `type:DDL ddl:"alter table vitess_version modify column id3 varbinary(16)" `, + `gtid`, + `other`, + `version`, + `gtid`, + `type:FIELD field_event: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + `gtid`, + `type:DDL ddl:"alter table vitess_version add column id4 varbinary(16)" `, + `type:FIELD field_event: fields: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + } + + expectLogs(ctx, t, "Past stream", eventCh, output) + + cancel() + + log.Infof("\n\n\n=============================================== PAST EVENTS WITHOUT TRACK VERSIONS START HERE ======================\n\n\n") + tsv.Historian().SetTrackSchemaVersions(false) + ctx, cancel = context.WithCancel(context.Background()) + defer cancel() + eventCh = make(chan []*binlogdatapb.VEvent) + send = func(events []*binlogdatapb.VEvent) error { + var evs []*binlogdatapb.VEvent + for _, event := range events { + if event.Type == binlogdatapb.VEventType_HEARTBEAT { + continue + } + log.Infof("Received event %v", event) + evs = append(evs, event) + } + select { + case eventCh <- evs: + case <-ctx.Done(): + t.Fatal("Context Done() in send") + } + return nil + } + go func() { + defer close(eventCh) + if err := tsv.VStream(ctx, target, startPos, filter, send); err != nil { + fmt.Printf("Error in tsv.VStream: %v", err) + t.Error(err) + } + }() + + // playing events from the past: same as earlier except one below, see comments + output = []string{ + `gtid`, + `type:DDL ddl:"create table vitess_version (id1 int, id2 int)" `, + `gtid`, + `other`, + `version`, + `gtid`, + `type:FIELD field_event: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + `gtid`, + `type:DDL ddl:"alter table vitess_version add column id3 int" `, + `gtid`, + `other`, + `version`, + `gtid`, + /*at this point we only have latest schema so we have types (int32, int32, varbinary, varbinary) so the types don't match. Hence the @ fieldnames*/ + `type:FIELD field_event: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + `gtid`, + `type:DDL ddl:"alter table vitess_version modify column id3 varbinary(16)" `, + `gtid`, + `other`, + `version`, + `gtid`, + /*at this point we only have latest schema so we have types (int32, int32, varbinary, varbinary), + but the three fields below match the first three types in the latest, so the field names are correct*/ + `type:FIELD field_event: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + `gtid`, + `type:DDL ddl:"alter table vitess_version add column id4 varbinary(16)" `, + `type:FIELD field_event: fields: fields: fields: > `, + `type:ROW row_event: > > `, + `gtid`, + } + + expectLogs(ctx, t, "Past stream", eventCh, output) + cancel() + + client := framework.NewClient() + client.Execute("drop table vitess_version", nil) + client.Execute("drop table _vt.schema_version", nil) + + log.Info("=== END OF TEST") +} + +func runCases(ctx context.Context, t *testing.T, tests []test, eventCh chan []*binlogdatapb.VEvent) { + client := framework.NewClient() + + for _, test := range tests { + query := test.query + client.Execute(query, nil) + if len(test.output) > 0 { + expectLogs(ctx, t, query, eventCh, test.output) + } + if strings.HasPrefix(query, "create") || strings.HasPrefix(query, "alter") || strings.HasPrefix(query, "drop") { + ok, err := waitForVersionInsert(client, query) + if err != nil || !ok { + t.Fatalf("Query %s never got inserted into the schema_version table", query) + } + } + } +} + +func expectLogs(ctx context.Context, t *testing.T, query string, eventCh chan []*binlogdatapb.VEvent, output []string) { + t.Helper() + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + var evs []*binlogdatapb.VEvent + log.Infof("In expectLogs for query %s, output len %s", query, len(output)) + for { + select { + case allevs, ok := <-eventCh: + if !ok { + t.Fatal("expectLogs: not ok, stream ended early") + } + for _, ev := range allevs { + // Ignore spurious heartbeats that can happen on slow machines. + if ev.Type == binlogdatapb.VEventType_HEARTBEAT { + continue + } + // Also ignore begin/commit to reduce list of events to expect, for readability ... + if ev.Type == binlogdatapb.VEventType_BEGIN { + continue + } + if ev.Type == binlogdatapb.VEventType_COMMIT { + continue + } + + evs = append(evs, ev) + } + log.Infof("In expectLogs, have got %d events, want %d", len(evs), len(output)) + case <-ctx.Done(): + t.Fatalf("expectLog: Done(), stream ended early") + case <-timer.C: + t.Fatalf("expectLog: timed out waiting for events: %v: evs\n%v, want\n%v, >> got length %d, wanted length %d", query, evs, output, len(evs), len(output)) + } + if len(evs) >= len(output) { + break + } + } + if len(evs) > len(output) { + t.Fatalf("expectLog: got too many events: %v: evs\n%v, want\n%v, >> got length %d, wanted length %d", query, evs, output, len(evs), len(output)) + } + for i, want := range output { + // CurrentTime is not testable. + evs[i].CurrentTime = 0 + switch want { + case "begin": + if evs[i].Type != binlogdatapb.VEventType_BEGIN { + t.Fatalf("%v (%d): event: %v, want begin", query, i, evs[i]) + } + case "gtid": + if evs[i].Type != binlogdatapb.VEventType_GTID { + t.Fatalf("%v (%d): event: %v, want gtid", query, i, evs[i]) + } + case "commit": + if evs[i].Type != binlogdatapb.VEventType_COMMIT { + t.Fatalf("%v (%d): event: %v, want commit", query, i, evs[i]) + } + case "other": + if evs[i].Type != binlogdatapb.VEventType_OTHER { + t.Fatalf("%v (%d): event: %v, want other", query, i, evs[i]) + } + case "version": + if evs[i].Type != binlogdatapb.VEventType_VERSION { + t.Fatalf("%v (%d): event: %v, want version", query, i, evs[i]) + } + default: + evs[i].Timestamp = 0 + if got := fmt.Sprintf("%v", evs[i]); got != want { + t.Fatalf("%v (%d): event:\n%q, want\n%q", query, i, got, want) + } + } + } +} + +func encodeString(in string) string { + buf := bytes.NewBuffer(nil) + sqltypes.NewVarChar(in).EncodeSQL(buf) + return buf.String() +} + +func validateSchemaInserted(client *framework.QueryClient, ddl string) (bool, error) { + qr, _ := client.Execute(fmt.Sprintf("select * from _vt.schema_version where ddl = %s", encodeString(ddl)), nil) + if len(qr.Rows) == 1 { + log.Infof("Found ddl in schema_version: %s", ddl) + return true, nil + } + return false, fmt.Errorf("Found %d rows for gtid %s", len(qr.Rows), ddl) +} + +// To avoid races between ddls and the historian refreshing its cache explicitly wait for tracker's insert to be visible +func waitForVersionInsert(client *framework.QueryClient, ddl string) (bool, error) { + timeout := time.After(1000 * time.Millisecond) + tick := time.Tick(100 * time.Millisecond) + for { + select { + case <-timeout: + return false, errors.New("waitForVersionInsert timed out") + case <-tick: + ok, err := validateSchemaInserted(client, ddl) + if err != nil { + return false, err + } else if ok { + log.Infof("Found version insert for %s", ddl) + return true, nil + } + } + } +} diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine.go b/go/vt/vttablet/tabletmanager/vreplication/engine.go index e5ca360ed4d..0fdee83bdbe 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine.go @@ -188,7 +188,7 @@ func (vre *Engine) executeFetchMaybeCreateTable(dbClient binlogplayer.DBClient, } if merr.Num == mysql.ERBadFieldError { log.Infof("Adding column to table %s", vreplicationTableName) - for _, query := range binlogplayer.AlterVReplicationTable() { + for _, query := range binlogplayer.AlterVReplicationTable { if _, merr := dbClient.ExecuteFetch(query, 0); merr != nil { merr, isSQLErr := err.(*mysql.SQLError) if !isSQLErr || !(merr.Num == mysql.ERDupFieldName) { diff --git a/go/vt/vttablet/tabletmanager/vreplication/external_connector.go b/go/vt/vttablet/tabletmanager/vreplication/external_connector.go index 6204c124a8f..bbbca25688a 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/external_connector.go +++ b/go/vt/vttablet/tabletmanager/vreplication/external_connector.go @@ -88,7 +88,8 @@ func (ec *externalConnector) Get(name string) (*mysqlConnector, error) { c := &mysqlConnector{} c.env = tabletenv.NewEnv(config, name) c.se = schema.NewEngine(c.env) - c.vstreamer = vstreamer.NewEngine(c.env, nil, c.se) + sh := schema.NewHistorian(c.se) + c.vstreamer = vstreamer.NewEngine(c.env, nil, c.se, sh) c.se.InitDBConfig(c.env.Config().DB.DbaWithDB()) // Open diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 9ad110f2133..e35481e7f75 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -27,6 +27,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" + "github.com/golang/protobuf/proto" "golang.org/x/net/context" @@ -88,7 +90,7 @@ func TestMain(m *testing.M) { // engines cannot be initialized in testenv because it introduces // circular dependencies. - streamerEngine = vstreamer.NewEngine(env.TabletEnv, env.SrvTopo, env.SchemaEngine) + streamerEngine = vstreamer.NewEngine(env.TabletEnv, env.SrvTopo, env.SchemaEngine, schema.NewHistorian(env.SchemaEngine)) streamerEngine.Open(env.KeyspaceName, env.Cells[0]) defer streamerEngine.Close() diff --git a/go/vt/vttablet/tabletmanager/vreplication/journal_test.go b/go/vt/vttablet/tabletmanager/vreplication/journal_test.go index 32402c1fdb4..884cc042790 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/journal_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/journal_test.go @@ -36,6 +36,7 @@ func TestJournalOneToOne(t *testing.T) { "drop table t", fmt.Sprintf("drop table %s.t", vrepldb), }) + env.SchemaEngine.Open() env.SchemaEngine.Reload(context.Background()) filter := &binlogdatapb.Filter{ diff --git a/go/vt/vttablet/tabletserver/replication_watcher.go b/go/vt/vttablet/tabletserver/replication_watcher.go index febfbbd1948..485a7dfdfb3 100644 --- a/go/vt/vttablet/tabletserver/replication_watcher.go +++ b/go/vt/vttablet/tabletserver/replication_watcher.go @@ -20,6 +20,7 @@ import ( "time" "golang.org/x/net/context" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" @@ -40,16 +41,18 @@ type ReplicationWatcher struct { env tabletenv.Env watchReplication bool vs VStreamer + subscriber schema.Subscriber cancel context.CancelFunc } // NewReplicationWatcher creates a new ReplicationWatcher. -func NewReplicationWatcher(env tabletenv.Env, vs VStreamer, config *tabletenv.TabletConfig) *ReplicationWatcher { +func NewReplicationWatcher(env tabletenv.Env, vs VStreamer, config *tabletenv.TabletConfig, schemaTracker schema.Subscriber) *ReplicationWatcher { return &ReplicationWatcher{ env: env, vs: vs, watchReplication: config.WatchReplication, + subscriber: schemaTracker, } } @@ -83,9 +86,19 @@ func (rpw *ReplicationWatcher) Process(ctx context.Context) { }}, } + var gtid string for { - // VStreamer will reload the schema when it encounters a DDL. + // The tracker will reload the schema and save it into _vt.schema_tracking when the vstream encounters a DDL. err := rpw.vs.Stream(ctx, "current", filter, func(events []*binlogdatapb.VEvent) error { + for _, event := range events { + if event.Type == binlogdatapb.VEventType_GTID { + gtid = event.Gtid + } + if event.Type == binlogdatapb.VEventType_DDL { + log.Infof("Calling schema updated for %s %s", gtid, event.Ddl) + rpw.subscriber.SchemaUpdated(gtid, event.Ddl, event.Timestamp) + } + } return nil }) select { diff --git a/go/vt/vttablet/tabletserver/replication_watcher_test.go b/go/vt/vttablet/tabletserver/replication_watcher_test.go new file mode 100644 index 00000000000..b4f645b2773 --- /dev/null +++ b/go/vt/vttablet/tabletserver/replication_watcher_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tabletserver + +import ( + "testing" + "time" + + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" + + "github.com/stretchr/testify/require" + "golang.org/x/net/context" + "vitess.io/vitess/go/test/utils" + "vitess.io/vitess/go/vt/dbconfigs" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" +) + +func (t *mockSubscriber) SchemaUpdated(gtid string, ddl string, timestamp int64) error { + t.gtids = append(t.gtids, gtid) + t.ddls = append(t.ddls, ddl) + t.timestamps = append(t.timestamps, timestamp) + return nil +} + +var _ schema.Subscriber = (*mockSubscriber)(nil) +var _ VStreamer = (*fakeVstreamer)(nil) +var _ tabletenv.Env = (*fakeEnv)(nil) + +var env = &fakeEnv{} + +func TestReplicationWatcher(t *testing.T) { + testCases := []struct { + name string + input [][]*binlogdatapb.VEvent + expected []string + }{ + { + name: "empty", + input: [][]*binlogdatapb.VEvent{{}}, + expected: nil, + }, { + name: "single create table", + input: [][]*binlogdatapb.VEvent{{{ + Type: binlogdatapb.VEventType_DDL, + Timestamp: 643, + CurrentTime: 943, + Gtid: "gtid", + Ddl: "create table", + }}}, + expected: []string{"create table"}, + }, { + name: "mixed load", + input: [][]*binlogdatapb.VEvent{{{ + Type: binlogdatapb.VEventType_DDL, + Timestamp: 643, + CurrentTime: 943, + Gtid: "gtid", + Ddl: "create table", + }, { + Type: binlogdatapb.VEventType_INSERT, + Timestamp: 644, + CurrentTime: 944, + Gtid: "gtid2", + Ddl: "insert", + }, { + Type: binlogdatapb.VEventType_DDL, + Timestamp: 645, + CurrentTime: 945, + Gtid: "gtid3", + Ddl: "alter table", + }}}, + expected: []string{"create table", "alter table"}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + subscriber := &mockSubscriber{} + streamer := &fakeVstreamer{ + events: testCase.input, + } + watcher := &ReplicationWatcher{ + env: env, + watchReplication: true, + vs: streamer, + subscriber: subscriber, + } + + // when + watcher.Open() + time.Sleep(1 * time.Millisecond) + watcher.Close() + + // then + require.True(t, streamer.called, "streamer never called") + utils.MustMatch(t, testCase.expected, subscriber.ddls, "didnt see ddls") + }) + } +} + +type mockSubscriber struct { + gtids []string + ddls []string + timestamps []int64 +} + +type fakeVstreamer struct { + called bool + events [][]*binlogdatapb.VEvent +} + +type fakeEnv struct{} + +func (f *fakeEnv) CheckMySQL() { +} + +func (f *fakeEnv) Config() *tabletenv.TabletConfig { + return nil +} + +func (f *fakeEnv) DBConfigs() *dbconfigs.DBConfigs { + return nil +} + +func (f *fakeEnv) Exporter() *servenv.Exporter { + return nil +} + +func (f *fakeEnv) Stats() *tabletenv.Stats { + return nil +} + +func (f *fakeEnv) LogError() { +} + +func (f *fakeVstreamer) Stream(ctx context.Context, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + f.called = true + for _, events := range f.events { + err := send(events) + if err != nil { + return err + } + } + return nil +} diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 0e9ee129b65..ec99ab48293 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -57,7 +57,9 @@ type Engine struct { tables map[string]*Table lastChange int64 reloadTime time.Duration - notifiers map[string]notifier + //the position at which the schema was last loaded. it is only used in conjunction with ReloadAt + reloadAtPos mysql.Position + notifiers map[string]notifier // The following fields have their own synchronization // and do not require locking mu. @@ -137,6 +139,11 @@ func (se *Engine) IsOpen() bool { return se.isOpen } +// GetConnection returns a connection from the pool +func (se *Engine) GetConnection(ctx context.Context) (*connpool.DBConn, error) { + return se.conns.Get(ctx) +} + // Close shuts down Engine and is idempotent. // It can be re-opened after Close. func (se *Engine) Close() { @@ -172,12 +179,29 @@ func (se *Engine) MakeNonMaster() { // Reload reloads the schema info from the db. // Any tables that have changed since the last load are updated. func (se *Engine) Reload(ctx context.Context) error { + return se.ReloadAt(ctx, mysql.Position{}) +} + +// ReloadAt reloads the schema info from the db. +// Any tables that have changed since the last load are updated. +// It maintains the position at which the schema was reloaded and if the same position is provided +// (say by multiple vstreams) it returns the cached schema. In case of a newer or empty pos it always reloads the schema +func (se *Engine) ReloadAt(ctx context.Context, pos mysql.Position) error { se.mu.Lock() defer se.mu.Unlock() if !se.isOpen { + log.Warning("Schema reload called for an engine that is not yet open") + return nil + } + if !pos.IsZero() && se.reloadAtPos.AtLeast(pos) { + log.V(2).Infof("ReloadAt: found cached schema at %s", mysql.EncodePosition(pos)) return nil } - return se.reload(ctx) + if err := se.reload(ctx); err != nil { + return err + } + se.reloadAtPos = pos + return nil } // reload reloads the schema. It can also be used to initialize it. @@ -218,8 +242,6 @@ func (se *Engine) reload(ctx context.Context) error { if _, ok := se.tables[tableName]; ok && createTime < se.lastChange { continue } - log.Infof("Reading schema for table: %s", tableName) - table, err := LoadTable(conn, tableName, row[1].ToString(), row[3].ToString()) if err != nil { rec.RecordError(err) @@ -256,7 +278,6 @@ func (se *Engine) reload(ctx context.Context) error { se.tables[k] = t } se.lastChange = curTime - se.broadcast(created, altered, dropped) return nil } diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index 0d4a9caf297..eb3b95f17e1 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -132,7 +132,6 @@ func TestOpenAndReload(t *testing.T) { } } se.RegisterNotifier("test", notifier) - err := se.Reload(context.Background()) require.NoError(t, err) @@ -160,6 +159,50 @@ func TestOpenAndReload(t *testing.T) { } delete(want, "msg") assert.Equal(t, want, se.GetSchema()) + + //ReloadAt tests + pos1, err := mysql.DecodePosition("MariaDB/0-41983-20") + require.NoError(t, err) + pos2, err := mysql.DecodePosition("MariaDB/0-41983-40") + require.NoError(t, err) + se.UnregisterNotifier("test") + + err = se.ReloadAt(context.Background(), mysql.Position{}) + require.NoError(t, err) + assert.Equal(t, want, se.GetSchema()) + + err = se.ReloadAt(context.Background(), pos1) + require.NoError(t, err) + assert.Equal(t, want, se.GetSchema()) + + // delete table test_table_03 + db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + // test_table_04 will in spite of older timestamp because it doesn't exist yet. + mysql.BaseShowTablesRow("test_table_04", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + }, + }) + db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{ + Fields: mysql.ShowPrimaryFields, + Rows: [][]sqltypes.Value{ + mysql.ShowPrimaryRow("test_table_01", "pk"), + mysql.ShowPrimaryRow("test_table_02", "pk"), + mysql.ShowPrimaryRow("test_table_04", "pk"), + mysql.ShowPrimaryRow("seq", "id"), + }, + }) + err = se.ReloadAt(context.Background(), pos1) + require.NoError(t, err) + assert.Equal(t, want, se.GetSchema()) + + delete(want, "test_table_03") + err = se.ReloadAt(context.Background(), pos2) + require.NoError(t, err) + assert.Equal(t, want, se.GetSchema()) } func TestOpenFailedDueToMissMySQLTime(t *testing.T) { diff --git a/go/vt/vttablet/tabletserver/schema/historian.go b/go/vt/vttablet/tabletserver/schema/historian.go new file mode 100644 index 00000000000..812b37a99a7 --- /dev/null +++ b/go/vt/vttablet/tabletserver/schema/historian.go @@ -0,0 +1,278 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "context" + "fmt" + "sort" + "sync" + + "github.com/gogo/protobuf/proto" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/log" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + "vitess.io/vitess/go/vt/vtgate/evalengine" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" + + "vitess.io/vitess/go/vt/sqlparser" +) + +const getSchemaVersions = "select id, pos, ddl, time_updated, schemax from _vt.schema_version where id > %d order by id asc" + +// vl defines the glog verbosity level for the package +const vl = 10 + +// Historian defines the interface to reload a db schema or get the schema of a table for a given position +type Historian interface { + RegisterVersionEvent() error + GetTableForPos(tableName sqlparser.TableIdent, pos string) *binlogdatapb.MinimalTable + Open() error + Close() + SetTrackSchemaVersions(val bool) +} + +// TrackedSchema has the snapshot of the table at a given pos (reached by ddl) +type TrackedSchema struct { + schema map[string]*binlogdatapb.MinimalTable + pos mysql.Position + ddl string +} + +var _ Historian = (*HistorianSvc)(nil) + +// HistorianSvc implements the Historian interface by calling schema.Engine for the underlying schema +// and supplying a schema for a specific version by loading the cached values from the schema_version table +// The schema version table is populated by the Tracker +type HistorianSvc struct { + se *Engine + lastID int64 + schemas []*TrackedSchema + mu sync.Mutex + trackSchemaVersions bool + latestSchema map[string]*binlogdatapb.MinimalTable + isOpen bool +} + +// NewHistorian creates a new historian. It expects a schema.Engine instance +func NewHistorian(se *Engine) *HistorianSvc { + sh := HistorianSvc{se: se, lastID: 0, trackSchemaVersions: true} + return &sh +} + +// SetTrackSchemaVersions can be used to turn on/off the use of the schema_version history in the historian +// Only used for testing +func (h *HistorianSvc) SetTrackSchemaVersions(val bool) { + h.mu.Lock() + defer h.mu.Unlock() + h.trackSchemaVersions = val +} + +// readRow converts a row from the schema_version table to a TrackedSchema +func (h *HistorianSvc) readRow(row []sqltypes.Value) (*TrackedSchema, int64, error) { + id, _ := evalengine.ToInt64(row[0]) + pos, err := mysql.DecodePosition(string(row[1].ToBytes())) + if err != nil { + return nil, 0, err + } + ddl := string(row[2].ToBytes()) + timeUpdated, err := evalengine.ToInt64(row[3]) + if err != nil { + return nil, 0, err + } + sch := &binlogdatapb.MinimalSchema{} + if err := proto.Unmarshal(row[4].ToBytes(), sch); err != nil { + return nil, 0, err + } + log.V(vl).Infof("Read tracked schema from db: id %d, pos %v, ddl %s, schema len %d, time_updated %d \n", + id, mysql.EncodePosition(pos), ddl, len(sch.Tables), timeUpdated) + + tables := map[string]*binlogdatapb.MinimalTable{} + for _, t := range sch.Tables { + tables[t.Name] = t + } + trackedSchema := &TrackedSchema{ + schema: tables, + pos: pos, + ddl: ddl, + } + return trackedSchema, id, nil +} + +// loadFromDB loads all rows from the schema_version table that the historian does not have as yet +// caller should have locked h.mu +func (h *HistorianSvc) loadFromDB(ctx context.Context) error { + conn, err := h.se.GetConnection(ctx) + if err != nil { + return err + } + defer conn.Recycle() + tableData, err := conn.Exec(ctx, fmt.Sprintf(getSchemaVersions, h.lastID), 10000, true) + if err != nil { + log.Infof("Error reading schema_tracking table %v, will operate with the latest available schema", err) + return nil + } + for _, row := range tableData.Rows { + trackedSchema, id, err := h.readRow(row) + if err != nil { + return err + } + h.schemas = append(h.schemas, trackedSchema) + h.lastID = id + } + h.sortSchemas() + return nil +} + +// Open opens the underlying schema Engine. Called directly by a user purely interested in schema.Engine functionality +func (h *HistorianSvc) Open() error { + h.mu.Lock() + defer h.mu.Unlock() + if h.isOpen { + return nil + } + if err := h.se.Open(); err != nil { + return err + } + ctx := tabletenv.LocalContext() + h.reload() + if err := h.loadFromDB(ctx); err != nil { + return err + } + h.se.RegisterNotifier("historian", h.schemaChanged) + + h.isOpen = true + return nil +} + +// Close closes the underlying schema engine and empties the version cache +func (h *HistorianSvc) Close() { + h.mu.Lock() + defer h.mu.Unlock() + + if !h.isOpen { + return + } + h.schemas = nil + h.se.UnregisterNotifier("historian") + h.se.Close() + h.isOpen = false +} + +// convert from schema table representation to minimal tables and store as latestSchema +func (h *HistorianSvc) storeLatest(tables map[string]*Table) { + minTables := make(map[string]*binlogdatapb.MinimalTable) + for _, t := range tables { + table := &binlogdatapb.MinimalTable{ + Name: t.Name.String(), + Fields: t.Fields, + } + var pkc []int64 + for _, pk := range t.PKColumns { + pkc = append(pkc, int64(pk)) + } + table.PKColumns = pkc + minTables[table.Name] = table + } + h.latestSchema = minTables +} + +// reload gets the latest schema and replaces the latest copy of the schema maintained by the historian +// caller should lock h.mu +func (h *HistorianSvc) reload() { + h.storeLatest(h.se.tables) +} + +// schema notifier callback +func (h *HistorianSvc) schemaChanged(tables map[string]*Table, _, _, _ []string) { + if !h.isOpen { + return + } + h.mu.Lock() + defer h.mu.Unlock() + h.storeLatest(tables) +} + +// GetTableForPos returns a best-effort schema for a specific gtid +func (h *HistorianSvc) GetTableForPos(tableName sqlparser.TableIdent, gtid string) *binlogdatapb.MinimalTable { + h.mu.Lock() + defer h.mu.Unlock() + + log.V(2).Infof("GetTableForPos called for %s with pos %s", tableName, gtid) + if gtid != "" { + pos, err := mysql.DecodePosition(gtid) + if err != nil { + log.Errorf("Error decoding position for %s: %v", gtid, err) + return nil + } + var t *binlogdatapb.MinimalTable + if h.trackSchemaVersions && len(h.schemas) > 0 { + t = h.getTableFromHistoryForPos(tableName, pos) + } + if t != nil { + log.V(2).Infof("Returning table %s from history for pos %s, schema %s", tableName, gtid, t) + return t + } + } + if h.latestSchema == nil || h.latestSchema[tableName.String()] == nil { + h.reload() + if h.latestSchema == nil { + return nil + } + } + log.V(2).Infof("Returning table %s from latest schema for pos %s, schema %s", tableName, gtid, h.latestSchema[tableName.String()]) + return h.latestSchema[tableName.String()] +} + +// sortSchemas sorts entries in ascending order of gtid, ex: 40,44,48 +func (h *HistorianSvc) sortSchemas() { + sort.Slice(h.schemas, func(i int, j int) bool { + return h.schemas[j].pos.AtLeast(h.schemas[i].pos) + }) +} + +// getTableFromHistoryForPos looks in the cache for a schema for a specific gtid +func (h *HistorianSvc) getTableFromHistoryForPos(tableName sqlparser.TableIdent, pos mysql.Position) *binlogdatapb.MinimalTable { + idx := sort.Search(len(h.schemas), func(i int) bool { + return pos.Equal(h.schemas[i].pos) || !pos.AtLeast(h.schemas[i].pos) + }) + if idx >= len(h.schemas) || idx == 0 && !pos.Equal(h.schemas[idx].pos) { // beyond the range of the cache + log.Infof("Schema not found in cache for %s with pos %s", tableName, pos) + return nil + } + if pos.Equal(h.schemas[idx].pos) { //exact match to a cache entry + return h.schemas[idx].schema[tableName.String()] + } + //not an exact match, so based on our sort algo idx is one less than found: from 40,44,48 : 43 < 44 but we want 40 + return h.schemas[idx-1].schema[tableName.String()] +} + +// RegisterVersionEvent is called by the vstream when it encounters a version event (an insert into _vt.schema_tracking) +// It triggers the historian to load the newer rows from the database to update its cache +func (h *HistorianSvc) RegisterVersionEvent() error { + h.mu.Lock() + defer h.mu.Unlock() + if !h.trackSchemaVersions { + return nil + } + ctx := tabletenv.LocalContext() + if err := h.loadFromDB(ctx); err != nil { + return err + } + return nil +} diff --git a/go/vt/vttablet/tabletserver/schema/historian_test.go b/go/vt/vttablet/tabletserver/schema/historian_test.go new file mode 100644 index 00000000000..ddddbe1a621 --- /dev/null +++ b/go/vt/vttablet/tabletserver/schema/historian_test.go @@ -0,0 +1,159 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "fmt" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/sqltypes" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/sqlparser" +) + +func getTable(name string, fieldNames []string, fieldTypes []querypb.Type, pks []int64) *binlogdatapb.MinimalTable { + if name == "" || len(fieldNames) == 0 || len(fieldNames) != len(fieldTypes) || len(pks) == 0 { + return nil + } + fields := []*querypb.Field{} + for i := range fieldNames { + fields = append(fields, &querypb.Field{ + Name: fieldNames[i], + Type: fieldTypes[i], + Table: name, + }) + } + table := &binlogdatapb.MinimalTable{ + Name: name, + Fields: fields, + PKColumns: pks, + } + return table +} + +func getDbSchemaBlob(t *testing.T, tables map[string]*binlogdatapb.MinimalTable) string { + dbSchema := &binlogdatapb.MinimalSchema{ + Tables: []*binlogdatapb.MinimalTable{}, + } + for name, table := range tables { + t := &binlogdatapb.MinimalTable{ + Name: name, + Fields: table.Fields, + } + pks := make([]int64, 0) + for _, pk := range table.PKColumns { + pks = append(pks, int64(pk)) + } + t.PKColumns = pks + dbSchema.Tables = append(dbSchema.Tables, t) + } + blob, err := proto.Marshal(dbSchema) + require.NoError(t, err) + return string(blob) +} + +func TestHistorian(t *testing.T) { + se, db, cancel := getTestSchemaEngine(t) + defer cancel() + h := NewHistorian(se) + h.Open() + + h.SetTrackSchemaVersions(false) + require.Nil(t, h.RegisterVersionEvent()) + gtidPrefix := "MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:" + gtid1 := gtidPrefix + "1-10" + ddl1 := "create table tracker_test (id int)" + ts1 := int64(1427325876) + _, _, _ = ddl1, ts1, db + require.Nil(t, h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid1)) + require.Equal(t, fmt.Sprintf("%v", h.GetTableForPos(sqlparser.NewTableIdent("dual"), gtid1)), `name:"dual" `) + h.SetTrackSchemaVersions(true) + require.Nil(t, h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid1)) + var blob1 string + + fields := []*querypb.Field{{ + Name: "id", + Type: sqltypes.Int32, + }, { + Name: "pos", + Type: sqltypes.VarBinary, + }, { + Name: "ddl", + Type: sqltypes.VarBinary, + }, { + Name: "time_updated", + Type: sqltypes.Int32, + }, { + Name: "schemax", + Type: sqltypes.Blob, + }} + + table := getTable("t1", []string{"id1", "id2"}, []querypb.Type{querypb.Type_INT32, querypb.Type_INT32}, []int64{0}) + tables := make(map[string]*binlogdatapb.MinimalTable) + tables["t1"] = table + blob1 = getDbSchemaBlob(t, tables) + db.AddQuery("select id, pos, ddl, time_updated, schemax from _vt.schema_version where id > 0 order by id asc", &sqltypes.Result{ + Fields: fields, + Rows: [][]sqltypes.Value{ + {sqltypes.NewInt32(1), sqltypes.NewVarBinary(gtid1), sqltypes.NewVarBinary(ddl1), sqltypes.NewInt32(int32(ts1)), sqltypes.NewVarBinary(blob1)}, + }, + }) + require.Nil(t, h.RegisterVersionEvent()) + exp1 := `name:"t1" fields: fields: p_k_columns:0 ` + require.Equal(t, exp1, fmt.Sprintf("%v", h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid1))) + gtid2 := gtidPrefix + "1-20" + require.Nil(t, h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid2)) + + table = getTable("t1", []string{"id1", "id2"}, []querypb.Type{querypb.Type_INT32, querypb.Type_VARBINARY}, []int64{0}) + tables["t1"] = table + blob2 := getDbSchemaBlob(t, tables) + ddl2 := "alter table t1 modify column id2 varbinary" + ts2 := ts1 + 100 + db.AddQuery("select id, pos, ddl, time_updated, schemax from _vt.schema_version where id > 1 order by id asc", &sqltypes.Result{ + Fields: fields, + Rows: [][]sqltypes.Value{ + {sqltypes.NewInt32(2), sqltypes.NewVarBinary(gtid2), sqltypes.NewVarBinary(ddl2), sqltypes.NewInt32(int32(ts2)), sqltypes.NewVarBinary(blob2)}, + }, + }) + require.Nil(t, h.RegisterVersionEvent()) + exp2 := `name:"t1" fields: fields: p_k_columns:0 ` + require.Equal(t, exp2, fmt.Sprintf("%v", h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid2))) + gtid3 := gtidPrefix + "1-30" + require.Nil(t, h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid3)) + + table = getTable("t1", []string{"id1", "id2", "id3"}, []querypb.Type{querypb.Type_INT32, querypb.Type_VARBINARY, querypb.Type_INT32}, []int64{0}) + tables["t1"] = table + blob3 := getDbSchemaBlob(t, tables) + ddl3 := "alter table t1 add column id3 int" + ts3 := ts2 + 100 + db.AddQuery("select id, pos, ddl, time_updated, schemax from _vt.schema_version where id > 2 order by id asc", &sqltypes.Result{ + Fields: fields, + Rows: [][]sqltypes.Value{ + {sqltypes.NewInt32(3), sqltypes.NewVarBinary(gtid3), sqltypes.NewVarBinary(ddl3), sqltypes.NewInt32(int32(ts3)), sqltypes.NewVarBinary(blob3)}, + }, + }) + require.Nil(t, h.RegisterVersionEvent()) + exp3 := `name:"t1" fields: fields: fields: p_k_columns:0 ` + require.Equal(t, exp3, fmt.Sprintf("%v", h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid3))) + + require.Equal(t, exp1, fmt.Sprintf("%v", h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid1))) + require.Equal(t, exp2, fmt.Sprintf("%v", h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid2))) + require.Equal(t, exp3, fmt.Sprintf("%v", h.GetTableForPos(sqlparser.NewTableIdent("t1"), gtid3))) +} diff --git a/go/vt/vttablet/tabletserver/schema/main_test.go b/go/vt/vttablet/tabletserver/schema/main_test.go new file mode 100644 index 00000000000..b0fd634c3f1 --- /dev/null +++ b/go/vt/vttablet/tabletserver/schema/main_test.go @@ -0,0 +1,46 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/sqltypes" +) + +func getTestSchemaEngine(t *testing.T) (*Engine, *fakesqldb.DB, func()) { + db := fakesqldb.New(t) + db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "t", + "int64"), + "1427325876", + )) + db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{}) + + db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{}) + se := newEngine(10, 10*time.Second, 10*time.Second, true, db) + require.NoError(t, se.Open()) + cancel := func() { + defer db.Close() + defer se.Close() + } + return se, db, cancel +} diff --git a/go/vt/vttablet/tabletserver/schema/tracker.go b/go/vt/vttablet/tabletserver/schema/tracker.go new file mode 100644 index 00000000000..918ce42421e --- /dev/null +++ b/go/vt/vttablet/tabletserver/schema/tracker.go @@ -0,0 +1,122 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "bytes" + "context" + "fmt" + + "github.com/gogo/protobuf/proto" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/log" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" +) + +const createSchemaTrackingTable = `CREATE TABLE IF NOT EXISTS _vt.schema_version ( + id INT AUTO_INCREMENT, + pos VARBINARY(10000) NOT NULL, + time_updated BIGINT(20) NOT NULL, + ddl VARBINARY(1000) DEFAULT NULL, + schemax BLOB NOT NULL, + PRIMARY KEY (id) + ) ENGINE=InnoDB` + +//Subscriber will get notified when the schema has been updated +type Subscriber interface { + SchemaUpdated(gtid string, ddl string, timestamp int64) error +} + +var _ Subscriber = (*Tracker)(nil) + +// Tracker implements Subscriber and persists versions into the ddb +type Tracker struct { + engine *Engine + enabled bool +} + +// NewTracker creates a Tracker, needs an Open SchemaEngine (which implements the trackerEngine interface) +func NewTracker(engine *Engine) *Tracker { + return &Tracker{engine: engine} +} + +// Open enables the tracker functionality +func (t *Tracker) Open() { + t.enabled = true +} + +// Close disables the tracker functionality +func (t *Tracker) Close() { + t.enabled = false +} + +// SchemaUpdated is called by a vstream when it encounters a DDL +func (t *Tracker) SchemaUpdated(gtid string, ddl string, timestamp int64) error { + if !t.enabled { + log.Infof("Tracker not enabled, ignoring SchemaUpdated event") + return nil + } + log.Infof("Processing SchemaUpdated event for gtid %s, ddl %s", gtid, ddl) + if gtid == "" || ddl == "" { + return fmt.Errorf("got invalid gtid or ddl in SchemaUpdated") + } + ctx := context.Background() + + // Engine will have reloaded the schema because vstream will reload it on a DDL + tables := t.engine.GetSchema() + dbSchema := &binlogdatapb.MinimalSchema{ + Tables: []*binlogdatapb.MinimalTable{}, + } + for name, table := range tables { + t := &binlogdatapb.MinimalTable{ + Name: name, + Fields: table.Fields, + } + pks := make([]int64, 0) + for _, pk := range table.PKColumns { + pks = append(pks, int64(pk)) + } + t.PKColumns = pks + dbSchema.Tables = append(dbSchema.Tables, t) + } + blob, _ := proto.Marshal(dbSchema) + + conn, err := t.engine.GetConnection(ctx) + if err != nil { + return err + } + defer conn.Recycle() + _, err = conn.Exec(ctx, createSchemaTrackingTable, 1, false) + if err != nil { + return err + } + query := fmt.Sprintf("insert into _vt.schema_version "+ + "(pos, ddl, schemax, time_updated) "+ + "values (%v, %v, %v, %d)", encodeString(gtid), encodeString(ddl), encodeString(string(blob)), timestamp) + + _, err = conn.Exec(ctx, query, 1, false) + if err != nil { + return err + } + return nil +} + +func encodeString(in string) string { + buf := bytes.NewBuffer(nil) + sqltypes.NewVarChar(in).EncodeSQL(buf) + return buf.String() +} diff --git a/go/vt/vttablet/tabletserver/schema/tracker_test.go b/go/vt/vttablet/tabletserver/schema/tracker_test.go new file mode 100644 index 00000000000..08631f0720a --- /dev/null +++ b/go/vt/vttablet/tabletserver/schema/tracker_test.go @@ -0,0 +1,48 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schema + +import ( + "fmt" + "regexp" + "testing" + + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/sqltypes" +) + +func TestTracker(t *testing.T) { + se, db, cancel := getTestSchemaEngine(t) + defer cancel() + + tracker := NewTracker(se) + tracker.Open() + + gtid1 := "MySQL56/7b04699f-f5e9-11e9-bf88-9cb6d089e1c3:1-10" + ddl1 := "create table tracker_test (id int)" + ts1 := int64(1427325876) + var query string + query = "CREATE TABLE IF NOT EXISTS _vt.schema_version.*" + db.AddQueryPattern(query, &sqltypes.Result{}) + + query = fmt.Sprintf("insert into _vt.schema_version.*%s.*%s.*%d.*", gtid1, regexp.QuoteMeta(ddl1), ts1) + db.AddQueryPattern(query, &sqltypes.Result{}) + + require.NoError(t, tracker.SchemaUpdated(gtid1, ddl1, ts1)) + require.Error(t, tracker.SchemaUpdated("", ddl1, ts1)) + require.Error(t, tracker.SchemaUpdated(gtid1, "", ts1)) +} diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index d8611ff1746..359eada15c9 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -109,6 +109,7 @@ func init() { flag.BoolVar(¤tConfig.TerseErrors, "queryserver-config-terse-errors", defaultConfig.TerseErrors, "prevent bind vars from escaping in returned errors") flag.StringVar(&deprecatedPoolNamePrefix, "pool-name-prefix", "", "Deprecated") flag.BoolVar(¤tConfig.WatchReplication, "watch_replication_stream", false, "When enabled, vttablet will stream the MySQL replication stream from the local server, and use it to support the include_event_token ExecuteOptions.") + flag.BoolVar(¤tConfig.TrackSchemaVersions, "track_schema_versions", true, "When enabled, vttablet will store versions of schemas at each position that a DDL is applied and allow retrieval of the schema corresponding to a position") flag.BoolVar(&deprecatedAutocommit, "enable-autocommit", true, "This flag is deprecated. Autocommit is always allowed.") flag.BoolVar(¤tConfig.TwoPCEnable, "twopc_enable", defaultConfig.TwoPCEnable, "if the flag is on, 2pc is enabled. Other 2pc flags must be supplied.") flag.StringVar(¤tConfig.TwoPCCoordinatorAddress, "twopc_coordinator_address", defaultConfig.TwoPCCoordinatorAddress, "address of the (VTGate) process(es) that will be used to notify of abandoned transactions.") @@ -205,6 +206,7 @@ type TabletConfig struct { QueryCacheSize int `json:"queryCacheSize,omitempty"` SchemaReloadIntervalSeconds int `json:"schemaReloadIntervalSeconds,omitempty"` WatchReplication bool `json:"watchReplication,omitempty"` + TrackSchemaVersions bool `json:"trackSchemaVersions,omitempty"` TerseErrors bool `json:"terseErrors,omitempty"` MessagePostponeParallelism int `json:"messagePostponeParallelism,omitempty"` CacheResultFields bool `json:"cacheResultFields,omitempty"` diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 6a7c9ea5d90..a786376194a 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -162,10 +162,12 @@ type TabletServer struct { // The following variables should only be accessed within // the context of a startRequest-endRequest. se *schema.Engine + sh schema.Historian qe *QueryEngine te *TxEngine hw *heartbeat.Writer hr *heartbeat.Reader + tracker *schema.Tracker watcher *ReplicationWatcher vstreamer *vstreamer.Engine messager *messager.Engine @@ -228,6 +230,8 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to alias: alias, } tsv.se = schema.NewEngine(tsv) + tsv.sh = schema.NewHistorian(tsv.se) + tsv.sh.SetTrackSchemaVersions(config.TrackSchemaVersions) tsv.qe = NewQueryEngine(tsv, tsv.se) tsv.te = NewTxEngine(tsv) tsv.txController = tsv.te @@ -235,10 +239,10 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to tsv.hr = heartbeat.NewReader(tsv) tsv.txThrottler = txthrottler.NewTxThrottler(tsv.config, topoServer) tsOnce.Do(func() { srvTopoServer = srvtopo.NewResilientServer(topoServer, "TabletSrvTopo") }) - tsv.vstreamer = vstreamer.NewEngine(tsv, srvTopoServer, tsv.se) - tsv.watcher = NewReplicationWatcher(tsv, tsv.vstreamer, tsv.config) + tsv.vstreamer = vstreamer.NewEngine(tsv, srvTopoServer, tsv.se, tsv.sh) + tsv.tracker = schema.NewTracker(tsv.se) + tsv.watcher = NewReplicationWatcher(tsv, tsv.vstreamer, tsv.config, tsv.tracker) tsv.messager = messager.NewEngine(tsv, tsv.se, tsv.vstreamer) - tsv.exporter.NewGaugeFunc("TabletState", "Tablet server state", func() int64 { tsv.mu.Lock() defer tsv.mu.Unlock() @@ -260,6 +264,25 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to return tsv } +// StartTracker starts a new replication watcher +// Only to be used for testing +func (tsv *TabletServer) StartTracker() { + tsv.config.TrackSchemaVersions = true + tsv.config.WatchReplication = true + tsv.tracker.Open() + //need to close and reopen watcher since it is already opened in the tsv init and is idempotent ... + tsv.watcher.Close() + tsv.watcher.watchReplication = true + tsv.watcher.Open() +} + +// StopTracker turns the watcher off +// Only to be used for testing +func (tsv *TabletServer) StopTracker() { + tsv.config.TrackSchemaVersions = false + tsv.tracker.Close() +} + // Register prepares TabletServer for serving by calling // all the registrations functions. func (tsv *TabletServer) Register() { @@ -517,8 +540,9 @@ func (tsv *TabletServer) fullStart() (err error) { } c.Close() - if err := tsv.se.Open(); err != nil { - log.Errorf("Could not load schema, but starting the query service anyways: %v", err) + // sh opens and closes se + if err := tsv.sh.Open(); err != nil { + log.Errorf("Could not load historian, but starting the query service anyways: %v", err) } if err := tsv.qe.Open(); err != nil { return err @@ -548,12 +572,19 @@ func (tsv *TabletServer) serveNewType() (err error) { tsv.messager.Open() tsv.hr.Close() tsv.hw.Open() + log.Info("Opening tracker, trackschemaversions is %t", tsv.config.TrackSchemaVersions) + tsv.tracker.Open() + if tsv.config.TrackSchemaVersions { + log.Info("Starting watcher") + tsv.watcher.Open() + } } else { tsv.txController.AcceptReadOnly() tsv.messager.Close() tsv.hr.Open() tsv.hw.Close() tsv.watcher.Open() + tsv.tracker.Close() // Reset the sequences. tsv.se.MakeNonMaster() @@ -1686,9 +1717,10 @@ func (tsv *TabletServer) BroadcastHealth(terTimestamp int64, stats *querypb.Real target := tsv.target tsv.mu.Unlock() shr := &querypb.StreamHealthResponse{ - Target: &target, - TabletAlias: &tsv.alias, - Serving: tsv.IsServing(), + Target: &target, + TabletAlias: &tsv.alias, + Serving: tsv.IsServing(), + TabletExternallyReparentedTimestamp: terTimestamp, RealtimeStats: stats, } @@ -1914,6 +1946,11 @@ func (tsv *TabletServer) SetConsolidatorMode(mode string) { tsv.qe.consolidatorMode = mode } +// Historian returns the associated historian service +func (tsv *TabletServer) Historian() schema.Historian { + return tsv.sh +} + // queryAsString returns a readable version of query+bind variables. func queryAsString(sql string, bindVariables map[string]*querypb.BindVariable) string { buf := &bytes.Buffer{} diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine.go b/go/vt/vttablet/tabletserver/vstreamer/engine.go index 834f5c5d878..ce362fd181a 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine.go @@ -64,6 +64,7 @@ type Engine struct { // The following members are initialized once at the beginning. ts srvtopo.Server se *schema.Engine + sh schema.Historian keyspace string cell string @@ -74,7 +75,7 @@ type Engine struct { // NewEngine creates a new Engine. // Initialization sequence is: NewEngine->InitDBConfig->Open. // Open and Close can be called multiple times and are idempotent. -func NewEngine(env tabletenv.Env, ts srvtopo.Server, se *schema.Engine) *Engine { +func NewEngine(env tabletenv.Env, ts srvtopo.Server, se *schema.Engine, sh schema.Historian) *Engine { vse := &Engine{ env: env, streamers: make(map[int]*vstreamer), @@ -83,6 +84,7 @@ func NewEngine(env tabletenv.Env, ts srvtopo.Server, se *schema.Engine) *Engine lvschema: &localVSchema{vschema: &vindexes.VSchema{}}, ts: ts, se: se, + sh: sh, vschemaErrors: env.Exporter().NewCounter("VSchemaErrors", "Count of VSchema errors"), vschemaUpdates: env.Exporter().NewCounter("VSchemaUpdates", "Count of VSchema updates. Does not include errors"), } @@ -156,7 +158,7 @@ func (vse *Engine) Stream(ctx context.Context, startPos string, filter *binlogda if !vse.isOpen { return nil, 0, errors.New("VStreamer is not open") } - streamer := newVStreamer(ctx, vse.env.Config().DB.AppWithDB(), vse.se, startPos, filter, vse.lvschema, send) + streamer := newVStreamer(ctx, vse.env.Config().DB.AppWithDB(), vse.se, vse.sh, startPos, filter, vse.lvschema, send) idx := vse.streamIdx vse.streamers[idx] = streamer vse.streamIdx++ @@ -196,7 +198,7 @@ func (vse *Engine) StreamRows(ctx context.Context, query string, lastpk []sqltyp if !vse.isOpen { return nil, 0, errors.New("VStreamer is not open") } - rowStreamer := newRowStreamer(ctx, vse.env.Config().DB.AppWithDB(), vse.se, query, lastpk, vse.lvschema, send) + rowStreamer := newRowStreamer(ctx, vse.env.Config().DB.AppWithDB(), vse.sh, query, lastpk, vse.lvschema, send) idx := vse.streamIdx vse.rowStreamers[idx] = rowStreamer vse.streamIdx++ @@ -318,7 +320,7 @@ func (vse *Engine) setWatch() { vschema: vschema, } b, _ := json.MarshalIndent(vschema, "", " ") - log.Infof("Updated vschema: %s", b) + log.V(2).Infof("Updated vschema: %s", b) for _, s := range vse.streamers { s.SetVSchema(vse.lvschema) } diff --git a/go/vt/vttablet/tabletserver/vstreamer/main_test.go b/go/vt/vttablet/tabletserver/vstreamer/main_test.go index 080817d0803..7dfbdea99a3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/main_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/main_test.go @@ -17,21 +17,27 @@ limitations under the License. package vstreamer import ( + "context" "flag" "fmt" "os" "testing" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + "vitess.io/vitess/go/vt/sqlparser" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/dbconfigs" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/testenv" ) var ( - engine *Engine - env *testenv.Env + engine *Engine + env *testenv.Env + historian schema.Historian ) func TestMain(m *testing.M) { @@ -51,8 +57,10 @@ func TestMain(m *testing.M) { defer env.Close() // engine cannot be initialized in testenv because it introduces - // circular dependencies. - engine = NewEngine(env.TabletEnv, env.SrvTopo, env.SchemaEngine) + // circular dependencies + historian = schema.NewHistorian(env.SchemaEngine) + historian.Open() + engine = NewEngine(env.TabletEnv, env.SrvTopo, env.SchemaEngine, historian) engine.Open(env.KeyspaceName, env.Cells[0]) defer engine.Close() @@ -67,7 +75,41 @@ func customEngine(t *testing.T, modifier func(mysql.ConnParams) mysql.ConnParams modified := modifier(*original) config := env.TabletEnv.Config().Clone() config.DB = dbconfigs.NewTestDBConfigs(modified, modified, modified.DbName) - engine := NewEngine(tabletenv.NewEnv(config, "VStreamerTest"), env.SrvTopo, env.SchemaEngine) + + engine := NewEngine(tabletenv.NewEnv(config, "VStreamerTest"), env.SrvTopo, env.SchemaEngine, historian) engine.Open(env.KeyspaceName, env.Cells[0]) return engine } + +type mockHistorian struct { + se *schema.Engine +} + +func (h *mockHistorian) SetTrackSchemaVersions(val bool) {} + +func (h *mockHistorian) Open() error { + return nil +} + +func (h *mockHistorian) Close() { +} + +func (h *mockHistorian) Reload(ctx context.Context) error { + return nil +} + +func newMockHistorian(se *schema.Engine) *mockHistorian { + sh := mockHistorian{se: se} + return &sh +} + +func (h *mockHistorian) GetTableForPos(tableName sqlparser.TableIdent, pos string) *binlogdatapb.MinimalTable { + return nil +} + +func (h *mockHistorian) RegisterVersionEvent() error { + numVersionEventsReceived++ + return nil +} + +var _ schema.Historian = (*mockHistorian)(nil) diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go index 35001615384..e6fa88c4989 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go @@ -38,8 +38,8 @@ type RowStreamer interface { } // NewRowStreamer returns a RowStreamer -func NewRowStreamer(ctx context.Context, cp dbconfigs.Connector, se *schema.Engine, query string, lastpk []sqltypes.Value, send func(*binlogdatapb.VStreamRowsResponse) error) RowStreamer { - return newRowStreamer(ctx, cp, se, query, lastpk, &localVSchema{vschema: &vindexes.VSchema{}}, send) +func NewRowStreamer(ctx context.Context, cp dbconfigs.Connector, sh *schema.HistorianSvc, query string, lastpk []sqltypes.Value, send func(*binlogdatapb.VStreamRowsResponse) error) RowStreamer { + return newRowStreamer(ctx, cp, sh, query, lastpk, &localVSchema{vschema: &vindexes.VSchema{}}, send) } // rowStreamer is used for copying the existing rows of a table @@ -55,7 +55,7 @@ type rowStreamer struct { cancel func() cp dbconfigs.Connector - se *schema.Engine + sh schema.Historian query string lastpk []sqltypes.Value send func(*binlogdatapb.VStreamRowsResponse) error @@ -66,13 +66,13 @@ type rowStreamer struct { sendQuery string } -func newRowStreamer(ctx context.Context, cp dbconfigs.Connector, se *schema.Engine, query string, lastpk []sqltypes.Value, vschema *localVSchema, send func(*binlogdatapb.VStreamRowsResponse) error) *rowStreamer { +func newRowStreamer(ctx context.Context, cp dbconfigs.Connector, sh schema.Historian, query string, lastpk []sqltypes.Value, vschema *localVSchema, send func(*binlogdatapb.VStreamRowsResponse) error) *rowStreamer { ctx, cancel := context.WithCancel(ctx) return &rowStreamer{ ctx: ctx, cancel: cancel, cp: cp, - se: se, + sh: sh, query: query, lastpk: lastpk, send: send, @@ -85,9 +85,9 @@ func (rs *rowStreamer) Cancel() { } func (rs *rowStreamer) Stream() error { - // Ensure se is Open. If vttablet came up in a non_serving role, + // Ensure sh is Open. If vttablet came up in a non_serving role, // the schema engine may not have been initialized. - if err := rs.se.Open(); err != nil { + if err := rs.sh.Open(); err != nil { return err } @@ -113,12 +113,12 @@ func (rs *rowStreamer) buildPlan() error { if err != nil { return err } - st := rs.se.GetTable(fromTable) + st := rs.sh.GetTableForPos(fromTable, "") if st == nil { return fmt.Errorf("unknown table %v in schema", fromTable) } ti := &Table{ - Name: st.Name.String(), + Name: st.Name, Fields: st.Fields, } // The plan we build is identical to the one for vstreamer. @@ -140,20 +140,22 @@ func (rs *rowStreamer) buildPlan() error { return err } -func buildPKColumns(st *schema.Table) ([]int, error) { +func buildPKColumns(st *binlogdatapb.MinimalTable) ([]int, error) { + var pkColumns = make([]int, 0) if len(st.PKColumns) == 0 { - pkColumns := make([]int, len(st.Fields)) + pkColumns = make([]int, len(st.Fields)) for i := range st.Fields { pkColumns[i] = i } return pkColumns, nil } for _, pk := range st.PKColumns { - if pk >= len(st.Fields) { + if pk >= int64(len(st.Fields)) { return nil, fmt.Errorf("primary key %d refers to non-existent column", pk) } + pkColumns = append(pkColumns, int(pk)) } - return st.PKColumns, nil + return pkColumns, nil } func (rs *rowStreamer) buildSelect() (string, error) { diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index 36e7dfcbf11..3157500b2e1 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -48,12 +48,14 @@ func TestStreamRowsScan(t *testing.T) { "create table t4(id1 int, id2 int, id3 int, val varbinary(128), primary key(id1, id2, id3))", "insert into t4 values (1, 2, 3, 'aaa'), (2, 3, 4, 'bbb')", }) + defer execStatements(t, []string{ "drop table t1", "drop table t2", "drop table t3", "drop table t4", }) + engine.se.Reload(context.Background()) // t1: all rows @@ -137,19 +139,22 @@ func TestStreamRowsUnicode(t *testing.T) { execStatements(t, []string{ "create table t1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id))", }) + defer execStatements(t, []string{ "drop table t1", }) // Use an engine with latin1 charset. savedEngine := engine - defer func() { engine = savedEngine }() + defer func() { + engine = savedEngine + }() engine = customEngine(t, func(in mysql.ConnParams) mysql.ConnParams { in.Charset = "latin1" return in }) defer engine.Close() - + engine.se.Reload(context.Background()) // We need a latin1 connection. conn, err := env.Mysqld.GetDbaConnection() if err != nil { @@ -195,6 +200,7 @@ func TestStreamRowsKeyRange(t *testing.T) { "create table t1(id1 int, val varbinary(128), primary key(id1))", "insert into t1 values (1, 'aaa'), (6, 'bbb')", }) + defer execStatements(t, []string{ "drop table t1", }) @@ -225,6 +231,7 @@ func TestStreamRowsFilterInt(t *testing.T) { "create table t1(id1 int, id2 int, val varbinary(128), primary key(id1))", "insert into t1 values (1, 100, 'aaa'), (2, 200, 'bbb'), (3, 200, 'ccc'), (4, 100, 'ddd'), (5, 200, 'eee')", }) + defer execStatements(t, []string{ "drop table t1", }) @@ -254,6 +261,7 @@ func TestStreamRowsFilterVarBinary(t *testing.T) { "create table t1(id1 int, val varbinary(128), primary key(id1))", "insert into t1 values (1,'kepler'), (2, 'newton'), (3, 'newton'), (4, 'kepler'), (5, 'newton'), (6, 'kepler')", }) + defer execStatements(t, []string{ "drop table t1", }) @@ -282,6 +290,7 @@ func TestStreamRowsMultiPacket(t *testing.T) { "create table t1(id int, val varbinary(128), primary key(id))", "insert into t1 values (1, '234'), (2, '6789'), (3, '1'), (4, '2345678901'), (5, '2')", }) + defer execStatements(t, []string{ "drop table t1", }) @@ -310,6 +319,7 @@ func TestStreamRowsCancel(t *testing.T) { "create table t1(id int, val varbinary(128), primary key(id))", "insert into t1 values (1, '234567890'), (2, '234')", }) + defer execStatements(t, []string{ "drop table t1", }) diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index afe047fb3bf..2373f79cc46 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -61,6 +61,7 @@ type vstreamer struct { cp dbconfigs.Connector se *schema.Engine + sh schema.Historian startPos string filter *binlogdatapb.Filter send func([]*binlogdatapb.VEvent) error @@ -69,6 +70,7 @@ type vstreamer struct { vschema *localVSchema plans map[uint64]*streamerPlan journalTableID uint64 + versionTableID uint64 // format and pos are updated by parseEvent. format mysql.BinlogFormat @@ -85,11 +87,11 @@ type streamerPlan struct { // newVStreamer creates a new vstreamer. // cp: the mysql conn params. -// se: the schema engine. The vstreamer uses it to convert the TableMap into field info. +// sh: the schema engine. The vstreamer uses it to convert the TableMap into field info. // startPos: a flavor compliant position to stream from. This can also contain the special // value "current", which means start from the current position. -// filter: the list of filtering rules. If a rule has a select expressinon for its filter, -// the select list can only reference direct columns. No other experssions are allowed. +// filter: the list of filtering rules. If a rule has a select expression for its filter, +// the select list can only reference direct columns. No other expressions are allowed. // The select expression is allowed to contain the special 'keyspace_id()' function which // will return the keyspace id of the row. Examples: // "select * from t", same as an empty Filter, @@ -101,13 +103,15 @@ type streamerPlan struct { // Other constructs like joins, group by, etc. are not supported. // vschema: the current vschema. This value can later be changed through the SetVSchema method. // send: callback function to send events. -func newVStreamer(ctx context.Context, cp dbconfigs.Connector, se *schema.Engine, startPos string, filter *binlogdatapb.Filter, vschema *localVSchema, send func([]*binlogdatapb.VEvent) error) *vstreamer { +func newVStreamer(ctx context.Context, cp dbconfigs.Connector, se *schema.Engine, sh schema.Historian, startPos string, + filter *binlogdatapb.Filter, vschema *localVSchema, send func([]*binlogdatapb.VEvent) error) *vstreamer { ctx, cancel := context.WithCancel(ctx) return &vstreamer{ ctx: ctx, cancel: cancel, cp: cp, se: se, + sh: sh, startPos: startPos, filter: filter, send: send, @@ -163,9 +167,9 @@ func (vs *vstreamer) Stream() error { vs.pos = pos } - // Ensure se is Open. If vttablet came up in a non_serving role, + // Ensure sh is Open. If vttablet came up in a non_serving role, // the schema engine may not have been initialized. - if err := vs.se.Open(); err != nil { + if err := vs.sh.Open(); err != nil { return wrapError(err, vs.pos) } @@ -214,12 +218,14 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog // If a single row exceeds the packet size, it will be in its own packet. bufferAndTransmit := func(vevent *binlogdatapb.VEvent) error { switch vevent.Type { - case binlogdatapb.VEventType_GTID, binlogdatapb.VEventType_BEGIN, binlogdatapb.VEventType_FIELD, binlogdatapb.VEventType_JOURNAL: + case binlogdatapb.VEventType_GTID, binlogdatapb.VEventType_BEGIN, binlogdatapb.VEventType_FIELD, + binlogdatapb.VEventType_JOURNAL: // We never have to send GTID, BEGIN, FIELD events on their own. // A JOURNAL event is always preceded by a BEGIN and followed by a COMMIT. // So, we don't have to send it right away. bufferedEvents = append(bufferedEvents, vevent) - case binlogdatapb.VEventType_COMMIT, binlogdatapb.VEventType_DDL, binlogdatapb.VEventType_OTHER, binlogdatapb.VEventType_HEARTBEAT: + case binlogdatapb.VEventType_COMMIT, binlogdatapb.VEventType_DDL, binlogdatapb.VEventType_OTHER, + binlogdatapb.VEventType_HEARTBEAT, binlogdatapb.VEventType_VERSION: // COMMIT, DDL, OTHER and HEARTBEAT must be immediately sent. // Although unlikely, it's possible to get a HEARTBEAT in the middle // of a transaction. If so, we still send the partial transaction along @@ -442,6 +448,7 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e }) } else { // If the DDL need not be sent, send a dummy OTHER event. + log.Infof("Not sending DDL for %s", q.SQL) vevents = append(vevents, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_GTID, Gtid: mysql.EncodePosition(vs.pos), @@ -449,10 +456,7 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e Type: binlogdatapb.VEventType_OTHER, }) } - // Proactively reload schema. - // If the DDL adds a column, comparing with an older snapshot of the - // schema will make us think that a column was dropped and error out. - vs.se.Reload(vs.ctx) + vs.se.ReloadAt(context.Background(), vs.pos) case sqlparser.StmtOther, sqlparser.StmtPriv: // These are either: // 1) DBA statements like REPAIR that can be ignored. @@ -475,9 +479,11 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e // A schema change will result in a change in table id, which // will generate a new plan and FIELD event. id := ev.TableID(vs.format) + if _, ok := vs.plans[id]; ok { return nil, nil } + tm, err := ev.TableMap(vs.format) if err != nil { return nil, err @@ -485,11 +491,15 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e if tm.Database == "_vt" && tm.Name == "resharding_journal" { // A journal is a special case that generates a JOURNAL event. return nil, vs.buildJournalPlan(id, tm) + } else if tm.Database == "_vt" && tm.Name == "schema_version" { + // Generates a Version event when it detects that a schema is stored in the schema_version table. + return nil, vs.buildVersionPlan(id, tm) } if tm.Database != "" && tm.Database != params.DbName { vs.plans[id] = nil return nil, nil } + vevent, err := vs.buildTablePlan(id, tm) if err != nil { return nil, err @@ -499,7 +509,7 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e } case ev.IsWriteRows() || ev.IsDeleteRows() || ev.IsUpdateRows(): // The existence of before and after images can be used to - // identify statememt types. It's also possible that the + // identify statement types. It's also possible that the // before and after images end up going to different shards. // If so, an update will be treated as delete on one shard // and insert on the other. @@ -513,7 +523,15 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e return nil, err } if id == vs.journalTableID { - vevents, err = vs.processJounalEvent(vevents, plan, rows) + vevents, err = vs.processJournalEvent(vevents, plan, rows) + } else if id == vs.versionTableID { + log.Infof("In vstreamer registering version event") + vs.sh.RegisterVersionEvent() + vevent := &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_VERSION, + } + vevents = append(vevents, vevent) + } else { vevents, err = vs.processRowEvent(vevents, plan, rows) } @@ -561,8 +579,41 @@ func (vs *vstreamer) buildJournalPlan(id uint64, tm *mysql.TableMap) error { return nil } +func (vs *vstreamer) buildVersionPlan(id uint64, tm *mysql.TableMap) error { + conn, err := vs.cp.Connect(vs.ctx) + if err != nil { + return err + } + defer conn.Close() + qr, err := conn.ExecuteFetch("select * from _vt.schema_version where 1 != 1", 1, true) + if err != nil { + return err + } + fields := qr.Fields + if len(fields) < len(tm.Types) { + return fmt.Errorf("cannot determine table columns for %s: event has %v, schema as %v", tm.Name, tm.Types, fields) + } + table := &Table{ + Name: "_vt.schema_version", + Fields: fields[:len(tm.Types)], + } + // Build a normal table plan, which means, return all rows + // and columns as is. Special handling is done when we actually + // receive the row event. We'll build a JOURNAL event instead. + plan, err := buildREPlan(table, nil, "") + if err != nil { + return err + } + vs.plans[id] = &streamerPlan{ + Plan: plan, + TableMap: tm, + } + vs.versionTableID = id + return nil +} + func (vs *vstreamer) buildTablePlan(id uint64, tm *mysql.TableMap) (*binlogdatapb.VEvent, error) { - cols, err := vs.buildTableColumns(id, tm) + cols, err := vs.buildTableColumns(tm) if err != nil { return nil, err } @@ -592,7 +643,7 @@ func (vs *vstreamer) buildTablePlan(id uint64, tm *mysql.TableMap) (*binlogdatap }, nil } -func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]*querypb.Field, error) { +func (vs *vstreamer) buildTableColumns(tm *mysql.TableMap) ([]*querypb.Field, error) { var fields []*querypb.Field for i, typ := range tm.Types { t, err := sqltypes.MySQLToType(int64(typ), 0) @@ -605,7 +656,7 @@ func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]*queryp }) } - st := vs.se.GetTable(sqlparser.NewTableIdent(tm.Name)) + st := vs.sh.GetTableForPos(sqlparser.NewTableIdent(tm.Name), mysql.EncodePosition(vs.pos)) if st == nil { if vs.filter.FieldEventMode == binlogdatapb.Filter_ERR_ON_MISMATCH { return nil, fmt.Errorf("unknown table %v in schema", tm.Name) @@ -632,7 +683,7 @@ func (vs *vstreamer) buildTableColumns(id uint64, tm *mysql.TableMap) ([]*queryp return fields, nil } -func (vs *vstreamer) processJounalEvent(vevents []*binlogdatapb.VEvent, plan *streamerPlan, rows mysql.Rows) ([]*binlogdatapb.VEvent, error) { +func (vs *vstreamer) processJournalEvent(vevents []*binlogdatapb.VEvent, plan *streamerPlan, rows mysql.Rows) ([]*binlogdatapb.VEvent, error) { // Get DbName params, err := vs.cp.MysqlParams() if err != nil { diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index 49cdbc535ef..a318aca681e 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -17,6 +17,7 @@ limitations under the License. package vstreamer import ( + "context" "fmt" "strconv" "strings" @@ -25,9 +26,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/net/context" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) @@ -37,6 +38,45 @@ type testcase struct { output [][]string } +var numVersionEventsReceived int + +func TestVersion(t *testing.T) { + if testing.Short() { + t.Skip() + } + + oldEngine := engine + defer func() { + engine = oldEngine + }() + + mh := newMockHistorian(env.SchemaEngine) + engine = NewEngine(engine.env, env.SrvTopo, env.SchemaEngine, mh) + engine.Open(env.KeyspaceName, env.Cells[0]) + defer engine.Close() + + execStatements(t, []string{ + "create table _vt.schema_version(id int, pos varbinary(10000), time_updated bigint(20), ddl varchar(10000), schemax blob, primary key(id))", + }) + defer execStatements(t, []string{ + "drop table _vt.schema_version", + }) + engine.se.Reload(context.Background()) + testcases := []testcase{{ + input: []string{ + fmt.Sprintf("insert into _vt.schema_version values(1, 'MariaDB/0-41983-20', 123, 'create table t1', 'abc')"), + }, + // External table events don't get sent. + output: [][]string{{ + `begin`, + `type:VERSION `}, { + `gtid`, + `commit`}}, + }} + runCases(t, nil, testcases, "") + assert.Equal(t, 1, numVersionEventsReceived) +} + func TestFilteredVarBinary(t *testing.T) { if testing.Short() { t.Skip() @@ -1297,6 +1337,7 @@ func runCases(t *testing.T, filter *binlogdatapb.Filter, testcases []testcase, p // If position is 'current', we wait for a heartbeat to be // sure the vstreamer has started. if position == "current" { + log.Infof("Starting stream with current position") expectLog(ctx, t, "current pos", ch, [][]string{{`gtid`, `type:OTHER `}}) } @@ -1327,7 +1368,7 @@ func expectLog(ctx context.Context, t *testing.T, input interface{}, ch <-chan [ select { case allevs, ok := <-ch: if !ok { - t.Fatal("stream ended early") + t.Fatal("expectLog: not ok, stream ended early") } for _, ev := range allevs { // Ignore spurious heartbeats that can happen on slow machines. @@ -1337,16 +1378,16 @@ func expectLog(ctx context.Context, t *testing.T, input interface{}, ch <-chan [ evs = append(evs, ev) } case <-ctx.Done(): - t.Fatal("stream ended early") + t.Fatalf("expectLog: Done(), stream ended early") case <-timer.C: - t.Fatalf("timed out waiting for events: %v", wantset) + t.Fatalf("expectLog: timed out waiting for events: %v", wantset) } if len(evs) != 0 { break } } if len(wantset) != len(evs) { - t.Fatalf("%v: evs\n%v, want\n%v", input, evs, wantset) + t.Fatalf("%v: evs\n%v, want\n%v, >> got length %d, wanted length %d", input, evs, wantset, len(evs), len(wantset)) } for i, want := range wantset { // CurrentTime is not testable. @@ -1374,6 +1415,8 @@ func expectLog(ctx context.Context, t *testing.T, input interface{}, ch <-chan [ } } +var lastPos string + func startStream(ctx context.Context, t *testing.T, filter *binlogdatapb.Filter, position string) <-chan []*binlogdatapb.VEvent { if position == "" { position = masterPosition(t) @@ -1396,11 +1439,22 @@ func vstream(ctx context.Context, t *testing.T, pos string, filter *binlogdatapb } } return engine.Stream(ctx, pos, filter, func(evs []*binlogdatapb.VEvent) error { - t.Logf("Received events: %v", evs) + if t.Name() == "TestVersion" { // emulate tracker only for the version test + for _, ev := range evs { + log.Infof("Original stream: %s event found %v", ev.Type, ev) + if ev.Type == binlogdatapb.VEventType_GTID { + lastPos = ev.Gtid + } + if ev.Type == binlogdatapb.VEventType_DDL { + schemaTracker := schema.NewTracker(env.SchemaEngine) + schemaTracker.SchemaUpdated(lastPos, ev.Ddl, ev.Timestamp) + } + } + } select { case ch <- evs: case <-ctx.Done(): - return fmt.Errorf("stream ended early") + return fmt.Errorf("engine.Stream Done() stream ended early") } return nil }) @@ -1414,7 +1468,6 @@ func execStatement(t *testing.T, query string) { } func execStatements(t *testing.T, queries []string) { - t.Helper() if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { t.Fatal(err) } diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 7268f3342d2..6f88e03c016 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -230,6 +230,7 @@ enum VEventType { // GTIDs. VGTID = 15; JOURNAL = 16; + VERSION = 17; } // RowChange represents one row change. @@ -342,6 +343,16 @@ message VEvent { int64 current_time = 20; } +message MinimalTable { + string name = 1; + repeated query.Field fields = 2; + repeated int64 p_k_columns = 3; +} + +message MinimalSchema { + repeated MinimalTable tables = 1; +} + // VStreamRequest is the payload for VStreamer message VStreamRequest { vtrpc.CallerID effective_caller_id = 1; @@ -378,6 +389,7 @@ message VStreamRowsResponse { // VStreamResultsRequest is the payload for VStreamResults // The ids match VStreamRows, in case we decide to merge the two. +// The ids match VStreamRows, in case we decide to merge the two. message VStreamResultsRequest { vtrpc.CallerID effective_caller_id = 1; query.VTGateCallerID immediate_caller_id = 2; diff --git a/proto/vstreamer.proto b/proto/vstreamer.proto new file mode 100644 index 00000000000..d713f890ad2 --- /dev/null +++ b/proto/vstreamer.proto @@ -0,0 +1,17 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file contains the proto types needed for vstreamer serializing \ No newline at end of file