From be4d68a2a013313a51532206168e39c3cabf767a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 23 Dec 2020 14:34:10 -0500 Subject: [PATCH 01/17] Add a proto that requires vtctld access Signed-off-by: Andrew Mason --- .github/CODEOWNERS | 2 +- go/vt/proto/vtadmin/vtadmin.pb.go | 302 ++++++++++++++++++++++++++---- proto/vtadmin.proto | 24 +++ 3 files changed, 288 insertions(+), 40 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8e47bcbd382..88eb14077c6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,8 +12,8 @@ /go/test/endtoend/vtgate @harshit-gangal @systay /go/vt/discovery @deepthi /go/vt/mysqlctl @deepthi -/go/vt/proto/vtadmin @ajm188 @doeg /go/vt/orchestrator @deepthi @shlomi-noach +/go/vt/proto/vtadmin @ajm188 @doeg /go/vt/schema @shlomi-noach /go/vt/sqlparser @harshit-gangal @systay /go/vt/vtadmin @ajm188 @doeg diff --git a/go/vt/proto/vtadmin/vtadmin.pb.go b/go/vt/proto/vtadmin/vtadmin.pb.go index c65c6cc7f42..6b2429ef346 100644 --- a/go/vt/proto/vtadmin/vtadmin.pb.go +++ b/go/vt/proto/vtadmin/vtadmin.pb.go @@ -14,6 +14,7 @@ import ( status "google.golang.org/grpc/status" topodata "vitess.io/vitess/go/vt/proto/topodata" + vtctldata "vitess.io/vitess/go/vt/proto/vtctldata" ) // Reference imports to suppress errors if they are not otherwise used. @@ -52,7 +53,7 @@ func (x Tablet_ServingState) String() string { } func (Tablet_ServingState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_609739e22a0a50b3, []int{1, 0} + return fileDescriptor_609739e22a0a50b3, []int{2, 0} } // Cluster represents information about a Vitess cluster. @@ -103,6 +104,55 @@ func (m *Cluster) GetName() string { return "" } +// Keyspace represents information about a keyspace in a particular Vitess +// cluster. +type Keyspace struct { + Cluster *Cluster `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"` + Keyspace *vtctldata.Keyspace `protobuf:"bytes,2,opt,name=keyspace,proto3" json:"keyspace,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Keyspace) Reset() { *m = Keyspace{} } +func (m *Keyspace) String() string { return proto.CompactTextString(m) } +func (*Keyspace) ProtoMessage() {} +func (*Keyspace) Descriptor() ([]byte, []int) { + return fileDescriptor_609739e22a0a50b3, []int{1} +} + +func (m *Keyspace) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Keyspace.Unmarshal(m, b) +} +func (m *Keyspace) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Keyspace.Marshal(b, m, deterministic) +} +func (m *Keyspace) XXX_Merge(src proto.Message) { + xxx_messageInfo_Keyspace.Merge(m, src) +} +func (m *Keyspace) XXX_Size() int { + return xxx_messageInfo_Keyspace.Size(m) +} +func (m *Keyspace) XXX_DiscardUnknown() { + xxx_messageInfo_Keyspace.DiscardUnknown(m) +} + +var xxx_messageInfo_Keyspace proto.InternalMessageInfo + +func (m *Keyspace) GetCluster() *Cluster { + if m != nil { + return m.Cluster + } + return nil +} + +func (m *Keyspace) GetKeyspace() *vtctldata.Keyspace { + if m != nil { + return m.Keyspace + } + return nil +} + // Tablet groups the topo information of a tablet together with the Vitess // cluster it belongs to. type Tablet struct { @@ -118,7 +168,7 @@ func (m *Tablet) Reset() { *m = Tablet{} } func (m *Tablet) String() string { return proto.CompactTextString(m) } func (*Tablet) ProtoMessage() {} func (*Tablet) Descriptor() ([]byte, []int) { - return fileDescriptor_609739e22a0a50b3, []int{1} + return fileDescriptor_609739e22a0a50b3, []int{2} } func (m *Tablet) XXX_Unmarshal(b []byte) error { @@ -160,6 +210,54 @@ func (m *Tablet) GetState() Tablet_ServingState { return Tablet_UNKNOWN } +// Vtctld represents information about a single Vtctld host. +type Vtctld struct { + Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` + Cluster *Cluster `protobuf:"bytes,2,opt,name=cluster,proto3" json:"cluster,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Vtctld) Reset() { *m = Vtctld{} } +func (m *Vtctld) String() string { return proto.CompactTextString(m) } +func (*Vtctld) ProtoMessage() {} +func (*Vtctld) Descriptor() ([]byte, []int) { + return fileDescriptor_609739e22a0a50b3, []int{3} +} + +func (m *Vtctld) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Vtctld.Unmarshal(m, b) +} +func (m *Vtctld) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Vtctld.Marshal(b, m, deterministic) +} +func (m *Vtctld) XXX_Merge(src proto.Message) { + xxx_messageInfo_Vtctld.Merge(m, src) +} +func (m *Vtctld) XXX_Size() int { + return xxx_messageInfo_Vtctld.Size(m) +} +func (m *Vtctld) XXX_DiscardUnknown() { + xxx_messageInfo_Vtctld.DiscardUnknown(m) +} + +var xxx_messageInfo_Vtctld proto.InternalMessageInfo + +func (m *Vtctld) GetHostname() string { + if m != nil { + return m.Hostname + } + return "" +} + +func (m *Vtctld) GetCluster() *Cluster { + if m != nil { + return m.Cluster + } + return nil +} + // VTGate represents information about a single VTGate host. type VTGate struct { // Hostname is the shortname of the VTGate. @@ -183,7 +281,7 @@ func (m *VTGate) Reset() { *m = VTGate{} } func (m *VTGate) String() string { return proto.CompactTextString(m) } func (*VTGate) ProtoMessage() {} func (*VTGate) Descriptor() ([]byte, []int) { - return fileDescriptor_609739e22a0a50b3, []int{2} + return fileDescriptor_609739e22a0a50b3, []int{4} } func (m *VTGate) XXX_Unmarshal(b []byte) error { @@ -250,7 +348,7 @@ func (m *GetGatesRequest) Reset() { *m = GetGatesRequest{} } func (m *GetGatesRequest) String() string { return proto.CompactTextString(m) } func (*GetGatesRequest) ProtoMessage() {} func (*GetGatesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_609739e22a0a50b3, []int{3} + return fileDescriptor_609739e22a0a50b3, []int{5} } func (m *GetGatesRequest) XXX_Unmarshal(b []byte) error { @@ -289,7 +387,7 @@ func (m *GetGatesResponse) Reset() { *m = GetGatesResponse{} } func (m *GetGatesResponse) String() string { return proto.CompactTextString(m) } func (*GetGatesResponse) ProtoMessage() {} func (*GetGatesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_609739e22a0a50b3, []int{4} + return fileDescriptor_609739e22a0a50b3, []int{6} } func (m *GetGatesResponse) XXX_Unmarshal(b []byte) error { @@ -317,6 +415,84 @@ func (m *GetGatesResponse) GetGates() []*VTGate { return nil } +type GetKeyspacesRequest struct { + ClusterIds []string `protobuf:"bytes,1,rep,name=cluster_ids,json=clusterIds,proto3" json:"cluster_ids,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetKeyspacesRequest) Reset() { *m = GetKeyspacesRequest{} } +func (m *GetKeyspacesRequest) String() string { return proto.CompactTextString(m) } +func (*GetKeyspacesRequest) ProtoMessage() {} +func (*GetKeyspacesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_609739e22a0a50b3, []int{7} +} + +func (m *GetKeyspacesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetKeyspacesRequest.Unmarshal(m, b) +} +func (m *GetKeyspacesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetKeyspacesRequest.Marshal(b, m, deterministic) +} +func (m *GetKeyspacesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetKeyspacesRequest.Merge(m, src) +} +func (m *GetKeyspacesRequest) XXX_Size() int { + return xxx_messageInfo_GetKeyspacesRequest.Size(m) +} +func (m *GetKeyspacesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetKeyspacesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetKeyspacesRequest proto.InternalMessageInfo + +func (m *GetKeyspacesRequest) GetClusterIds() []string { + if m != nil { + return m.ClusterIds + } + return nil +} + +type GetKeyspacesResponse struct { + Keyspaces []*Keyspace `protobuf:"bytes,1,rep,name=keyspaces,proto3" json:"keyspaces,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetKeyspacesResponse) Reset() { *m = GetKeyspacesResponse{} } +func (m *GetKeyspacesResponse) String() string { return proto.CompactTextString(m) } +func (*GetKeyspacesResponse) ProtoMessage() {} +func (*GetKeyspacesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_609739e22a0a50b3, []int{8} +} + +func (m *GetKeyspacesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetKeyspacesResponse.Unmarshal(m, b) +} +func (m *GetKeyspacesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetKeyspacesResponse.Marshal(b, m, deterministic) +} +func (m *GetKeyspacesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetKeyspacesResponse.Merge(m, src) +} +func (m *GetKeyspacesResponse) XXX_Size() int { + return xxx_messageInfo_GetKeyspacesResponse.Size(m) +} +func (m *GetKeyspacesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetKeyspacesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetKeyspacesResponse proto.InternalMessageInfo + +func (m *GetKeyspacesResponse) GetKeyspaces() []*Keyspace { + if m != nil { + return m.Keyspaces + } + return nil +} + type GetTabletRequest struct { Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` // ClusterIDs is an optional parameter to narrow the scope of the search, if @@ -332,7 +508,7 @@ func (m *GetTabletRequest) Reset() { *m = GetTabletRequest{} } func (m *GetTabletRequest) String() string { return proto.CompactTextString(m) } func (*GetTabletRequest) ProtoMessage() {} func (*GetTabletRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_609739e22a0a50b3, []int{5} + return fileDescriptor_609739e22a0a50b3, []int{9} } func (m *GetTabletRequest) XXX_Unmarshal(b []byte) error { @@ -378,7 +554,7 @@ func (m *GetTabletsRequest) Reset() { *m = GetTabletsRequest{} } func (m *GetTabletsRequest) String() string { return proto.CompactTextString(m) } func (*GetTabletsRequest) ProtoMessage() {} func (*GetTabletsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_609739e22a0a50b3, []int{6} + return fileDescriptor_609739e22a0a50b3, []int{10} } func (m *GetTabletsRequest) XXX_Unmarshal(b []byte) error { @@ -417,7 +593,7 @@ func (m *GetTabletsResponse) Reset() { *m = GetTabletsResponse{} } func (m *GetTabletsResponse) String() string { return proto.CompactTextString(m) } func (*GetTabletsResponse) ProtoMessage() {} func (*GetTabletsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_609739e22a0a50b3, []int{7} + return fileDescriptor_609739e22a0a50b3, []int{11} } func (m *GetTabletsResponse) XXX_Unmarshal(b []byte) error { @@ -448,10 +624,14 @@ func (m *GetTabletsResponse) GetTablets() []*Tablet { func init() { proto.RegisterEnum("vtadmin.Tablet_ServingState", Tablet_ServingState_name, Tablet_ServingState_value) proto.RegisterType((*Cluster)(nil), "vtadmin.Cluster") + proto.RegisterType((*Keyspace)(nil), "vtadmin.Keyspace") proto.RegisterType((*Tablet)(nil), "vtadmin.Tablet") + proto.RegisterType((*Vtctld)(nil), "vtadmin.Vtctld") proto.RegisterType((*VTGate)(nil), "vtadmin.VTGate") proto.RegisterType((*GetGatesRequest)(nil), "vtadmin.GetGatesRequest") proto.RegisterType((*GetGatesResponse)(nil), "vtadmin.GetGatesResponse") + proto.RegisterType((*GetKeyspacesRequest)(nil), "vtadmin.GetKeyspacesRequest") + proto.RegisterType((*GetKeyspacesResponse)(nil), "vtadmin.GetKeyspacesResponse") proto.RegisterType((*GetTabletRequest)(nil), "vtadmin.GetTabletRequest") proto.RegisterType((*GetTabletsRequest)(nil), "vtadmin.GetTabletsRequest") proto.RegisterType((*GetTabletsResponse)(nil), "vtadmin.GetTabletsResponse") @@ -460,37 +640,43 @@ func init() { func init() { proto.RegisterFile("vtadmin.proto", fileDescriptor_609739e22a0a50b3) } var fileDescriptor_609739e22a0a50b3 = []byte{ - // 474 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x5f, 0x8b, 0xd3, 0x4e, - 0x14, 0x6d, 0xb2, 0xdb, 0x66, 0x73, 0xf3, 0xfb, 0xb5, 0xf5, 0x3e, 0xc5, 0xb8, 0x60, 0x19, 0x54, - 0xaa, 0x60, 0x03, 0xd1, 0x97, 0x3e, 0xc9, 0x2a, 0x52, 0x16, 0x21, 0x85, 0x69, 0xad, 0xe0, 0xcb, - 0x92, 0x6d, 0x86, 0x1a, 0xcc, 0x76, 0x62, 0x67, 0xb6, 0xe0, 0x17, 0xf1, 0x5b, 0x09, 0x7e, 0x24, - 0x99, 0x3f, 0x49, 0xbb, 0xed, 0xa2, 0xbe, 0xdd, 0x7b, 0xcf, 0x39, 0x77, 0xce, 0x3d, 0x6d, 0xe0, - 0xff, 0xad, 0xcc, 0xf2, 0x9b, 0x62, 0x3d, 0xaa, 0x36, 0x5c, 0x72, 0xf4, 0x6c, 0x1b, 0x75, 0x25, - 0xaf, 0x78, 0x9e, 0xc9, 0xcc, 0x00, 0xe4, 0x25, 0x78, 0xef, 0xca, 0x5b, 0x21, 0xd9, 0x06, 0xbb, - 0xe0, 0x16, 0x79, 0xe8, 0x0c, 0x9c, 0xa1, 0x4f, 0xdd, 0x22, 0x47, 0x84, 0xd3, 0x75, 0x76, 0xc3, - 0x42, 0x57, 0x4f, 0x74, 0x4d, 0x7e, 0x3a, 0xd0, 0x99, 0x67, 0xd7, 0x25, 0x93, 0xf8, 0x02, 0xbc, - 0xa5, 0x51, 0x6a, 0x4d, 0x90, 0xf4, 0x47, 0xf5, 0x9b, 0x76, 0x23, 0xad, 0x09, 0x38, 0x84, 0x8e, - 0xd4, 0x2a, 0xbd, 0x4c, 0x51, 0x1b, 0x1b, 0x66, 0x1b, 0xb5, 0x38, 0x26, 0xd0, 0x16, 0x32, 0x93, - 0x2c, 0x3c, 0x19, 0x38, 0xc3, 0x6e, 0x72, 0xde, 0xec, 0x34, 0xbc, 0xd1, 0x8c, 0x6d, 0xb6, 0xc5, - 0x7a, 0x35, 0x53, 0x1c, 0x6a, 0xa8, 0x64, 0x0c, 0xff, 0xed, 0x8f, 0x31, 0x00, 0xef, 0x63, 0xfa, - 0x21, 0x9d, 0x7e, 0x4a, 0xfb, 0x2d, 0xd5, 0xcc, 0xde, 0xd3, 0xc5, 0x65, 0x3a, 0xe9, 0x3b, 0xd8, - 0x83, 0x20, 0x9d, 0xce, 0xaf, 0xea, 0x81, 0x4b, 0x7e, 0x38, 0xd0, 0x59, 0xcc, 0x27, 0x4a, 0x15, - 0xc1, 0xd9, 0x17, 0x2e, 0xa4, 0x3e, 0xd9, 0x84, 0xd0, 0xf4, 0x2a, 0x8a, 0x8a, 0xf3, 0xb2, 0x8e, - 0x42, 0xd5, 0x6a, 0xb6, 0x64, 0x65, 0xa9, 0x8d, 0xfa, 0x54, 0xd7, 0xfb, 0x99, 0x9c, 0xfe, 0x2d, - 0x93, 0x73, 0xf0, 0xbf, 0xb2, 0xef, 0xa2, 0xca, 0x96, 0x4c, 0x84, 0xed, 0xc1, 0xc9, 0xd0, 0xa7, - 0xbb, 0x01, 0x49, 0xa0, 0x37, 0x61, 0x52, 0x19, 0x13, 0x94, 0x7d, 0xbb, 0x65, 0x42, 0xe2, 0x63, - 0x08, 0xac, 0xf6, 0xaa, 0xc8, 0x45, 0xe8, 0x68, 0x09, 0xd8, 0xd1, 0x65, 0x2e, 0xc8, 0x18, 0xfa, - 0x3b, 0x8d, 0xa8, 0xf8, 0x5a, 0x30, 0x7c, 0x0a, 0xed, 0x95, 0x1a, 0x68, 0x7a, 0x90, 0xf4, 0x1a, - 0x3f, 0xe6, 0x6a, 0x6a, 0x50, 0x32, 0xd5, 0x52, 0xfb, 0x5b, 0xd8, 0xf7, 0xfe, 0x14, 0xc8, 0x81, - 0x17, 0xf7, 0xc8, 0xcb, 0x6b, 0x78, 0xd0, 0x2c, 0xfc, 0xf7, 0x0b, 0xde, 0x00, 0xee, 0xab, 0xec, - 0x0d, 0xcf, 0xc1, 0x33, 0xff, 0x8e, 0xe3, 0x2b, 0xac, 0xe3, 0x1a, 0x4f, 0x7e, 0x39, 0xe0, 0x2d, - 0xe6, 0x17, 0x0a, 0xc3, 0x0b, 0x38, 0xab, 0xe3, 0xc0, 0xb0, 0x51, 0x1c, 0xa4, 0x1a, 0x3d, 0xbc, - 0x07, 0x31, 0xef, 0x92, 0x16, 0x8e, 0xc1, 0x6f, 0xfc, 0xe0, 0x1d, 0xe6, 0x9d, 0xa8, 0xa2, 0x43, - 0x43, 0xa4, 0x85, 0x13, 0x80, 0xdd, 0x29, 0x18, 0x1d, 0x6b, 0x1b, 0x07, 0x8f, 0xee, 0xc5, 0x6a, - 0x0f, 0x6f, 0x9f, 0x7d, 0x7e, 0xb2, 0x2d, 0x24, 0x13, 0x62, 0x54, 0xf0, 0xd8, 0x54, 0xf1, 0x8a, - 0xc7, 0x5b, 0x19, 0xeb, 0x2f, 0x38, 0xb6, 0xe2, 0xeb, 0x8e, 0x6e, 0x5f, 0xfd, 0x0e, 0x00, 0x00, - 0xff, 0xff, 0x34, 0xe6, 0x6b, 0x4d, 0xfa, 0x03, 0x00, 0x00, + // 568 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x51, 0x8f, 0xd2, 0x40, + 0x10, 0xa6, 0x3d, 0xa0, 0x30, 0x3d, 0x81, 0x9b, 0xf3, 0xa1, 0x56, 0x8c, 0xa4, 0x51, 0x83, 0x26, + 0xb6, 0x49, 0x35, 0x26, 0x3c, 0x99, 0xd3, 0x98, 0xe6, 0x72, 0x11, 0x4c, 0x41, 0x4c, 0x7c, 0xb9, + 0xf4, 0xe8, 0x06, 0x1b, 0x7b, 0x14, 0xd9, 0x3d, 0x12, 0xff, 0x88, 0xaf, 0xfe, 0x22, 0xff, 0x93, + 0xe9, 0xee, 0x76, 0x29, 0x70, 0xde, 0x9d, 0x6f, 0xbb, 0x33, 0xf3, 0xcd, 0xf7, 0x7d, 0x33, 0x9b, + 0x85, 0x7b, 0x6b, 0x16, 0xc5, 0x97, 0xc9, 0xc2, 0x5d, 0xae, 0x32, 0x96, 0xa1, 0x21, 0xaf, 0x76, + 0x8b, 0x65, 0xcb, 0x2c, 0x8e, 0x58, 0x24, 0x12, 0x76, 0x7b, 0xcd, 0x66, 0x2c, 0xdd, 0x04, 0x9c, + 0x97, 0x60, 0xbc, 0x4f, 0xaf, 0x28, 0x23, 0x2b, 0x6c, 0x81, 0x9e, 0xc4, 0x96, 0xd6, 0xd3, 0xfa, + 0xcd, 0x50, 0x4f, 0x62, 0x44, 0xa8, 0x2e, 0xa2, 0x4b, 0x62, 0xe9, 0x3c, 0xc2, 0xcf, 0xce, 0x1c, + 0x1a, 0x67, 0xe4, 0x27, 0x5d, 0x46, 0x33, 0x82, 0x2f, 0xc0, 0x98, 0x09, 0x28, 0x07, 0x99, 0x7e, + 0xc7, 0x2d, 0x54, 0xc8, 0x96, 0x61, 0x51, 0x80, 0x1e, 0x34, 0xbe, 0x4b, 0x1c, 0xef, 0x67, 0xfa, + 0xc7, 0xee, 0x46, 0x4a, 0xd1, 0x32, 0x54, 0x45, 0xce, 0x1f, 0x0d, 0xea, 0x93, 0xe8, 0x22, 0x25, + 0xec, 0xbf, 0x78, 0xfa, 0x50, 0x67, 0x1c, 0x25, 0x59, 0x3a, 0xae, 0x1a, 0x80, 0xe8, 0x16, 0xca, + 0x3c, 0xfa, 0x50, 0xa3, 0x2c, 0x62, 0xc4, 0x3a, 0xe8, 0x69, 0xfd, 0x96, 0xdf, 0x55, 0x3d, 0x45, + 0x9d, 0x3b, 0x26, 0xab, 0x75, 0xb2, 0x98, 0x8f, 0xf3, 0x9a, 0x50, 0x94, 0x3a, 0x03, 0x38, 0x2c, + 0x87, 0xd1, 0x04, 0xe3, 0xf3, 0xf0, 0x6c, 0x38, 0xfa, 0x32, 0xec, 0x54, 0xf2, 0xcb, 0xf8, 0x43, + 0x38, 0x3d, 0x1d, 0x06, 0x1d, 0x0d, 0xdb, 0x60, 0x0e, 0x47, 0x93, 0xf3, 0x22, 0xa0, 0x3b, 0x9f, + 0xa0, 0x3e, 0xe5, 0x7e, 0xd1, 0x86, 0xc6, 0xb7, 0x8c, 0x32, 0x3e, 0x5a, 0x31, 0x6c, 0x75, 0x2f, + 0x5b, 0xd5, 0x6f, 0xb1, 0xea, 0xfc, 0xd2, 0xa0, 0x3e, 0x9d, 0x04, 0xb9, 0x8e, 0x9b, 0x5a, 0x22, + 0x54, 0x97, 0x59, 0x96, 0x16, 0x5b, 0xcc, 0xcf, 0x79, 0x6c, 0x46, 0xd2, 0x94, 0x5b, 0x6f, 0x86, + 0xfc, 0x5c, 0xa6, 0xae, 0xde, 0x36, 0xe5, 0x2e, 0x34, 0x8b, 0x45, 0x51, 0xab, 0xd6, 0x3b, 0xe8, + 0x37, 0xc3, 0x4d, 0xc0, 0xf1, 0xa1, 0x1d, 0x10, 0x96, 0x0b, 0xa3, 0x21, 0xf9, 0x71, 0x45, 0x28, + 0xc3, 0xc7, 0x60, 0x4a, 0xec, 0x79, 0x12, 0x53, 0x4b, 0xe3, 0x10, 0x90, 0xa1, 0xd3, 0x98, 0x3a, + 0x03, 0xe8, 0x6c, 0x30, 0x74, 0x99, 0x2d, 0x28, 0xc1, 0xa7, 0x50, 0x9b, 0xe7, 0x01, 0x5e, 0x6e, + 0xfa, 0x6d, 0xa5, 0x47, 0xb8, 0x0e, 0x45, 0xd6, 0x79, 0x03, 0xc7, 0x01, 0x61, 0xc5, 0x13, 0xba, + 0x3b, 0x65, 0x00, 0xf7, 0xb7, 0x71, 0x92, 0xd6, 0x2b, 0x9b, 0x13, 0xd4, 0x47, 0x8a, 0x5a, 0xbd, + 0xd4, 0x92, 0xdf, 0x11, 0xd7, 0x2e, 0x9f, 0x97, 0x64, 0xbf, 0x69, 0x23, 0x3b, 0xca, 0xf4, 0x3d, + 0x65, 0xaf, 0xe1, 0x48, 0x35, 0xbc, 0xbb, 0x9f, 0xb7, 0x80, 0x65, 0x94, 0x74, 0xf3, 0x1c, 0x0c, + 0xf1, 0xe0, 0xf7, 0xc7, 0x28, 0x15, 0x17, 0x79, 0xff, 0xb7, 0x0e, 0xc6, 0x74, 0x72, 0x92, 0xe7, + 0xf0, 0x04, 0x1a, 0xc5, 0x3e, 0xd0, 0x52, 0x88, 0x9d, 0xb5, 0xda, 0x0f, 0xae, 0xc9, 0x08, 0x5e, + 0xa7, 0x82, 0x1f, 0xe1, 0xb0, 0x3c, 0x5f, 0xec, 0x96, 0x8b, 0x77, 0xd7, 0x65, 0x3f, 0xfa, 0x47, + 0x56, 0xb5, 0x1b, 0x40, 0x53, 0xd9, 0xc3, 0x2d, 0xe2, 0xad, 0xc9, 0xdb, 0xbb, 0xfe, 0x9c, 0x0a, + 0x06, 0x00, 0x9b, 0xc9, 0xa0, 0xbd, 0x8f, 0x55, 0x2a, 0x1e, 0x5e, 0x9b, 0x2b, 0x34, 0xbc, 0x7b, + 0xf6, 0xf5, 0xc9, 0x3a, 0x61, 0x84, 0x52, 0x37, 0xc9, 0x3c, 0x71, 0xf2, 0xe6, 0x99, 0xb7, 0x66, + 0x1e, 0xff, 0x4c, 0x3d, 0x09, 0xbe, 0xa8, 0xf3, 0xeb, 0xab, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, + 0x9a, 0xe6, 0x88, 0x48, 0x96, 0x05, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -507,6 +693,8 @@ const _ = grpc.SupportPackageIsVersion4 type VTAdminClient interface { // GetGates returns all gates across all the specified clusters. GetGates(ctx context.Context, in *GetGatesRequest, opts ...grpc.CallOption) (*GetGatesResponse, error) + // GetKeyspaces returns all keyspaces across the specified clusters. + GetKeyspaces(ctx context.Context, in *GetKeyspacesRequest, opts ...grpc.CallOption) (*GetKeyspacesResponse, error) // GetTablet looks up a tablet by hostname across all clusters and returns // the result. GetTablet(ctx context.Context, in *GetTabletRequest, opts ...grpc.CallOption) (*Tablet, error) @@ -531,6 +719,15 @@ func (c *vTAdminClient) GetGates(ctx context.Context, in *GetGatesRequest, opts return out, nil } +func (c *vTAdminClient) GetKeyspaces(ctx context.Context, in *GetKeyspacesRequest, opts ...grpc.CallOption) (*GetKeyspacesResponse, error) { + out := new(GetKeyspacesResponse) + err := c.cc.Invoke(ctx, "/vtadmin.VTAdmin/GetKeyspaces", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *vTAdminClient) GetTablet(ctx context.Context, in *GetTabletRequest, opts ...grpc.CallOption) (*Tablet, error) { out := new(Tablet) err := c.cc.Invoke(ctx, "/vtadmin.VTAdmin/GetTablet", in, out, opts...) @@ -553,6 +750,8 @@ func (c *vTAdminClient) GetTablets(ctx context.Context, in *GetTabletsRequest, o type VTAdminServer interface { // GetGates returns all gates across all the specified clusters. GetGates(context.Context, *GetGatesRequest) (*GetGatesResponse, error) + // GetKeyspaces returns all keyspaces across the specified clusters. + GetKeyspaces(context.Context, *GetKeyspacesRequest) (*GetKeyspacesResponse, error) // GetTablet looks up a tablet by hostname across all clusters and returns // the result. GetTablet(context.Context, *GetTabletRequest) (*Tablet, error) @@ -567,6 +766,9 @@ type UnimplementedVTAdminServer struct { func (*UnimplementedVTAdminServer) GetGates(ctx context.Context, req *GetGatesRequest) (*GetGatesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetGates not implemented") } +func (*UnimplementedVTAdminServer) GetKeyspaces(ctx context.Context, req *GetKeyspacesRequest) (*GetKeyspacesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetKeyspaces not implemented") +} func (*UnimplementedVTAdminServer) GetTablet(ctx context.Context, req *GetTabletRequest) (*Tablet, error) { return nil, status.Errorf(codes.Unimplemented, "method GetTablet not implemented") } @@ -596,6 +798,24 @@ func _VTAdmin_GetGates_Handler(srv interface{}, ctx context.Context, dec func(in return interceptor(ctx, in, info, handler) } +func _VTAdmin_GetKeyspaces_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetKeyspacesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VTAdminServer).GetKeyspaces(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vtadmin.VTAdmin/GetKeyspaces", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VTAdminServer).GetKeyspaces(ctx, req.(*GetKeyspacesRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _VTAdmin_GetTablet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetTabletRequest) if err := dec(in); err != nil { @@ -640,6 +860,10 @@ var _VTAdmin_serviceDesc = grpc.ServiceDesc{ MethodName: "GetGates", Handler: _VTAdmin_GetGates_Handler, }, + { + MethodName: "GetKeyspaces", + Handler: _VTAdmin_GetKeyspaces_Handler, + }, { MethodName: "GetTablet", Handler: _VTAdmin_GetTablet_Handler, diff --git a/proto/vtadmin.proto b/proto/vtadmin.proto index bf9cc73bfe5..dfd5fed38ad 100644 --- a/proto/vtadmin.proto +++ b/proto/vtadmin.proto @@ -22,6 +22,7 @@ option go_package = "vitess.io/vitess/go/vt/proto/vtadmin"; package vtadmin; import "topodata.proto"; +import "vtctldata.proto"; /* Services */ @@ -30,6 +31,8 @@ import "topodata.proto"; service VTAdmin { // GetGates returns all gates across all the specified clusters. rpc GetGates(GetGatesRequest) returns (GetGatesResponse) {}; + // GetKeyspaces returns all keyspaces across the specified clusters. + rpc GetKeyspaces(GetKeyspacesRequest) returns (GetKeyspacesResponse) {}; // GetTablet looks up a tablet by hostname across all clusters and returns // the result. rpc GetTablet(GetTabletRequest) returns (Tablet) {}; @@ -45,6 +48,13 @@ message Cluster { string name = 2; } +// Keyspace represents information about a keyspace in a particular Vitess +// cluster. +message Keyspace { + Cluster cluster = 1; + vtctldata.Keyspace keyspace = 2; +} + // Tablet groups the topo information of a tablet together with the Vitess // cluster it belongs to. message Tablet { @@ -60,6 +70,12 @@ message Tablet { ServingState state = 3; } +// Vtctld represents information about a single Vtctld host. +message Vtctld { + string hostname = 1; + Cluster cluster = 2; +} + // VTGate represents information about a single VTGate host. message VTGate { // Hostname is the shortname of the VTGate. @@ -86,6 +102,14 @@ message GetGatesResponse { repeated VTGate gates = 1; } +message GetKeyspacesRequest { + repeated string cluster_ids = 1; +} + +message GetKeyspacesResponse { + repeated Keyspace keyspaces = 1; +} + message GetTabletRequest { string hostname = 1; // ClusterIDs is an optional parameter to narrow the scope of the search, if From eecbf8c2cbeda4a4eafbc3b8c420955c51430bc5 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 23 Dec 2020 18:49:43 -0500 Subject: [PATCH 02/17] Start implementing vtctld discovery Signed-off-by: Andrew Mason --- go/vt/vtadmin/cluster/discovery/discovery.go | 18 +++ .../cluster/discovery/discovery_consul.go | 114 ++++++++++++++++++ .../discovery/fakediscovery/discovery.go | 80 +++++++++++- 3 files changed, 211 insertions(+), 1 deletion(-) diff --git a/go/vt/vtadmin/cluster/discovery/discovery.go b/go/vt/vtadmin/cluster/discovery/discovery.go index 3dbcb41f996..7e1a65a1180 100644 --- a/go/vt/vtadmin/cluster/discovery/discovery.go +++ b/go/vt/vtadmin/cluster/discovery/discovery.go @@ -33,6 +33,9 @@ var ( // ErrNoVTGates should be returned from DiscoverVTGate* methods when they // are unable to find any vtgates for the given filter/query/tags. ErrNoVTGates = errors.New("no vtgates found") + // ErrNoVtctlds should be returned from DiscoverVtctld* methods when they + // are unable to find any vtctlds for the given filter/query/tags. + ErrNoVtctlds = errors.New("no vtctlds found") ) // Discovery defines the interface that service discovery plugins must @@ -53,6 +56,21 @@ type Discovery interface { // Tags can optionally be used to filter gates. Order of the gates is not // specified by the interface, and can be implementation-specific. DiscoverVTGates(ctx context.Context, tags []string) ([]*vtadminpb.VTGate, error) + // DiscoverVtctld returns a vtctld found in the discovery service. + // Tags can optionally be used to filter the set of potential vtctlds + // further. Which vtctld in a set of found vtctlds is returned is not + // specified by the interface, and can be implementation-specific. + DiscoverVtctld(ctx context.Context, tags []string) (*vtadminpb.Vtctld, error) + // DiscoverVtctldAddr returns the address of a of vtctld found in the + // discovery service. Tags can optionally be used to filter the set of + // potential vtctld further. Which gate in a set of found vtctld is used to + // return an address is not specified by the interface, and can be + // implementation-specific. + DiscoverVtctldAddr(ctx context.Context, tags []string) (string, error) + // DiscoverVtctlds returns a list of vtctlds found in the discovery service. + // Tags can optionally be used to filter vtctlds. Order of the vtctlds is + // not specified by the interface, and can be implementation-specific. + DiscoverVtctlds(ctx context.Context, tags []string) ([]*vtadminpb.Vtctld, error) } // Factory represents a function that can create a Discovery implementation. diff --git a/go/vt/vtadmin/cluster/discovery/discovery_consul.go b/go/vt/vtadmin/cluster/discovery/discovery_consul.go index e67daabe10b..a452b65600c 100644 --- a/go/vt/vtadmin/cluster/discovery/discovery_consul.go +++ b/go/vt/vtadmin/cluster/discovery/discovery_consul.go @@ -48,6 +48,11 @@ type ConsulDiscovery struct { vtgateCellTag string vtgateKeyspacesToWatchTag string vtgateAddrTmpl *template.Template + + /* vtctld options */ + vtctldDatacenter string + vtctldService string + vtctldAddrTmpl *template.Template } // NewConsul returns a ConsulDiscovery for the given cluster. Args are a slice @@ -77,6 +82,7 @@ func NewConsul(cluster *vtadminpb.Cluster, flags *pflag.FlagSet, args []string) flags.StringVar(&disco.queryOptions.Token, "token", "", "consul ACL token to use for requests") flags.BoolVar(&disco.passingOnly, "passing-only", true, "whether to include only nodes passing healthchecks") + /* vtgate discovery config options */ flags.StringVar(&disco.vtgateService, "vtgate-service-name", "vtgate", "consul service name vtgates register as") flags.StringVar(&disco.vtgatePoolTag, "vtgate-pool-tag", "pool", "consul service tag to group vtgates by pool") flags.StringVar(&disco.vtgateCellTag, "vtgate-cell-tag", "cell", "consul service tag to group vtgates by cell") @@ -90,6 +96,16 @@ func NewConsul(cluster *vtadminpb.Cluster, flags *pflag.FlagSet, args []string) "The meta information about the cluster is provided to the template via {{ .Cluster }}. "+ "Used once during initialization.") + /* vtctld discovery config options */ + flags.StringVar(&disco.vtctldService, "vtctld-service-name", "vtctld", "consul service name vtctlds register as") + + vtctldAddrTmplStr := flags.String("vtctld-addr-tmpl", "{{ .Hostname }}", + "Go template string to produce a dialable address from a *vtadminpb.Vtctld") + vtctldDatacenterTmplStr := flags.String("vtctld-datacenter-tmpl", "", + "Go template string to generate the datacenter for vtgate consul queries. "+ + "The cluster name is provided to the template via {{ .Cluster }}. "+ + "Used once during initialization.") + if err := flags.Parse(args); err != nil { return nil, err } @@ -119,6 +135,31 @@ func NewConsul(cluster *vtadminpb.Cluster, flags *pflag.FlagSet, args []string) return nil, err } + if *vtctldDatacenterTmplStr != "" { + tmpl, err := template.New("consul-vtctld-datacenter-" + cluster.Id).Parse(*vtctldDatacenterTmplStr) + if err != nil { + return nil, err + } + + buf := bytes.NewBuffer(nil) + err = tmpl.Execute(buf, &struct { + Cluster *vtadminpb.Cluster + }{ + Cluster: cluster, + }) + + if err != nil { + return nil, err + } + + disco.vtctldDatacenter = buf.String() + } + + disco.vtctldAddrTmpl, err = template.New("consul-vtctld-address-template").Parse(*vtctldAddrTmplStr) + if err != nil { + return nil, err + } + return disco, nil } @@ -222,6 +263,79 @@ func (c *ConsulDiscovery) discoverVTGates(_ context.Context, tags []string) ([]* return vtgates, nil } +// DiscoverVtctld is part of the Discovery interface. +func (c *ConsulDiscovery) DiscoverVtctld(ctx context.Context, tags []string) (*vtadminpb.Vtctld, error) { + span, ctx := trace.NewSpan(ctx, "ConsulDiscovery.DiscoverVtctld") + defer span.Finish() + + return c.discoverVtctld(ctx, tags) +} + +func (c *ConsulDiscovery) discoverVtctld(ctx context.Context, tags []string) (*vtadminpb.Vtctld, error) { + vtctlds, err := c.discoverVtctlds(ctx, tags) + if err != nil { + return nil, err + } + + if len(vtctlds) == 0 { + return nil, ErrNoVtctlds + } + + return vtctlds[rand.Intn(len(vtctlds))], nil +} + +// DiscoverVtctldAddr is part of the Discovery interface. +func (c *ConsulDiscovery) DiscoverVtctldAddr(ctx context.Context, tags []string) (string, error) { + span, ctx := trace.NewSpan(ctx, "ConsulDiscovery.DiscoverVtctldAddr") + defer span.Finish() + + vtctld, err := c.discoverVtctld(ctx, tags) + if err != nil { + return "", err + } + + buf := bytes.NewBuffer(nil) + if err := c.vtctldAddrTmpl.Execute(buf, vtctld); err != nil { + return "", err + } + + return buf.String(), nil +} + +// DiscoverVtctlds is part of the Discovery interface. +func (c *ConsulDiscovery) DiscoverVtctlds(ctx context.Context, tags []string) ([]*vtadminpb.Vtctld, error) { + span, ctx := trace.NewSpan(ctx, "ConsulDiscovery.DiscoverVtctlds") + defer span.Finish() + + return c.discoverVtctlds(ctx, tags) +} + +func (c *ConsulDiscovery) discoverVtctlds(_ context.Context, tags []string) ([]*vtadminpb.Vtctld, error) { + opts := c.getQueryOptions() + opts.Datacenter = c.vtctldDatacenter + + entries, _, err := c.client.Health().ServiceMultipleTags(c.vtctldService, tags, c.passingOnly, &opts) + if err != nil { + return nil, err + } + + vtctlds := make([]*vtadminpb.Vtctld, len(entries)) + + for i, entry := range entries { + vtctld := &vtadminpb.Vtctld{ + Cluster: &vtadminpb.Cluster{ + Id: c.cluster.Id, + Name: c.cluster.Name, + }, + Hostname: entry.Node.Node, + } + + vtctlds[i] = vtctld + } + + return vtctlds, nil +} + // getQueryOptions returns a shallow copy so we can swap in the vtgateDatacenter. // If we were to set it directly, we'd need a mutex to guard against concurrent // vtgate and (soon) vtctld queries. diff --git a/go/vt/vtadmin/cluster/discovery/fakediscovery/discovery.go b/go/vt/vtadmin/cluster/discovery/fakediscovery/discovery.go index 896902d41d2..8ff0030556b 100644 --- a/go/vt/vtadmin/cluster/discovery/fakediscovery/discovery.go +++ b/go/vt/vtadmin/cluster/discovery/fakediscovery/discovery.go @@ -28,6 +28,12 @@ import ( vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" ) +type vtctlds struct { + byTag map[string][]*vtadminpb.Vtctld + byName map[string]*vtadminpb.Vtctld + shouldErr bool +} + type gates struct { byTag map[string][]*vtadminpb.VTGate byName map[string]*vtadminpb.VTGate @@ -36,7 +42,8 @@ type gates struct { // Fake is a fake discovery implementation for use in testing. type Fake struct { - gates *gates + gates *gates + vtctlds *vtctlds } // New returns a new fake. @@ -46,6 +53,10 @@ func New() *Fake { byTag: map[string][]*vtadminpb.VTGate{}, byName: map[string]*vtadminpb.VTGate{}, }, + vtctlds: &vtctlds{ + byTag: map[string][]*vtadminpb.Vtctld{}, + byName: map[string]*vtadminpb.Vtctld{}, + }, } } @@ -137,3 +148,70 @@ func (d *Fake) DiscoverVTGateAddr(ctx context.Context, tags []string) (string, e return gate.Hostname, nil } + +// DiscoverVtctlds is part of the discover.Discovery interface. +func (d *Fake) DiscoverVtctlds(ctx context.Context, tags []string) ([]*vtadminpb.Vtctld, error) { + if d.vtctlds.shouldErr { + return nil, assert.AnError + } + + if len(tags) == 0 { + results := make([]*vtadminpb.Vtctld, 0, len(d.vtctlds.byName)) + for _, vtctld := range d.vtctlds.byName { + results = append(results, vtctld) + } + + return results, nil + } + + set := d.vtctlds.byName + + for _, tag := range tags { + intermediate := map[string]*vtadminpb.Vtctld{} + + vtctlds, ok := d.vtctlds.byTag[tag] + if !ok { + return []*vtadminpb.Vtctld{}, nil + } + + for _, v := range vtctlds { + if _, ok := set[v.Hostname]; ok { + intermediate[v.Hostname] = v + } + } + + set = intermediate + } + + results := make([]*vtadminpb.Vtctld, 0, len(set)) + + for _, vtctld := range set { + results = append(results, vtctld) + } + + return results, nil +} + +// DiscoverVtctldAddr is part of the discover.Discovery interface. +func (d *Fake) DiscoverVtctldAddr(ctx context.Context, tags []string) (string, error) { + vtctld, err := d.DiscoverVtctld(ctx, tags) + if err != nil { + return "", err + } + + return vtctld.Hostname, nil +} + +// DiscoverVtctld is part of the discover.Discovery interface. +func (d *Fake) DiscoverVtctld(ctx context.Context, tags []string) (*vtadminpb.Vtctld, error) { + vtctlds, err := d.DiscoverVtctlds(ctx, tags) + if err != nil { + return nil, err + } + + if len(vtctlds) == 0 { + return nil, assert.AnError + } + + return vtctlds[rand.Intn(len(vtctlds))], nil +} From b9198f412843acd84e11ac98ea10cc2a1a75fd92 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 23 Dec 2020 22:22:50 -0500 Subject: [PATCH 03/17] Extract the datacenter template generation Signed-off-by: Andrew Mason --- .../cluster/discovery/discovery_consul.go | 52 ++++++++----------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/go/vt/vtadmin/cluster/discovery/discovery_consul.go b/go/vt/vtadmin/cluster/discovery/discovery_consul.go index a452b65600c..48ce42203cc 100644 --- a/go/vt/vtadmin/cluster/discovery/discovery_consul.go +++ b/go/vt/vtadmin/cluster/discovery/discovery_consul.go @@ -111,56 +111,50 @@ func NewConsul(cluster *vtadminpb.Cluster, flags *pflag.FlagSet, args []string) } if *vtgateDatacenterTmplStr != "" { - tmpl, err := template.New("consul-vtgate-datacenter-" + cluster.Id).Parse(*vtgateDatacenterTmplStr) + disco.vtgateDatacenter, err = generateConsulDatacenter("vtgate", cluster, *vtgateDatacenterTmplStr) if err != nil { return nil, err } - - buf := bytes.NewBuffer(nil) - err = tmpl.Execute(buf, &struct { - Cluster *vtadminpb.Cluster - }{ - Cluster: cluster, - }) - - if err != nil { - return nil, err - } - - disco.vtgateDatacenter = buf.String() } - disco.vtgateAddrTmpl, err = template.New("consul-vtgate-address-template").Parse(*vtgateAddrTmplStr) + disco.vtgateAddrTmpl, err = template.New("consul-vtgate-address-template-" + cluster.Id).Parse(*vtgateAddrTmplStr) if err != nil { return nil, err } if *vtctldDatacenterTmplStr != "" { - tmpl, err := template.New("consul-vtctld-datacenter-" + cluster.Id).Parse(*vtctldDatacenterTmplStr) + disco.vtctldDatacenter, err = generateConsulDatacenter("vtctld", cluster, *vtctldDatacenterTmplStr) if err != nil { return nil, err } + } - buf := bytes.NewBuffer(nil) - err = tmpl.Execute(buf, &struct { - Cluster *vtadminpb.Cluster - }{ - Cluster: cluster, - }) + disco.vtctldAddrTmpl, err = template.New("consul-vtctld-address-template-" + cluster.Id).Parse(*vtctldAddrTmplStr) + if err != nil { + return nil, err + } - if err != nil { - return nil, err - } + return disco, nil +} - disco.vtctldDatacenter = buf.String() +func generateConsulDatacenter(component string, cluster *vtadminpb.Cluster, tmplStr string) (string, error) { + tmpl, err := template.New("consul-" + component + "-datacenter-" + cluster.Id).Parse(tmplStr) + if err != nil { + return "", err } - disco.vtctldAddrTmpl, err = template.New("consul-vtctld-address-template").Parse(*vtctldAddrTmplStr) + buf := bytes.NewBuffer(nil) + err = tmpl.Execute(buf, &struct { + Cluster *vtadminpb.Cluster + }{ + Cluster: cluster, + }) + if err != nil { - return nil, err + return "", err } - return disco, nil + return buf.String(), err } // DiscoverVTGate is part of the Discovery interface. From c8d506d944d4c562dce60a3b388b23cbd38fc106 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 6 Jan 2021 20:18:58 -0500 Subject: [PATCH 04/17] Move the LoadFromTemplate helper to a separate package This will allow both vtsql and vtctldclient to access it Signed-off-by: Andrew Mason --- go/vt/vtadmin/credentials/credentials.go | 72 +++++++++++++++++++ .../credentials_test.go | 4 +- go/vt/vtadmin/vtsql/config.go | 30 +------- go/vt/vtadmin/vtsql/credentials.go | 17 ----- 4 files changed, 76 insertions(+), 47 deletions(-) create mode 100644 go/vt/vtadmin/credentials/credentials.go rename go/vt/vtadmin/{vtsql => credentials}/credentials_test.go (96%) diff --git a/go/vt/vtadmin/credentials/credentials.go b/go/vt/vtadmin/credentials/credentials.go new file mode 100644 index 00000000000..54e87dec4f6 --- /dev/null +++ b/go/vt/vtadmin/credentials/credentials.go @@ -0,0 +1,72 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package credentials + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "text/template" + + "vitess.io/vitess/go/vt/grpcclient" +) + +// LoadFromTemplate renders a template string into a path, using the data +// provided as the template data. It then loads the contents at the resulting +// path as a JSON file containing a grpcclient.StaticAuthClientCreds, and +// returns both the parsed credentials as well as the concrete path used. +func LoadFromTemplate(tmplStr string, data interface{}) (*grpcclient.StaticAuthClientCreds, string, error) { + path, err := renderTemplate(tmplStr, data) + if err != nil { + return nil, "", err + } + + creds, err := loadCredentials(path) + if err != nil { + return nil, "", err + } + + return creds, path, nil +} + +func renderTemplate(tmplStr string, data interface{}) (string, error) { + tmpl, err := template.New("").Parse(tmplStr) + if err != nil { + return "", err + } + + buf := bytes.NewBuffer(nil) + if err := tmpl.Execute(buf, data); err != nil { + return "", err + } + + return buf.String(), nil +} + +func loadCredentials(path string) (*grpcclient.StaticAuthClientCreds, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var creds grpcclient.StaticAuthClientCreds + if err := json.Unmarshal(data, &creds); err != nil { + return nil, err + } + + return &creds, nil +} diff --git a/go/vt/vtadmin/vtsql/credentials_test.go b/go/vt/vtadmin/credentials/credentials_test.go similarity index 96% rename from go/vt/vtadmin/vtsql/credentials_test.go rename to go/vt/vtadmin/credentials/credentials_test.go index 1a20c22816b..530f73eddca 100644 --- a/go/vt/vtadmin/vtsql/credentials_test.go +++ b/go/vt/vtadmin/credentials/credentials_test.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Vitess Authors. +Copyright 2021 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package vtsql +package credentials import ( "io/ioutil" diff --git a/go/vt/vtadmin/vtsql/config.go b/go/vt/vtadmin/vtsql/config.go index 1268a0c48a3..285925c5959 100644 --- a/go/vt/vtadmin/vtsql/config.go +++ b/go/vt/vtadmin/vtsql/config.go @@ -17,14 +17,13 @@ limitations under the License. package vtsql import ( - "bytes" "fmt" - "text/template" "github.com/spf13/pflag" "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/vtadmin/cluster/discovery" + "vitess.io/vitess/go/vt/vtadmin/credentials" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" ) @@ -83,7 +82,7 @@ func (c *Config) Parse(args []string) error { var creds *grpcclient.StaticAuthClientCreds if *credentialsTmplStr != "" { - _creds, path, err := c.loadCredentialsFromTemplate(*credentialsTmplStr) + _creds, path, err := credentials.LoadFromTemplate(*credentialsTmplStr, c) if err != nil { return fmt.Errorf("cannot load credentials from path template %s: %w", *credentialsTmplStr, err) } @@ -107,28 +106,3 @@ func (c *Config) Parse(args []string) error { return nil } - -func (c Config) loadCredentialsFromTemplate(tmplStr string) (*grpcclient.StaticAuthClientCreds, string, error) { - path, err := c.renderTemplate(tmplStr) - if err != nil { - return nil, "", err - } - - creds, err := loadCredentials(path) - - return creds, path, err -} - -func (c Config) renderTemplate(tmplStr string) (string, error) { - tmpl, err := template.New("").Parse(tmplStr) - if err != nil { - return "", err - } - - buf := bytes.NewBuffer(nil) - if err := tmpl.Execute(buf, &c); err != nil { - return "", err - } - - return buf.String(), nil -} diff --git a/go/vt/vtadmin/vtsql/credentials.go b/go/vt/vtadmin/vtsql/credentials.go index 70fa31c5ac7..6e06fee4361 100644 --- a/go/vt/vtadmin/vtsql/credentials.go +++ b/go/vt/vtadmin/vtsql/credentials.go @@ -17,9 +17,6 @@ limitations under the License. package vtsql import ( - "encoding/json" - "io/ioutil" - "google.golang.org/grpc/credentials" "vitess.io/vitess/go/vt/grpcclient" @@ -54,17 +51,3 @@ func (creds *StaticAuthCredentials) GetEffectiveUsername() string { func (creds *StaticAuthCredentials) GetUsername() string { return creds.Username } - -func loadCredentials(path string) (*grpcclient.StaticAuthClientCreds, error) { - data, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - var creds grpcclient.StaticAuthClientCreds - if err := json.Unmarshal(data, &creds); err != nil { - return nil, err - } - - return &creds, nil -} From 3370c06fe9f19ed1df885a4998ce76122a177512 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 15:56:25 -0500 Subject: [PATCH 05/17] Add grpcvtctldclient constructor for use in vtadmin Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldclient/client.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/go/vt/vtctl/grpcvtctldclient/client.go b/go/vt/vtctl/grpcvtctldclient/client.go index af3e8e07802..a0f5a14ef77 100644 --- a/go/vt/vtctl/grpcvtctldclient/client.go +++ b/go/vt/vtctl/grpcvtctldclient/client.go @@ -72,6 +72,20 @@ func gRPCVtctldClientFactory(addr string) (vtctldclient.VtctldClient, error) { }, nil } +// NewWithDialOpts returns a vtctldclient.VtctldClient configured with the given +// DialOptions. It is exported for use in vtadmin. +func NewWithDialOpts(addr string, failFast grpcclient.FailFast, opts ...grpc.DialOption) (vtctldclient.VtctldClient, error) { + conn, err := grpcclient.Dial(addr, failFast, opts...) + if err != nil { + return nil, err + } + + return &gRPCVtctldClient{ + cc: conn, + c: vtctlservicepb.NewVtctldClient(conn), + }, nil +} + func (client *gRPCVtctldClient) Close() error { err := client.cc.Close() if err == nil { From 939aff6db781cb3d140ed08e6e31b42e1a62c1b6 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 15:57:30 -0500 Subject: [PATCH 06/17] Add minimally-viable vtctldclient Proxy package Signed-off-by: Andrew Mason --- go/vt/vtadmin/vtctldclient/config.go | 83 +++++++++++++ go/vt/vtadmin/vtctldclient/config_test.go | 93 ++++++++++++++ go/vt/vtadmin/vtctldclient/proxy.go | 141 ++++++++++++++++++++++ 3 files changed, 317 insertions(+) create mode 100644 go/vt/vtadmin/vtctldclient/config.go create mode 100644 go/vt/vtadmin/vtctldclient/config_test.go create mode 100644 go/vt/vtadmin/vtctldclient/proxy.go diff --git a/go/vt/vtadmin/vtctldclient/config.go b/go/vt/vtadmin/vtctldclient/config.go new file mode 100644 index 00000000000..8a4eec7f9d9 --- /dev/null +++ b/go/vt/vtadmin/vtctldclient/config.go @@ -0,0 +1,83 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctldclient + +import ( + "fmt" + + "github.com/spf13/pflag" + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/vtadmin/cluster/discovery" + "vitess.io/vitess/go/vt/vtadmin/credentials" + + vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" +) + +// Config represents the options that modify the behavior of a Proxy. +type Config struct { + Discovery discovery.Discovery + Credentials *grpcclient.StaticAuthClientCreds + + CredentialsPath string + + Cluster *vtadminpb.Cluster +} + +// Parse returns a new config with the given cluster and discovery, after +// attempting to parse the command-line pflags into that Config. See +// (*Config).Parse() for more details. +func Parse(cluster *vtadminpb.Cluster, disco discovery.Discovery, args []string) (*Config, error) { + cfg := &Config{ + Cluster: cluster, + Discovery: disco, + } + + err := cfg.Parse(args) + if err != nil { + return nil, err + } + + return cfg, nil +} + +// Parse reads options specified as command-line pflags (--key=value, note the +// double-dash!) into a Config. It is meant to be called from +// (*cluster.Cluster).New(). +func (c *Config) Parse(args []string) error { + fs := pflag.NewFlagSet("", pflag.ContinueOnError) + + credentialsTmplStr := fs.String("credentials-path-tmpl", "", + "Go template used to specify a path to a credentials file, which is a json file containing "+ + "a Username and Password. Templates are given the context of the vtctldclient.Config, "+ + "and primarily interoplate the cluster name and ID variables.") + + if err := fs.Parse(args); err != nil { + return err + } + + if *credentialsTmplStr != "" { + creds, path, err := credentials.LoadFromTemplate(*credentialsTmplStr, c) + if err != nil { + return fmt.Errorf("cannot load credentials from path template %s: %w", *credentialsTmplStr, err) + } + + c.CredentialsPath = path + c.Credentials = creds + } + + return nil +} diff --git a/go/vt/vtadmin/vtctldclient/config_test.go b/go/vt/vtadmin/vtctldclient/config_test.go new file mode 100644 index 00000000000..987838d3f77 --- /dev/null +++ b/go/vt/vtadmin/vtctldclient/config_test.go @@ -0,0 +1,93 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctldclient + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/grpcclient" + + vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" +) + +func withTempFile(t *testing.T, tmpdir string, name string, f func(*testing.T, *os.File)) { + tmpfile, err := ioutil.TempFile(tmpdir, name) + require.NoError(t, err, "TempFile(%s, %s)", tmpdir, name) + defer os.Remove(tmpfile.Name()) + + f(t, tmpfile) +} + +func TestParse(t *testing.T) { + t.Run("no credentials provided", func(t *testing.T) { + cfg, err := Parse(nil, nil, []string{}) + require.NoError(t, err) + + expected := &Config{ + Cluster: nil, + Discovery: nil, + Credentials: nil, + CredentialsPath: "", + } + assert.Equal(t, expected, cfg) + }) + + t.Run("credential loading", func(t *testing.T) { + withTempFile(t, "", "vtctldclient.config_test.testcluster.*", func(t *testing.T, credsfile *os.File) { + creds := &grpcclient.StaticAuthClientCreds{ + Username: "admin", + Password: "hunter2", + } + + data, err := json.Marshal(creds) + require.NoError(t, err, "cannot marshal credentials %+v into credsfile", creds) + + _, err = credsfile.Write(data) + require.NoError(t, err, "cannot write credentials to file") + + credsdir := filepath.Dir(credsfile.Name()) + baseParts := strings.Split(filepath.Base(credsfile.Name()), ".") + tmplParts := append(baseParts[:2], "{{ .Cluster.Name }}", baseParts[3]) + + args := []string{ + fmt.Sprintf("--credentials-path-tmpl=%s", filepath.Join(credsdir, strings.Join(tmplParts, "."))), + } + + cfg, err := Parse(&vtadminpb.Cluster{Name: "testcluster"}, nil, args) + require.NoError(t, err) + + expected := &Config{ + Cluster: &vtadminpb.Cluster{ + Name: "testcluster", + }, + Discovery: nil, + Credentials: creds, + CredentialsPath: credsfile.Name(), + } + + assert.Equal(t, expected, cfg) + }) + }) +} diff --git a/go/vt/vtadmin/vtctldclient/proxy.go b/go/vt/vtadmin/vtctldclient/proxy.go new file mode 100644 index 00000000000..dc7913c8083 --- /dev/null +++ b/go/vt/vtadmin/vtctldclient/proxy.go @@ -0,0 +1,141 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctldclient + +import ( + "context" + + "google.golang.org/grpc" + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/vtadmin/cluster/discovery" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldclient" + "vitess.io/vitess/go/vt/vtctl/vtctldclient" + + vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" + vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" +) + +// Proxy defines the connection interface of a proxied vtctldclient used by +// VTAdmin clusters. +type Proxy interface { + // Dial opens a gRPC connection to a vtctld in the cluster. If the Proxy + // already has a valid connection, this is a no-op. + Dial(ctx context.Context) error + + // Hostname returns the hostname the Proxy is currently connected to. + Hostname() string + + // Close closes the underlying vtctldclient connection. This is a no-op if + // the Proxy has no current, valid connection. It is safe to call repeatedly. + // Users may call Dial on a previously-closed Proxy to create a new + // connection, but that connection may not be to the same particular vtctld. + Close() error + + vtctlservicepb.VtctldClient +} + +// ClientProxy implements the Proxy interface relying on a discovery.Discovery +// implementation to handle vtctld discovery and connection management. +type ClientProxy struct { + vtctldclient.VtctldClient // embedded to provide easy implementation of the vtctlservicepb.VtctldClient interface + + cluster *vtadminpb.Cluster + creds *grpcclient.StaticAuthClientCreds + discovery discovery.Discovery + + // DialFunc is called to open a new vtctdclient connection. In production, + // this should always be grpcvtctldclient.NewWithDialOpts, but it is + // exported for testing purposes. + DialFunc func(addr string, ff grpcclient.FailFast, opts ...grpc.DialOption) (vtctldclient.VtctldClient, error) + + closed bool + host string +} + +// New returns a ClientProxy to the given cluster. When Dial-ing, it will use +// the given discovery implementation to find a vtctld to connect to, and the +// given creds to dial the underlying gRPC connection, both of which are +// provided by the Config. +// +// It does not open a connection to a vtctld; users must call Dial before first +// use. +func New(cfg *Config) *ClientProxy { + return &ClientProxy{ + cluster: cfg.Cluster, + creds: cfg.Credentials, + discovery: cfg.Discovery, + DialFunc: grpcvtctldclient.NewWithDialOpts, + } +} + +// Dial is part of the Proxy interface. +func (vtctld *ClientProxy) Dial(ctx context.Context) error { + if vtctld.VtctldClient != nil { + if !vtctld.closed { + return nil + } + + // close before reopen. this is safe to call on an already-closed client. + if err := vtctld.Close(); err != nil { + return err + } + } + + addr, err := vtctld.discovery.DiscoverVtctldAddr(ctx, nil) + if err != nil { + return err + } + + opts := []grpc.DialOption{} + if vtctld.creds != nil { + opts = append(opts, grpc.WithPerRPCCredentials(vtctld.creds)) + } + + client, err := vtctld.DialFunc(addr, grpcclient.FailFast(false), opts...) + if err != nil { + return err + } + + vtctld.host = addr + vtctld.VtctldClient = client + vtctld.closed = false + + return nil +} + +// Hostname is part of the Proxy interface. +func (vtctld *ClientProxy) Hostname() string { + return vtctld.host +} + +// Close is part of the Proxy interface. +func (vtctld *ClientProxy) Close() error { + if vtctld.VtctldClient == nil { + vtctld.closed = true + + return nil + } + + err := vtctld.VtctldClient.Close() + if err != nil { + return err + } + + vtctld.closed = true + + return nil +} From e2cc7851c5f38e1bfdb8dc1293f52f26057ebd94 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 16:52:18 -0500 Subject: [PATCH 07/17] Add vtctld config support to cluster, update tests Signed-off-by: Andrew Mason --- go/vt/vtadmin/cluster/cluster.go | 21 +++++++++---- go/vt/vtadmin/cluster/config.go | 5 +++ go/vt/vtadmin/cluster/config_test.go | 19 ++++++++++-- go/vt/vtadmin/cluster/file_config_test.go | 37 +++++++++++++++-------- go/vt/vtadmin/cluster/flags.go | 6 ++++ 5 files changed, 68 insertions(+), 20 deletions(-) diff --git a/go/vt/vtadmin/cluster/cluster.go b/go/vt/vtadmin/cluster/cluster.go index 3f5b0af7e5d..c7a23e0d280 100644 --- a/go/vt/vtadmin/cluster/cluster.go +++ b/go/vt/vtadmin/cluster/cluster.go @@ -20,6 +20,7 @@ import ( "fmt" "vitess.io/vitess/go/vt/vtadmin/cluster/discovery" + "vitess.io/vitess/go/vt/vtadmin/vtctldclient" "vitess.io/vitess/go/vt/vtadmin/vtsql" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" @@ -33,10 +34,8 @@ type Cluster struct { Name string Discovery discovery.Discovery - // (TODO|@amason): after merging #7128, this still requires some additional - // work, so deferring this for now! - // vtctl vtctldclient.VtctldClient - DB vtsql.DB + DB vtsql.DB + Vtctld vtctldclient.Proxy // These fields are kept to power debug endpoints. // (TODO|@amason): Figure out if these are needed or if there's a way to @@ -60,14 +59,24 @@ func New(cfg Config) (*Cluster, error) { cluster.Discovery = disco + protocluster := cluster.ToProto() + vtsqlargs := buildPFlagSlice(cfg.VtSQLFlags) - vtsqlCfg, err := vtsql.Parse(cluster.ToProto(), disco, vtsqlargs) + vtsqlCfg, err := vtsql.Parse(protocluster, disco, vtsqlargs) + if err != nil { + return nil, fmt.Errorf("error while creating vtsql connection config: %w", err) + } + + vtctldargs := buildPFlagSlice(cfg.VtctldFlags) + + vtctldCfg, err := vtctldclient.Parse(protocluster, disco, vtctldargs) if err != nil { - return nil, fmt.Errorf("error while creating vtsql connection: %w", err) + return nil, fmt.Errorf("error while creating vtctldclient proxy config: %w", err) } cluster.DB = vtsql.New(vtsqlCfg) + cluster.Vtctld = vtctldclient.New(vtctldCfg) return cluster, nil } diff --git a/go/vt/vtadmin/cluster/config.go b/go/vt/vtadmin/cluster/config.go index 1b641f60458..bd693e5ad88 100644 --- a/go/vt/vtadmin/cluster/config.go +++ b/go/vt/vtadmin/cluster/config.go @@ -25,6 +25,7 @@ type Config struct { DiscoveryImpl string DiscoveryFlagsByImpl FlagsByImpl VtSQLFlags map[string]string + VtctldFlags map[string]string } // Cluster returns a new cluster instance from the given config. @@ -82,6 +83,7 @@ func (cfg Config) Merge(override Config) Config { DiscoveryImpl: cfg.DiscoveryImpl, DiscoveryFlagsByImpl: map[string]map[string]string{}, VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, } if override.ID != "" { @@ -104,6 +106,9 @@ func (cfg Config) Merge(override Config) Config { mergeStringMap(merged.VtSQLFlags, cfg.VtSQLFlags) mergeStringMap(merged.VtSQLFlags, override.VtSQLFlags) + mergeStringMap(merged.VtctldFlags, cfg.VtctldFlags) + mergeStringMap(merged.VtctldFlags, override.VtctldFlags) + return merged } diff --git a/go/vt/vtadmin/cluster/config_test.go b/go/vt/vtadmin/cluster/config_test.go index 22839b38323..e087fa1a300 100644 --- a/go/vt/vtadmin/cluster/config_test.go +++ b/go/vt/vtadmin/cluster/config_test.go @@ -46,6 +46,7 @@ func TestMergeConfig(t *testing.T) { DiscoveryImpl: "consul", DiscoveryFlagsByImpl: FlagsByImpl{}, VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, }, { @@ -81,11 +82,12 @@ func TestMergeConfig(t *testing.T) { "foo": "baz", }, }, - VtSQLFlags: map[string]string{}, + VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, }, { - name: "merging vtsql flags", + name: "merging vtsql/vtctld flags", base: Config{ ID: "c1", Name: "cluster1", @@ -93,6 +95,10 @@ func TestMergeConfig(t *testing.T) { "one": "one", "two": "2", }, + VtctldFlags: map[string]string{ + "a": "A", + "b": "B", + }, }, override: Config{ ID: "c1", @@ -101,6 +107,10 @@ func TestMergeConfig(t *testing.T) { "two": "two", "three": "three", }, + VtctldFlags: map[string]string{ + "a": "alpha", + "c": "C", + }, }, expected: Config{ ID: "c1", @@ -111,6 +121,11 @@ func TestMergeConfig(t *testing.T) { "two": "two", "three": "three", }, + VtctldFlags: map[string]string{ + "a": "alpha", + "b": "B", + "c": "C", + }, }, }, } diff --git a/go/vt/vtadmin/cluster/file_config_test.go b/go/vt/vtadmin/cluster/file_config_test.go index 00e3108db3c..68d5092df8f 100644 --- a/go/vt/vtadmin/cluster/file_config_test.go +++ b/go/vt/vtadmin/cluster/file_config_test.go @@ -35,16 +35,19 @@ func TestFileConfigUnmarshalYAML(t *testing.T) { name: "simple", yaml: `defaults: discovery: consul - discovery-consul-vtgate-datacenter-tmpl: "dev-{{ .Cluster }}" + discovery-consul-vtctld-datacenter-tmpl: "dev-{{ .Cluster.Name }}" + discovery-consul-vtctld-service-name: vtctld-svc + discovery-consul-vtctld-addr-tmpl: "{{ .Hostname }}.example.com:15000" + discovery-consul-vtgate-datacenter-tmpl: "dev-{{ .Cluster.Name }}" discovery-consul-vtgate-service-name: vtgate-svc discovery-consul-vtgate-pool-tag: type discovery-consul-vtgate-cell-tag: zone - discovery-consul-vtgate-addr-tmpl: "{{ .Name }}.example.com:15999" + discovery-consul-vtgate-addr-tmpl: "{{ .Hostname }}.example.com:15999" clusters: c1: name: testcluster1 - discovery-consul-vtgate-datacenter-tmpl: "dev-{{ .Cluster }}-test" + discovery-consul-vtgate-datacenter-tmpl: "dev-{{ .Cluster.Name }}-test" c2: name: devcluster`, config: FileConfig{ @@ -52,11 +55,14 @@ clusters: DiscoveryImpl: "consul", DiscoveryFlagsByImpl: map[string]map[string]string{ "consul": { - "vtgate-datacenter-tmpl": "dev-{{ .Cluster }}", + "vtctld-datacenter-tmpl": "dev-{{ .Cluster.Name }}", + "vtctld-service-name": "vtctld-svc", + "vtctld-addr-tmpl": "{{ .Hostname }}.example.com:15000", + "vtgate-datacenter-tmpl": "dev-{{ .Cluster.Name }}", "vtgate-service-name": "vtgate-svc", "vtgate-pool-tag": "type", "vtgate-cell-tag": "zone", - "vtgate-addr-tmpl": "{{ .Name }}.example.com:15999", + "vtgate-addr-tmpl": "{{ .Hostname }}.example.com:15999", }, }, }, @@ -66,16 +72,18 @@ clusters: Name: "testcluster1", DiscoveryFlagsByImpl: map[string]map[string]string{ "consul": { - "vtgate-datacenter-tmpl": "dev-{{ .Cluster }}-test", + "vtgate-datacenter-tmpl": "dev-{{ .Cluster.Name }}-test", }, }, - VtSQLFlags: map[string]string{}, + VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, "c2": { ID: "c2", Name: "devcluster", DiscoveryFlagsByImpl: map[string]map[string]string{}, VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, }, }, @@ -153,7 +161,8 @@ func TestCombine(t *testing.T) { "vtgate-datacenter-tmpl": "dev-{{ .Cluster }}", }, }, - VtSQLFlags: map[string]string{}, + VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, { ID: "2", @@ -164,7 +173,8 @@ func TestCombine(t *testing.T) { "vtgate-datacenter-tmpl": "dev-{{ .Cluster }}-test", }, }, - VtSQLFlags: map[string]string{}, + VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, }, }, @@ -212,7 +222,8 @@ func TestCombine(t *testing.T) { "flag": "val", }, }, - VtSQLFlags: map[string]string{}, + VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, { ID: "c2", @@ -223,7 +234,8 @@ func TestCombine(t *testing.T) { "flag": "val", }, }, - VtSQLFlags: map[string]string{}, + VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, { ID: "c3", @@ -234,7 +246,8 @@ func TestCombine(t *testing.T) { "flag": "val", }, }, - VtSQLFlags: map[string]string{}, + VtSQLFlags: map[string]string{}, + VtctldFlags: map[string]string{}, }, }, }, diff --git a/go/vt/vtadmin/cluster/flags.go b/go/vt/vtadmin/cluster/flags.go index bb9b180183a..f0c600109c5 100644 --- a/go/vt/vtadmin/cluster/flags.go +++ b/go/vt/vtadmin/cluster/flags.go @@ -150,6 +150,12 @@ func parseOne(cfg *Config, name string, val string) error { cfg.VtSQLFlags[strings.TrimPrefix(name, "vtsql-")] = val return nil + } else if strings.HasPrefix(name, "vtctld-") { + if cfg.VtctldFlags == nil { + cfg.VtctldFlags = map[string]string{} + } + + cfg.VtctldFlags[strings.TrimPrefix(name, "vtctld-")] = val } match := discoveryFlagRegexp.FindStringSubmatch(name) From b00527953c32ff9c04e8974bd7209deba5a0a13b Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 18:47:37 -0500 Subject: [PATCH 08/17] Add testutil for grpcvtctldserver, to use both here, grpcvtctldclient, and vtadmin Signed-off-by: Andrew Mason --- go/vt/vtctl/grpcvtctldclient/client_test.go | 49 ++----------- go/vt/vtctl/grpcvtctldserver/server_test.go | 16 ++--- go/vt/vtctl/grpcvtctldserver/testutil/util.go | 68 +++++++++++++++++++ 3 files changed, 79 insertions(+), 54 deletions(-) create mode 100644 go/vt/vtctl/grpcvtctldserver/testutil/util.go diff --git a/go/vt/vtctl/grpcvtctldclient/client_test.go b/go/vt/vtctl/grpcvtctldclient/client_test.go index b4a34be38b2..77025fa3887 100644 --- a/go/vt/vtctl/grpcvtctldclient/client_test.go +++ b/go/vt/vtctl/grpcvtctldclient/client_test.go @@ -22,62 +22,27 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/net/nettest" - "google.golang.org/grpc" - - "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" "vitess.io/vitess/go/vt/vtctl/vtctldclient" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/proto/vtctldata" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" - vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" ) -// annoyingly, this is duplicated with theu tests in package grpcvtctldserver. -// fine for now, I suppose. -func addKeyspace(ctx context.Context, t *testing.T, ts *topo.Server, ks *vtctldatapb.Keyspace) { - in := *ks.Keyspace // take a copy to avoid the XXX_ fields changing - - err := ts.CreateKeyspace(ctx, ks.Name, &in) - require.NoError(t, err) -} - -func withTestServer( - t *testing.T, - server vtctlservicepb.VtctldServer, - test func(t *testing.T, client vtctldclient.VtctldClient), -) { - lis, err := nettest.NewLocalListener("tcp") - require.NoError(t, err, "cannot create nettest listener") - - defer lis.Close() - - s := grpc.NewServer() - vtctlservicepb.RegisterVtctldServer(s, server) - - go s.Serve(lis) - defer s.Stop() - - client, err := vtctldclient.New("grpc", lis.Addr().String()) - require.NoError(t, err, "cannot create vtctld client") - - test(t, client) -} - func TestFindAllShardsInKeyspace(t *testing.T) { ctx := context.Background() ts := memorytopo.NewServer("cell1") vtctld := grpcvtctldserver.NewVtctldServer(ts) - withTestServer(t, vtctld, func(t *testing.T, client vtctldclient.VtctldClient) { + testutil.WithTestServer(t, vtctld, func(t *testing.T, client vtctldclient.VtctldClient) { ks := &vtctldatapb.Keyspace{ Name: "testkeyspace", Keyspace: &topodatapb.Keyspace{}, } - addKeyspace(ctx, t, ts, ks) + testutil.AddKeyspace(ctx, t, ts, ks) si1, err := ts.GetOrCreateShard(ctx, ks.Name, "-80") require.NoError(t, err) @@ -115,7 +80,7 @@ func TestGetKeyspace(t *testing.T) { ts := memorytopo.NewServer("cell1") vtctld := grpcvtctldserver.NewVtctldServer(ts) - withTestServer(t, vtctld, func(t *testing.T, client vtctldclient.VtctldClient) { + testutil.WithTestServer(t, vtctld, func(t *testing.T, client vtctldclient.VtctldClient) { expected := &vtctldatapb.GetKeyspaceResponse{ Keyspace: &vtctldata.Keyspace{ Name: "testkeyspace", @@ -124,7 +89,7 @@ func TestGetKeyspace(t *testing.T) { }, }, } - addKeyspace(ctx, t, ts, expected.Keyspace) + testutil.AddKeyspace(ctx, t, ts, expected.Keyspace) resp, err := client.GetKeyspace(ctx, &vtctldatapb.GetKeyspaceRequest{Keyspace: expected.Keyspace.Name}) assert.NoError(t, err) @@ -142,7 +107,7 @@ func TestGetKeyspaces(t *testing.T) { ts := memorytopo.NewServer("cell1") vtctld := grpcvtctldserver.NewVtctldServer(ts) - withTestServer(t, vtctld, func(t *testing.T, client vtctldclient.VtctldClient) { + testutil.WithTestServer(t, vtctld, func(t *testing.T, client vtctldclient.VtctldClient) { resp, err := client.GetKeyspaces(ctx, &vtctldatapb.GetKeyspacesRequest{}) assert.NoError(t, err) assert.Empty(t, resp.Keyspaces) @@ -151,7 +116,7 @@ func TestGetKeyspaces(t *testing.T) { Name: "testkeyspace", Keyspace: &topodatapb.Keyspace{}, } - addKeyspace(ctx, t, ts, expected) + testutil.AddKeyspace(ctx, t, ts, expected) resp, err = client.GetKeyspaces(ctx, &vtctldatapb.GetKeyspacesRequest{}) assert.NoError(t, err) diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index aa3ecf2b1b4..2070cbca148 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -23,9 +23,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" @@ -40,7 +39,7 @@ func TestFindAllShardsInKeyspace(t *testing.T) { Name: "testkeyspace", Keyspace: &topodatapb.Keyspace{}, } - addKeyspace(ctx, t, ts, ks) + testutil.AddKeyspace(ctx, t, ts, ks) si1, err := ts.GetOrCreateShard(ctx, ks.Name, "-80") require.NoError(t, err) @@ -83,7 +82,7 @@ func TestGetKeyspace(t *testing.T) { }, }, } - addKeyspace(ctx, t, ts, expected.Keyspace) + testutil.AddKeyspace(ctx, t, ts, expected.Keyspace) ks, err := vtctld.GetKeyspace(ctx, &vtctldatapb.GetKeyspaceRequest{Keyspace: expected.Keyspace.Name}) assert.NoError(t, err) @@ -93,13 +92,6 @@ func TestGetKeyspace(t *testing.T) { assert.Error(t, err) } -func addKeyspace(ctx context.Context, t *testing.T, ts *topo.Server, ks *vtctldatapb.Keyspace) { - in := *ks.Keyspace // take a copy to avoid the XXX_ fields changing - - err := ts.CreateKeyspace(ctx, ks.Name, &in) - require.NoError(t, err) -} - func TestGetKeyspaces(t *testing.T) { ctx := context.Background() ts, topofactory := memorytopo.NewServerAndFactory("cell1") @@ -130,7 +122,7 @@ func TestGetKeyspaces(t *testing.T) { }, } for _, ks := range expected { - addKeyspace(ctx, t, ts, ks) + testutil.AddKeyspace(ctx, t, ts, ks) } resp, err = vtctld.GetKeyspaces(ctx, &vtctldatapb.GetKeyspacesRequest{}) diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/util.go b/go/vt/vtctl/grpcvtctldserver/testutil/util.go new file mode 100644 index 00000000000..ef98509da81 --- /dev/null +++ b/go/vt/vtctl/grpcvtctldserver/testutil/util.go @@ -0,0 +1,68 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testutil contains utility functions for writing tests for the +// grpcvtctldserver. +package testutil + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/net/nettest" + "google.golang.org/grpc" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/vtctl/vtctldclient" + + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" + vtctlservicepb "vitess.io/vitess/go/vt/proto/vtctlservice" +) + +// WithTestServer creates a gRPC server listening locally with the given RPC +// implementation, then runs the test func with a client created to point at +// that server. +func WithTestServer( + t *testing.T, + server vtctlservicepb.VtctldServer, + test func(t *testing.T, client vtctldclient.VtctldClient), +) { + lis, err := nettest.NewLocalListener("tcp") + require.NoError(t, err, "cannot create local listener") + + defer lis.Close() + + s := grpc.NewServer() + vtctlservicepb.RegisterVtctldServer(s, server) + + go s.Serve(lis) + defer s.Stop() + + client, err := vtctldclient.New("grpc", lis.Addr().String()) + require.NoError(t, err, "cannot create vtctld client") + + test(t, client) +} + +// AddKeyspace adds a keyspace to a topology, failing a test if that keyspace +// could not be added. It shallow copies the proto struct to prevent XXX_ fields +// from changing in the marshalling. +func AddKeyspace(ctx context.Context, t *testing.T, ts *topo.Server, ks *vtctldatapb.Keyspace) { + in := *ks.Keyspace // take a copy to avoid XXX_ fields changing. + + err := ts.CreateKeyspace(ctx, ks.Name, &in) + require.NoError(t, err) +} From 2d3d6f86728240edf3fc033586389a1527621f71 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 18:55:55 -0500 Subject: [PATCH 09/17] Adapt buildCluster to also inject a vtctld client Signed-off-by: Andrew Mason --- go/vt/vtadmin/api_test.go | 26 ++++++++++++++----- .../discovery/discovery_static_file.go | 15 +++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/go/vt/vtadmin/api_test.go b/go/vt/vtadmin/api_test.go index 990f8a01954..e7a2cd4aaf9 100644 --- a/go/vt/vtadmin/api_test.go +++ b/go/vt/vtadmin/api_test.go @@ -23,14 +23,17 @@ import ( "testing" "github.com/stretchr/testify/assert" - + "google.golang.org/grpc" + "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/vitessdriver" "vitess.io/vitess/go/vt/vtadmin/cluster" "vitess.io/vitess/go/vt/vtadmin/cluster/discovery/fakediscovery" "vitess.io/vitess/go/vt/vtadmin/grpcserver" "vitess.io/vitess/go/vt/vtadmin/http" + vtadminvtctldclient "vitess.io/vitess/go/vt/vtadmin/vtctldclient" "vitess.io/vitess/go/vt/vtadmin/vtsql" "vitess.io/vitess/go/vt/vtadmin/vtsql/fakevtsql" + "vitess.io/vitess/go/vt/vtctl/vtctldclient" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" @@ -250,7 +253,7 @@ func TestGetTablets(t *testing.T) { clusters := make([]*cluster.Cluster, len(tt.clusterTablets)) for i, tablets := range tt.clusterTablets { - cluster := buildCluster(i, tablets, tt.dbconfigs) + cluster := buildCluster(i, nil, tablets, tt.dbconfigs) clusters[i] = cluster } @@ -511,7 +514,7 @@ func TestGetTablet(t *testing.T) { clusters := make([]*cluster.Cluster, len(tt.clusterTablets)) for i, tablets := range tt.clusterTablets { - cluster := buildCluster(i, tablets, tt.dbconfigs) + cluster := buildCluster(i, nil, tablets, tt.dbconfigs) clusters[i] = cluster } @@ -532,10 +535,10 @@ type dbcfg struct { shouldErr bool } -// shared helper for building a cluster that contains the given tablets. -// dbconfigs contains an optional config for controlling the behavior of the -// cluster's DB at the package sql level. -func buildCluster(i int, tablets []*vtadminpb.Tablet, dbconfigs map[string]*dbcfg) *cluster.Cluster { +// shared helper for building a cluster that contains the given tablets and +// talking to the given vtctld server. dbconfigs contains an optional config +// for controlling the behavior of the cluster's DB at the package sql level. +func buildCluster(i int, vtctldClient vtctldclient.VtctldClient, tablets []*vtadminpb.Tablet, dbconfigs map[string]*dbcfg) *cluster.Cluster { disco := fakediscovery.New() disco.AddTaggedGates(nil, &vtadminpb.VTGate{Hostname: fmt.Sprintf("cluster%d-gate", i)}) @@ -558,7 +561,16 @@ func buildCluster(i int, tablets []*vtadminpb.Tablet, dbconfigs map[string]*dbcf return sql.OpenDB(&fakevtsql.Connector{Tablets: tablets, ShouldErr: dbconfig.shouldErr}), nil } + vtctld := vtadminvtctldclient.New(&vtadminvtctldclient.Config{ + Cluster: cluster.ToProto(), + Discovery: disco, + }) + vtctld.DialFunc = func(addr string, ff grpcclient.FailFast, opts ...grpc.DialOption) (vtctldclient.VtctldClient, error) { + return vtctldClient, nil + } + cluster.DB = db + cluster.Vtctld = vtctld return cluster } diff --git a/go/vt/vtadmin/cluster/discovery/discovery_static_file.go b/go/vt/vtadmin/cluster/discovery/discovery_static_file.go index fdb94e65f21..992e0f14983 100644 --- a/go/vt/vtadmin/cluster/discovery/discovery_static_file.go +++ b/go/vt/vtadmin/cluster/discovery/discovery_static_file.go @@ -175,3 +175,18 @@ func (d *StaticFileDiscovery) DiscoverVTGates(ctx context.Context, tags []string return results, nil } + +// DiscoverVtctld is part of the Discovery interface. +func (d *StaticFileDiscovery) DiscoverVtctld(ctx context.Context, tags []string) (*vtadminpb.Vtctld, error) { + panic("unimplemented") +} + +// DiscoverVtctldAddr is part of the Discovery interface. +func (d *StaticFileDiscovery) DiscoverVtctldAddr(ctx context.Context, tags []string) (string, error) { + panic("unimplemented") +} + +// DiscoverVtctlds is part of the Discovery interface. +func (d *StaticFileDiscovery) DiscoverVtctlds(ctx context.Context, tags []string) ([]*vtadminpb.Vtctld, error) { + panic("unimplemented") +} From 5091f7c9419787f3c6f0d4998495015cedd91f94 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 19:27:12 -0500 Subject: [PATCH 10/17] Add GetKeyspaces implementation and tests Signed-off-by: Andrew Mason --- go/vt/vtadmin/api.go | 54 +++++++++++ go/vt/vtadmin/api_test.go | 92 +++++++++++++++++++ .../discovery/fakediscovery/discovery.go | 16 +++- 3 files changed, 161 insertions(+), 1 deletion(-) diff --git a/go/vt/vtadmin/api.go b/go/vt/vtadmin/api.go index 012db642970..804dcc43288 100644 --- a/go/vt/vtadmin/api.go +++ b/go/vt/vtadmin/api.go @@ -34,6 +34,7 @@ import ( "vitess.io/vitess/go/vt/vterrors" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) @@ -154,6 +155,59 @@ func (api *API) GetGates(ctx context.Context, req *vtadminpb.GetGatesRequest) (* }, nil } +// GetKeyspaces is part of the vtadminpb.VTAdminServer interface. +func (api *API) GetKeyspaces(ctx context.Context, req *vtadminpb.GetKeyspacesRequest) (*vtadminpb.GetKeyspacesResponse, error) { + span, ctx := trace.NewSpan(ctx, "API.GetKeyspaces") + defer span.Finish() + + clusters, _ := api.getClustersForRequest(req.ClusterIds) + + var ( + keyspaces []*vtadminpb.Keyspace + wg sync.WaitGroup + er concurrency.AllErrorRecorder + m sync.Mutex + ) + + for _, c := range clusters { + wg.Add(1) + + go func(c *cluster.Cluster) { + defer wg.Done() + + if err := c.Vtctld.Dial(ctx); err != nil { + er.RecordError(err) + return + } + + resp, err := c.Vtctld.GetKeyspaces(ctx, &vtctldatapb.GetKeyspacesRequest{}) + if err != nil { + er.RecordError(err) + return + } + + m.Lock() + for _, ks := range resp.Keyspaces { + keyspaces = append(keyspaces, &vtadminpb.Keyspace{ + Cluster: c.ToProto(), + Keyspace: ks, + }) + } + m.Unlock() + }(c) + } + + wg.Wait() + + if er.HasErrors() { + return nil, er.Error() + } + + return &vtadminpb.GetKeyspacesResponse{ + Keyspaces: keyspaces, + }, nil +} + // GetTablet is part of the vtadminpb.VTAdminServer interface. func (api *API) GetTablet(ctx context.Context, req *vtadminpb.GetTabletRequest) (*vtadminpb.Tablet, error) { span, ctx := trace.NewSpan(ctx, "API.GetTablet") diff --git a/go/vt/vtadmin/api_test.go b/go/vt/vtadmin/api_test.go index e7a2cd4aaf9..58760923ce5 100644 --- a/go/vt/vtadmin/api_test.go +++ b/go/vt/vtadmin/api_test.go @@ -23,8 +23,10 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/grpc" "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vitessdriver" "vitess.io/vitess/go/vt/vtadmin/cluster" "vitess.io/vitess/go/vt/vtadmin/cluster/discovery/fakediscovery" @@ -33,10 +35,14 @@ import ( vtadminvtctldclient "vitess.io/vitess/go/vt/vtadmin/vtctldclient" "vitess.io/vitess/go/vt/vtadmin/vtsql" "vitess.io/vitess/go/vt/vtadmin/vtsql/fakevtsql" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" "vitess.io/vitess/go/vt/vtctl/vtctldclient" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" + "vitess.io/vitess/go/vt/proto/vttime" ) func TestGetGates(t *testing.T) { @@ -92,6 +98,91 @@ func TestGetGates(t *testing.T) { assert.Nil(t, resp) } +func TestGetKeyspaces(t *testing.T) { + ts1 := memorytopo.NewServer("c1_cell1") + ts2 := memorytopo.NewServer("c2_cell1") + + testutil.AddKeyspace(context.Background(), t, ts1, &vtctldatapb.Keyspace{ + Name: "testkeyspace", + Keyspace: &topodatapb.Keyspace{}, + }) + testutil.AddKeyspace(context.Background(), t, ts1, &vtctldatapb.Keyspace{ + Name: "snapshot", + Keyspace: &topodatapb.Keyspace{ + KeyspaceType: topodatapb.KeyspaceType_SNAPSHOT, + BaseKeyspace: "testkeyspace", + SnapshotTime: &vttime.Time{Seconds: 10, Nanoseconds: 1}, + }, + }) + + testutil.AddKeyspace(context.Background(), t, ts2, &vtctldatapb.Keyspace{ + Name: "customer", + Keyspace: &topodatapb.Keyspace{}, + }) + + testutil.WithTestServer(t, grpcvtctldserver.NewVtctldServer(ts1), func(t *testing.T, cluster1Client vtctldclient.VtctldClient) { + testutil.WithTestServer(t, grpcvtctldserver.NewVtctldServer(ts2), func(t *testing.T, cluster2Client vtctldclient.VtctldClient) { + c1 := buildCluster(1, cluster1Client, nil, nil) + c2 := buildCluster(2, cluster2Client, nil, nil) + + api := NewAPI([]*cluster.Cluster{c1, c2}, grpcserver.Options{}, http.Options{}) + resp, err := api.GetKeyspaces(context.Background(), &vtadminpb.GetKeyspacesRequest{}) + require.NoError(t, err) + + expected := &vtadminpb.GetKeyspacesResponse{ + Keyspaces: []*vtadminpb.Keyspace{ + { + Cluster: &vtadminpb.Cluster{ + Id: "c1", + Name: "cluster1", + }, + Keyspace: &vtctldatapb.Keyspace{ + Name: "testkeyspace", + Keyspace: &topodatapb.Keyspace{}, + }, + }, + { + Cluster: &vtadminpb.Cluster{ + Id: "c1", + Name: "cluster1", + }, + Keyspace: &vtctldatapb.Keyspace{ + Name: "snapshot", + Keyspace: &topodatapb.Keyspace{ + KeyspaceType: topodatapb.KeyspaceType_SNAPSHOT, + BaseKeyspace: "testkeyspace", + SnapshotTime: &vttime.Time{Seconds: 10, Nanoseconds: 1}, + }, + }, + }, + { + Cluster: &vtadminpb.Cluster{ + Id: "c2", + Name: "cluster2", + }, + Keyspace: &vtctldatapb.Keyspace{ + Name: "customer", + Keyspace: &topodatapb.Keyspace{}, + }, + }, + }, + } + assert.ElementsMatch(t, expected.Keyspaces, resp.Keyspaces) + + resp, err = api.GetKeyspaces( + context.Background(), + &vtadminpb.GetKeyspacesRequest{ + ClusterIds: []string{"c1"}, + }, + ) + require.NoError(t, err) + + expected.Keyspaces = expected.Keyspaces[:2] // just c1 + assert.ElementsMatch(t, expected.Keyspaces, resp.Keyspaces) + }) + }) +} + func TestGetTablets(t *testing.T) { tests := []struct { name string @@ -541,6 +632,7 @@ type dbcfg struct { func buildCluster(i int, vtctldClient vtctldclient.VtctldClient, tablets []*vtadminpb.Tablet, dbconfigs map[string]*dbcfg) *cluster.Cluster { disco := fakediscovery.New() disco.AddTaggedGates(nil, &vtadminpb.VTGate{Hostname: fmt.Sprintf("cluster%d-gate", i)}) + disco.AddTaggedVtctlds(nil, &vtadminpb.Vtctld{Hostname: "doesn't matter"}) cluster := &cluster.Cluster{ ID: fmt.Sprintf("c%d", i), diff --git a/go/vt/vtadmin/cluster/discovery/fakediscovery/discovery.go b/go/vt/vtadmin/cluster/discovery/fakediscovery/discovery.go index 8ff0030556b..87f7dd5fdda 100644 --- a/go/vt/vtadmin/cluster/discovery/fakediscovery/discovery.go +++ b/go/vt/vtadmin/cluster/discovery/fakediscovery/discovery.go @@ -62,7 +62,7 @@ func New() *Fake { // AddTaggedGates adds the given gates to the discovery fake, associating each // gate with each tag. To tag different gates with multiple tags, call multiple -// times with the same gates but different tag slices. Gates an uniquely +// times with the same gates but different tag slices. Gates are uniquely // identified by hostname. func (d *Fake) AddTaggedGates(tags []string, gates ...*vtadminpb.VTGate) { for _, tag := range tags { @@ -74,6 +74,20 @@ func (d *Fake) AddTaggedGates(tags []string, gates ...*vtadminpb.VTGate) { } } +// AddTaggedVtctlds adds the given vtctlds to the discovery fake, associating +// each vtctld with each tag. To tag different vtctlds with multiple tags, call +// multiple times with the same vtctlds but different tag slices. Vtctlds are +// uniquely identified by hostname. +func (d *Fake) AddTaggedVtctlds(tags []string, vtctlds ...*vtadminpb.Vtctld) { + for _, tag := range tags { + d.vtctlds.byTag[tag] = append(d.vtctlds.byTag[tag], vtctlds...) + } + + for _, vtctld := range vtctlds { + d.vtctlds.byName[vtctld.Hostname] = vtctld + } +} + // SetGatesError instructs whether the fake should return an error on gate // discovery functions. func (d *Fake) SetGatesError(shouldErr bool) { From e4e37a9fecd22fe12dd09e0df558fbaa74145b59 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 19:45:09 -0500 Subject: [PATCH 11/17] Implement vtctld discovery for staticfile Signed-off-by: Andrew Mason --- .../discovery/discovery_static_file.go | 81 ++++++++++++++++++- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/go/vt/vtadmin/cluster/discovery/discovery_static_file.go b/go/vt/vtadmin/cluster/discovery/discovery_static_file.go index 992e0f14983..73379a3da01 100644 --- a/go/vt/vtadmin/cluster/discovery/discovery_static_file.go +++ b/go/vt/vtadmin/cluster/discovery/discovery_static_file.go @@ -52,11 +52,16 @@ type StaticFileDiscovery struct { byName map[string]*vtadminpb.VTGate byTag map[string][]*vtadminpb.VTGate } + vtctlds struct { + byName map[string]*vtadminpb.Vtctld + byTag map[string][]*vtadminpb.Vtctld + } } // StaticFileClusterConfig configures Vitess components for a single cluster. type StaticFileClusterConfig struct { VTGates []*StaticFileVTGateConfig `json:"vtgates,omitempty"` + Vtctlds []*StaticFileVtctldConfig `json:"vtctlds,omitempty"` } // StaticFileVTGateConfig contains host and tag information for a single VTGate in a cluster. @@ -65,6 +70,13 @@ type StaticFileVTGateConfig struct { Tags []string `json:"tags"` } +// StaticFileVtctldConfig contains a host and tag information for a single +// Vtctld in a cluster. +type StaticFileVtctldConfig struct { + Host *vtadminpb.Vtctld `json:"host"` + Tags []string `json:"tags"` +} + // NewStaticFile returns a StaticFileDiscovery for the given cluster. func NewStaticFile(cluster *vtadminpb.Cluster, flags *pflag.FlagSet, args []string) (Discovery, error) { disco := &StaticFileDiscovery{ @@ -108,6 +120,19 @@ func (d *StaticFileDiscovery) parseConfig(bytes []byte) error { d.gates.byTag[tag] = append(d.gates.byTag[tag], gate.Host) } } + + d.vtctlds.byName = make(map[string]*vtadminpb.Vtctld, len(d.config.Vtctlds)) + d.vtctlds.byTag = make(map[string][]*vtadminpb.Vtctld) + + // Index the vtctlds by name and by tag for easier lookups + for _, vtctld := range d.config.Vtctlds { + d.vtctlds.byName[vtctld.Host.Hostname] = vtctld.Host + + for _, tag := range vtctld.Tags { + d.vtctlds.byTag[tag] = append(d.vtctlds.byTag[tag], vtctld.Host) + } + } + return nil } @@ -178,15 +203,65 @@ func (d *StaticFileDiscovery) DiscoverVTGates(ctx context.Context, tags []string // DiscoverVtctld is part of the Discovery interface. func (d *StaticFileDiscovery) DiscoverVtctld(ctx context.Context, tags []string) (*vtadminpb.Vtctld, error) { - panic("unimplemented") + vtctlds, err := d.DiscoverVtctlds(ctx, tags) + if err != nil { + return nil, err + } + + count := len(vtctlds) + if count == 0 { + return nil, ErrNoVTGates + } + + vtctld := vtctlds[rand.Intn(len(vtctlds))] + return vtctld, nil } // DiscoverVtctldAddr is part of the Discovery interface. func (d *StaticFileDiscovery) DiscoverVtctldAddr(ctx context.Context, tags []string) (string, error) { - panic("unimplemented") + vtctld, err := d.DiscoverVtctld(ctx, tags) + if err != nil { + return "", err + } + + return vtctld.Hostname, nil } // DiscoverVtctlds is part of the Discovery interface. func (d *StaticFileDiscovery) DiscoverVtctlds(ctx context.Context, tags []string) ([]*vtadminpb.Vtctld, error) { - panic("unimplemented") + if len(tags) == 0 { + results := []*vtadminpb.Vtctld{} + for _, v := range d.vtctlds.byName { + results = append(results, v) + } + + return results, nil + } + + set := d.vtctlds.byName + + for _, tag := range tags { + intermediate := map[string]*vtadminpb.Vtctld{} + + vtctlds, ok := d.vtctlds.byTag[tag] + if !ok { + return []*vtadminpb.Vtctld{}, nil + } + + for _, v := range vtctlds { + if _, ok := set[v.Hostname]; ok { + intermediate[v.Hostname] = v + } + } + + set = intermediate + } + + results := make([]*vtadminpb.Vtctld, 0, len(set)) + + for _, vtctld := range set { + results = append(results, vtctld) + } + + return results, nil } From dc6ebc8ad05308877d6b5272d693459e8e7ab9b9 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 19:49:49 -0500 Subject: [PATCH 12/17] Add staticfile tests for vtctld discovery Signed-off-by: Andrew Mason --- .../discovery/discovery_static_file_test.go | 228 +++++++++++++++++- 1 file changed, 225 insertions(+), 3 deletions(-) diff --git a/go/vt/vtadmin/cluster/discovery/discovery_static_file_test.go b/go/vt/vtadmin/cluster/discovery/discovery_static_file_test.go index 934f8ea3da8..52cf970b3f3 100644 --- a/go/vt/vtadmin/cluster/discovery/discovery_static_file_test.go +++ b/go/vt/vtadmin/cluster/discovery/discovery_static_file_test.go @@ -97,7 +97,7 @@ func TestDiscoverVTGate(t *testing.T) { gate, err := disco.DiscoverVTGate(context.Background(), tt.tags) if tt.shouldErr { - assert.Error(t, err, assert.AnError) + assert.Error(t, err) return } @@ -232,14 +232,14 @@ func TestDiscoverVTGates(t *testing.T) { err := disco.parseConfig(tt.contents) if tt.shouldErrConfig { - assert.Error(t, err, assert.AnError) + assert.Error(t, err) } else { require.NoError(t, err) } gates, err := disco.DiscoverVTGates(context.Background(), tt.tags) if tt.shouldErr { - assert.Error(t, err, assert.AnError) + assert.Error(t, err) return } @@ -248,3 +248,225 @@ func TestDiscoverVTGates(t *testing.T) { }) } } + +func TestDiscoverVtctld(t *testing.T) { + tests := []struct { + name string + contents []byte + expected *vtadminpb.Vtctld + tags []string + shouldErr bool + }{ + { + name: "empty config", + contents: []byte(`{}`), + expected: nil, + shouldErr: true, + }, + { + name: "one vtctld", + contents: []byte(` + { + "vtctlds": [{ + "host": { + "hostname": "127.0.0.1:12345" + } + }] + } + `), + expected: &vtadmin.Vtctld{ + Hostname: "127.0.0.1:12345", + }, + }, + { + name: "filtered by tags (one match)", + contents: []byte(` + { + "vtctlds": [ + { + "host": { + "hostname": "127.0.0.1:11111" + }, + "tags": ["cell:cellA"] + }, + { + "host": { + "hostname": "127.0.0.1:22222" + }, + "tags": ["cell:cellB"] + }, + { + "host": { + "hostname": "127.0.0.1:33333" + }, + "tags": ["cell:cellA"] + } + ] + } + `), + expected: &vtadminpb.Vtctld{ + Hostname: "127.0.0.1:22222", + }, + tags: []string{"cell:cellB"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + disco := &StaticFileDiscovery{} + err := disco.parseConfig(tt.contents) + require.NoError(t, err) + + vtctld, err := disco.DiscoverVtctld(context.Background(), tt.tags) + if tt.shouldErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expected, vtctld) + }) + } +} + +func TestDiscoverVtctlds(t *testing.T) { + tests := []struct { + name string + contents []byte + tags []string + expected []*vtadminpb.Vtctld + // True if the test should produce an error on the DiscoverVTGates call + shouldErr bool + // True if the test should produce an error on the disco.parseConfig step + shouldErrConfig bool + }{ + { + name: "empty config", + contents: []byte(`{}`), + expected: []*vtadminpb.Vtctld{}, + shouldErr: false, + }, + { + name: "no tags", + contents: []byte(` + { + "vtctlds": [ + { + "host": { + "hostname": "127.0.0.1:12345" + } + }, + { + "host": { + "hostname": "127.0.0.1:67890" + } + } + ] + } + `), + expected: []*vtadminpb.Vtctld{ + {Hostname: "127.0.0.1:12345"}, + {Hostname: "127.0.0.1:67890"}, + }, + shouldErr: false, + }, + { + name: "filtered by tags", + contents: []byte(` + { + "vtctlds": [ + { + "host": { + "hostname": "127.0.0.1:11111" + }, + "tags": ["cell:cellA"] + }, + { + "host": { + "hostname": "127.0.0.1:22222" + }, + "tags": ["cell:cellB"] + }, + { + "host": { + "hostname": "127.0.0.1:33333" + }, + "tags": ["cell:cellA"] + } + ] + } + `), + tags: []string{"cell:cellA"}, + expected: []*vtadminpb.Vtctld{ + {Hostname: "127.0.0.1:11111"}, + {Hostname: "127.0.0.1:33333"}, + }, + shouldErr: false, + }, + { + name: "filtered by multiple tags", + contents: []byte(` + { + "vtctlds": [ + { + "host": { + "hostname": "127.0.0.1:11111" + }, + "tags": ["cell:cellA"] + }, + { + "host": { + "hostname": "127.0.0.1:22222" + }, + "tags": ["cell:cellA", "pool:poolZ"] + }, + { + "host": { + "hostname": "127.0.0.1:33333" + }, + "tags": ["pool:poolZ"] + } + ] + } + `), + tags: []string{"cell:cellA", "pool:poolZ"}, + expected: []*vtadminpb.Vtctld{ + {Hostname: "127.0.0.1:22222"}, + }, + shouldErr: false, + }, + { + name: "invalid json", + contents: []byte(` + { + "vtctlds": "malformed" + } + `), + tags: []string{}, + shouldErr: false, + shouldErrConfig: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + disco := &StaticFileDiscovery{} + + err := disco.parseConfig(tt.contents) + if tt.shouldErrConfig { + assert.Error(t, err) + } else { + require.NoError(t, err) + } + + vtctlds, err := disco.DiscoverVtctlds(context.Background(), tt.tags) + if tt.shouldErr { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expected, vtctlds) + }) + } +} From 2a032aa7aad8665010bc42b96db621571284da82 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 7 Jan 2021 20:56:55 -0500 Subject: [PATCH 13/17] Fix lints Signed-off-by: Andrew Mason --- go/vt/vtadmin/api_test.go | 1 + go/vt/vtadmin/vtctldclient/config.go | 1 + go/vt/vtadmin/vtctldclient/config_test.go | 1 + go/vt/vtadmin/vtctldclient/proxy.go | 1 + go/vt/vtctl/grpcvtctldclient/client_test.go | 1 + go/vt/vtctl/grpcvtctldserver/server_test.go | 1 + go/vt/vtctl/grpcvtctldserver/testutil/util.go | 1 + 7 files changed, 7 insertions(+) diff --git a/go/vt/vtadmin/api_test.go b/go/vt/vtadmin/api_test.go index 58760923ce5..fcd310b21b9 100644 --- a/go/vt/vtadmin/api_test.go +++ b/go/vt/vtadmin/api_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vitessdriver" diff --git a/go/vt/vtadmin/vtctldclient/config.go b/go/vt/vtadmin/vtctldclient/config.go index 8a4eec7f9d9..2ff3ad1d34f 100644 --- a/go/vt/vtadmin/vtctldclient/config.go +++ b/go/vt/vtadmin/vtctldclient/config.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/spf13/pflag" + "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/vtadmin/cluster/discovery" "vitess.io/vitess/go/vt/vtadmin/credentials" diff --git a/go/vt/vtadmin/vtctldclient/config_test.go b/go/vt/vtadmin/vtctldclient/config_test.go index 987838d3f77..a9979420d4e 100644 --- a/go/vt/vtadmin/vtctldclient/config_test.go +++ b/go/vt/vtadmin/vtctldclient/config_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/grpcclient" vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" diff --git a/go/vt/vtadmin/vtctldclient/proxy.go b/go/vt/vtadmin/vtctldclient/proxy.go index dc7913c8083..a5ae63e78f0 100644 --- a/go/vt/vtadmin/vtctldclient/proxy.go +++ b/go/vt/vtadmin/vtctldclient/proxy.go @@ -20,6 +20,7 @@ import ( "context" "google.golang.org/grpc" + "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/vtadmin/cluster/discovery" "vitess.io/vitess/go/vt/vtctl/grpcvtctldclient" diff --git a/go/vt/vtctl/grpcvtctldclient/client_test.go b/go/vt/vtctl/grpcvtctldclient/client_test.go index 77025fa3887..d365b376b34 100644 --- a/go/vt/vtctl/grpcvtctldclient/client_test.go +++ b/go/vt/vtctl/grpcvtctldclient/client_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index 2070cbca148..eb16e7538b2 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/util.go b/go/vt/vtctl/grpcvtctldserver/testutil/util.go index ef98509da81..34400b35d58 100644 --- a/go/vt/vtctl/grpcvtctldserver/testutil/util.go +++ b/go/vt/vtctl/grpcvtctldserver/testutil/util.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/net/nettest" "google.golang.org/grpc" + "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vtctl/vtctldclient" From 9b926a3d5f5f685a807ef56754b47230fe166a2b Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 8 Jan 2021 08:28:30 -0500 Subject: [PATCH 14/17] Add HTTP route Signed-off-by: Andrew Mason --- go/vt/vtadmin/api.go | 1 + go/vt/vtadmin/http/keyspaces.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 go/vt/vtadmin/http/keyspaces.go diff --git a/go/vt/vtadmin/api.go b/go/vt/vtadmin/api.go index 804dcc43288..aef57176ea9 100644 --- a/go/vt/vtadmin/api.go +++ b/go/vt/vtadmin/api.go @@ -78,6 +78,7 @@ func NewAPI(clusters []*cluster.Cluster, opts grpcserver.Options, httpOpts vtadm httpAPI := vtadminhttp.NewAPI(api) router.HandleFunc("/gates", httpAPI.Adapt(vtadminhttp.GetGates)).Name("API.GetGates") + router.HandleFunc("/keyspaces", httpAPI.Adapt(vtadminhttp.GetKeyspaces)).Name("API.GetKeyspaces") router.HandleFunc("/tablets", httpAPI.Adapt(vtadminhttp.GetTablets)).Name("API.GetTablets") router.HandleFunc("/tablet/{tablet}", httpAPI.Adapt(vtadminhttp.GetTablet)).Name("API.GetTablet") diff --git a/go/vt/vtadmin/http/keyspaces.go b/go/vt/vtadmin/http/keyspaces.go new file mode 100644 index 00000000000..a2a9f15c52a --- /dev/null +++ b/go/vt/vtadmin/http/keyspaces.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package http + +import ( + "context" + + vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin" +) + +// GetKeyspaces implements the http wrapper for /keyspaces[?cluster=[&cluster=]]. +func GetKeyspaces(ctx context.Context, r Request, api *API) *JSONResponse { + keyspaces, err := api.server.GetKeyspaces(ctx, &vtadminpb.GetKeyspacesRequest{ + ClusterIds: r.URL.Query()["cluster"], + }) + + return NewJSONResponse(keyspaces, err) +} From eea61144ee2354989a927ce0cd9c3faf25614575 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 8 Jan 2021 08:31:59 -0500 Subject: [PATCH 15/17] Need to pair WithInsecure with WithPerRPCCredentials Signed-off-by: Andrew Mason --- go/vt/vtadmin/vtctldclient/proxy.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go/vt/vtadmin/vtctldclient/proxy.go b/go/vt/vtadmin/vtctldclient/proxy.go index a5ae63e78f0..c90e698def5 100644 --- a/go/vt/vtadmin/vtctldclient/proxy.go +++ b/go/vt/vtadmin/vtctldclient/proxy.go @@ -103,7 +103,11 @@ func (vtctld *ClientProxy) Dial(ctx context.Context) error { opts := []grpc.DialOption{} if vtctld.creds != nil { - opts = append(opts, grpc.WithPerRPCCredentials(vtctld.creds)) + opts = append(opts, grpc.WithPerRPCCredentials(vtctld.creds), + // TODO: make configurable. right now, omitting this and attempting + // to not use TLS results in: + // grpc: no transport security set (use grpc.WithInsecure() explicitly or set credentials) + grpc.WithInsecure()) } client, err := vtctld.DialFunc(addr, grpcclient.FailFast(false), opts...) From 01ea3575583e6cee6499981860b3873568f2d3fa Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 13 Jan 2021 19:28:45 -0500 Subject: [PATCH 16/17] Fix mistyped documentation Signed-off-by: Andrew Mason --- go/vt/vtadmin/cluster/discovery/discovery.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/vt/vtadmin/cluster/discovery/discovery.go b/go/vt/vtadmin/cluster/discovery/discovery.go index 7e1a65a1180..8a2f9e50217 100644 --- a/go/vt/vtadmin/cluster/discovery/discovery.go +++ b/go/vt/vtadmin/cluster/discovery/discovery.go @@ -61,9 +61,9 @@ type Discovery interface { // further. Which vtctld in a set of found vtctlds is returned is not // specified by the interface, and can be implementation-specific. DiscoverVtctld(ctx context.Context, tags []string) (*vtadminpb.Vtctld, error) - // DiscoverVtctldAddr returns the address of a of vtctld found in the - // discovery service. Tags can optionally be used to filter the set of - // potential vtctld further. Which gate in a set of found vtctld is used to + // DiscoverVtctldAddr returns the address of a vtctld found in the discovery + // service. Tags can optionally be used to filter the set of potential + // vtctlds further. Which vtctld in a set of potential vtctld is used to // return an address is not specified by the interface, and can be // implementation-specific. DiscoverVtctldAddr(ctx context.Context, tags []string) (string, error) From 00fba1e064b79f209ef6a861e21320a1242323b3 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 15 Jan 2021 07:11:16 -0500 Subject: [PATCH 17/17] Fix error name from copy-paste :shamebell: Signed-off-by: Andrew Mason --- go/vt/vtadmin/cluster/discovery/discovery_static_file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtadmin/cluster/discovery/discovery_static_file.go b/go/vt/vtadmin/cluster/discovery/discovery_static_file.go index 73379a3da01..653b526e3cd 100644 --- a/go/vt/vtadmin/cluster/discovery/discovery_static_file.go +++ b/go/vt/vtadmin/cluster/discovery/discovery_static_file.go @@ -210,7 +210,7 @@ func (d *StaticFileDiscovery) DiscoverVtctld(ctx context.Context, tags []string) count := len(vtctlds) if count == 0 { - return nil, ErrNoVTGates + return nil, ErrNoVtctlds } vtctld := vtctlds[rand.Intn(len(vtctlds))]