From cffa4e6e3e69f9007142e8d5d1dd5c4496865541 Mon Sep 17 00:00:00 2001 From: Spencer Kimball Date: Wed, 28 Sep 2016 17:55:04 -0400 Subject: [PATCH] rpc: enforce max clock offset across nodes in a cluster Commits suicide if offset is different than any contacted node. In order to change the max clock offset, the cluster must be stopped and restarted. --- rpc/context.go | 5 ++- rpc/heartbeat.go | 7 +++++ rpc/heartbeat.pb.go | 71 +++++++++++++++++++++++++++++-------------- rpc/heartbeat.proto | 2 ++ rpc/heartbeat_test.go | 30 ++++++++++++++++++ 5 files changed, 92 insertions(+), 23 deletions(-) diff --git a/rpc/context.go b/rpc/context.go index 82066dffc5e6..5af5afe201e7 100644 --- a/rpc/context.go +++ b/rpc/context.go @@ -285,7 +285,10 @@ func (ctx *Context) IsConnHealthy(remoteAddr string) bool { } func (ctx *Context) runHeartbeat(cc *grpc.ClientConn, remoteAddr string) error { - request := PingRequest{Addr: ctx.Addr} + request := PingRequest{ + Addr: ctx.Addr, + MaxOffsetNanos: ctx.localClock.MaxOffset().Nanoseconds(), + } heartbeatClient := NewHeartbeatClient(cc) var heartbeatTimer timeutil.Timer diff --git a/rpc/heartbeat.go b/rpc/heartbeat.go index 03e1ad588bca..b69161b5c343 100644 --- a/rpc/heartbeat.go +++ b/rpc/heartbeat.go @@ -62,6 +62,13 @@ type HeartbeatService struct { // The requester should also estimate its offset from this server along // with the requester's address. func (hs *HeartbeatService) Ping(ctx context.Context, args *PingRequest) (*PingResponse, error) { + // Enforce that clock max offsets are identical between nodes. + // Commit suicide in the event that this is ever untrue. + // This check is ignored if either offset is set to 0 (for unittests). + mo, amo := hs.clock.MaxOffset(), time.Duration(args.MaxOffsetNanos) + if mo != 0 && amo != 0 && mo != amo { + panic(fmt.Sprintf("clock max offset mismatch node=%s vs %s=%s", mo, args.Addr, amo)) + } serverOffset := args.Offset // The server offset should be the opposite of the client offset. serverOffset.Offset = -serverOffset.Offset diff --git a/rpc/heartbeat.pb.go b/rpc/heartbeat.pb.go index a84880aeb7ff..4063dff55905 100644 --- a/rpc/heartbeat.pb.go +++ b/rpc/heartbeat.pb.go @@ -67,6 +67,8 @@ type PingRequest struct { Offset RemoteOffset `protobuf:"bytes,2,opt,name=offset" json:"offset"` // The address of the client. Addr string `protobuf:"bytes,3,opt,name=addr" json:"addr"` + // The configured maximum clock offset (in nanoseconds) on the server. + MaxOffsetNanos int64 `protobuf:"varint,4,opt,name=max_offset_nanos,json=maxOffsetNanos" json:"max_offset_nanos"` } func (m *PingRequest) Reset() { *m = PingRequest{} } @@ -222,6 +224,9 @@ func (m *PingRequest) MarshalTo(data []byte) (int, error) { i++ i = encodeVarintHeartbeat(data, i, uint64(len(m.Addr))) i += copy(data[i:], m.Addr) + data[i] = 0x20 + i++ + i = encodeVarintHeartbeat(data, i, uint64(m.MaxOffsetNanos)) return i, nil } @@ -295,6 +300,7 @@ func (m *PingRequest) Size() (n int) { n += 1 + l + sovHeartbeat(uint64(l)) l = len(m.Addr) n += 1 + l + sovHeartbeat(uint64(l)) + n += 1 + sovHeartbeat(uint64(m.MaxOffsetNanos)) return n } @@ -544,6 +550,25 @@ func (m *PingRequest) Unmarshal(data []byte) error { } m.Addr = string(data[iNdEx:postIndex]) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxOffsetNanos", wireType) + } + m.MaxOffsetNanos = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHeartbeat + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + m.MaxOffsetNanos |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipHeartbeat(data[iNdEx:]) @@ -771,26 +796,28 @@ var ( func init() { proto.RegisterFile("cockroach/rpc/heartbeat.proto", fileDescriptorHeartbeat) } var fileDescriptorHeartbeat = []byte{ - // 326 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x91, 0xb1, 0x4e, 0xf3, 0x30, - 0x14, 0x85, 0xe3, 0xbf, 0xd1, 0x2f, 0xf5, 0xa6, 0x2c, 0x16, 0x43, 0xd4, 0xd2, 0x14, 0x45, 0x02, - 0x31, 0xa5, 0x52, 0x37, 0xd8, 0xda, 0x89, 0x89, 0xa2, 0x8a, 0x89, 0xa5, 0x32, 0xee, 0x6d, 0x1a, - 0xa1, 0xc4, 0xc6, 0x76, 0x91, 0x90, 0x58, 0x78, 0x03, 0x46, 0x46, 0x1e, 0xa7, 0x23, 0x23, 0x13, - 0x82, 0xf0, 0x22, 0x28, 0x4e, 0x1b, 0xa5, 0x85, 0xed, 0xea, 0x7e, 0xc7, 0xf7, 0x9c, 0x23, 0x43, - 0x97, 0x0b, 0x7e, 0xab, 0x04, 0xe3, 0x8b, 0xbe, 0x92, 0xbc, 0xbf, 0x40, 0xa6, 0xcc, 0x0d, 0x32, - 0x13, 0x49, 0x25, 0x8c, 0xa0, 0x7b, 0x15, 0x8e, 0x94, 0xe4, 0xed, 0xfd, 0x58, 0xc4, 0xc2, 0x92, - 0x7e, 0x31, 0x95, 0xa2, 0xf0, 0x89, 0x40, 0x6b, 0x82, 0xa9, 0x30, 0x38, 0x9e, 0xcf, 0x35, 0x1a, - 0x7a, 0x00, 0xff, 0x85, 0x9d, 0x7c, 0x72, 0x48, 0x4e, 0x1a, 0x23, 0x77, 0xf5, 0xd1, 0x73, 0x26, - 0xeb, 0x1d, 0x3d, 0x06, 0x6f, 0x99, 0x71, 0x54, 0x86, 0x25, 0x99, 0x79, 0xf0, 0xff, 0xd5, 0x24, - 0x75, 0x40, 0x8f, 0xc0, 0x4b, 0x91, 0xe9, 0xa5, 0xc2, 0xd9, 0x94, 0x19, 0xbf, 0x51, 0xd3, 0xc1, - 0x06, 0x0c, 0xcd, 0x99, 0xfb, 0xf2, 0xda, 0x73, 0xc2, 0x47, 0xf0, 0x2e, 0x93, 0x2c, 0x9e, 0xe0, - 0xdd, 0x12, 0xb5, 0xa1, 0x3e, 0xb8, 0x32, 0xc9, 0x62, 0xeb, 0xdf, 0x5c, 0x3f, 0xb2, 0x1b, 0x7a, - 0x5a, 0x65, 0x2b, 0x8c, 0xbd, 0x41, 0x27, 0xda, 0xaa, 0x18, 0xd5, 0x8b, 0xec, 0x04, 0xf7, 0xc1, - 0x65, 0xb3, 0x99, 0xb2, 0x49, 0xaa, 0xa3, 0xc5, 0x26, 0x1c, 0x43, 0xab, 0x74, 0xd7, 0x52, 0x64, - 0x1a, 0xad, 0xbd, 0xf8, 0x65, 0x2f, 0xb2, 0xb8, 0x28, 0xa5, 0x51, 0xdd, 0xa3, 0x9a, 0x9a, 0x24, - 0xc5, 0xad, 0xf2, 0x50, 0x82, 0xab, 0x24, 0xc5, 0xc1, 0x05, 0x34, 0xcf, 0x37, 0x5f, 0x41, 0x87, - 0xe0, 0x16, 0xd7, 0x69, 0x7b, 0x27, 0x6a, 0xad, 0x70, 0xbb, 0xf3, 0x27, 0x2b, 0xe3, 0x84, 0xce, - 0xa8, 0xbb, 0xfa, 0x0a, 0x9c, 0x55, 0x1e, 0x90, 0xb7, 0x3c, 0x20, 0xef, 0x79, 0x40, 0x3e, 0xf3, - 0x80, 0x3c, 0x7f, 0x07, 0xce, 0x75, 0x43, 0x49, 0xfe, 0x13, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x04, - 0x3e, 0x16, 0x06, 0x02, 0x00, 0x00, + // 353 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x91, 0x31, 0x4e, 0xeb, 0x40, + 0x10, 0x86, 0xbd, 0x2f, 0xd6, 0x93, 0x32, 0xce, 0x7b, 0x42, 0x2b, 0x0a, 0x2b, 0x21, 0x0e, 0xb2, + 0x04, 0xa2, 0x72, 0xa4, 0x74, 0xd0, 0x25, 0x15, 0x55, 0x82, 0x22, 0x2a, 0x1a, 0x6b, 0xd9, 0x4c, + 0x1c, 0x0b, 0x79, 0xd7, 0xec, 0x6e, 0x50, 0x28, 0xb9, 0x01, 0x25, 0x25, 0x07, 0xe0, 0x20, 0x29, + 0x29, 0xa9, 0x10, 0x98, 0x8b, 0x20, 0xdb, 0x49, 0xe4, 0x04, 0xba, 0xd1, 0x7c, 0xbf, 0xc7, 0xdf, + 0xcc, 0x42, 0x9b, 0x4b, 0x7e, 0xa3, 0x24, 0xe3, 0xb3, 0xae, 0x4a, 0x79, 0x77, 0x86, 0x4c, 0x99, + 0x6b, 0x64, 0x26, 0x48, 0x95, 0x34, 0x92, 0xfe, 0xdb, 0xe0, 0x40, 0xa5, 0xbc, 0xb9, 0x1f, 0xc9, + 0x48, 0x16, 0xa4, 0x9b, 0x57, 0x65, 0xc8, 0x7f, 0x20, 0xd0, 0x18, 0x63, 0x22, 0x0d, 0x8e, 0xa6, + 0x53, 0x8d, 0x86, 0x1e, 0xc0, 0x5f, 0x59, 0x54, 0x2e, 0x39, 0x24, 0x27, 0xb5, 0x81, 0xbd, 0x7c, + 0xef, 0x58, 0xe3, 0x55, 0x8f, 0x1e, 0x83, 0x33, 0x17, 0x1c, 0x95, 0x61, 0xb1, 0x30, 0xf7, 0xee, + 0x9f, 0x4a, 0xa4, 0x0a, 0xe8, 0x11, 0x38, 0x09, 0x32, 0x3d, 0x57, 0x38, 0x09, 0x99, 0x71, 0x6b, + 0x95, 0x1c, 0xac, 0x41, 0xdf, 0x9c, 0xd9, 0x4f, 0xcf, 0x1d, 0xcb, 0x7f, 0x21, 0xe0, 0x5c, 0xc4, + 0x22, 0x1a, 0xe3, 0xed, 0x1c, 0xb5, 0xa1, 0x2e, 0xd8, 0x69, 0x2c, 0xa2, 0x42, 0xa0, 0xbe, 0xfa, + 0xaa, 0xe8, 0xd0, 0xd3, 0x8d, 0x5c, 0xfe, 0x67, 0xa7, 0xd7, 0x0a, 0xb6, 0x76, 0x0c, 0xaa, 0x9b, + 0xec, 0x98, 0xbb, 0x60, 0xb3, 0xc9, 0x44, 0x15, 0x2a, 0x9b, 0xa1, 0x79, 0x87, 0x06, 0xb0, 0x97, + 0xb0, 0x45, 0x58, 0xe6, 0x42, 0xc1, 0x84, 0xd4, 0xae, 0x5d, 0x11, 0xfe, 0x9f, 0xb0, 0x45, 0x39, + 0x72, 0x98, 0x33, 0x7f, 0x04, 0x8d, 0xd2, 0x56, 0xa7, 0x52, 0x68, 0x2c, 0x74, 0xe5, 0x0f, 0x5d, + 0x29, 0xa2, 0xfc, 0x0a, 0x1a, 0xd5, 0x1d, 0xaa, 0xd0, 0xc4, 0x09, 0x6e, 0x5d, 0x0b, 0x4a, 0x70, + 0x19, 0x27, 0xd8, 0x1b, 0x42, 0xfd, 0x7c, 0xfd, 0x76, 0xb4, 0x0f, 0x76, 0x3e, 0x9d, 0x36, 0x77, + 0x56, 0xab, 0x1c, 0xa8, 0xd9, 0xfa, 0x95, 0x95, 0x3a, 0xbe, 0x35, 0x68, 0x2f, 0x3f, 0x3d, 0x6b, + 0x99, 0x79, 0xe4, 0x35, 0xf3, 0xc8, 0x5b, 0xe6, 0x91, 0x8f, 0xcc, 0x23, 0x8f, 0x5f, 0x9e, 0x75, + 0x55, 0x53, 0x29, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x73, 0x29, 0x81, 0xa0, 0x37, 0x02, 0x00, + 0x00, } diff --git a/rpc/heartbeat.proto b/rpc/heartbeat.proto index eb54b4534d8e..759a0f9f8b11 100644 --- a/rpc/heartbeat.proto +++ b/rpc/heartbeat.proto @@ -49,6 +49,8 @@ message PingRequest { optional RemoteOffset offset = 2 [(gogoproto.nullable) = false]; // The address of the client. optional string addr = 3 [(gogoproto.nullable) = false]; + // The configured maximum clock offset (in nanoseconds) on the server. + optional int64 max_offset_nanos = 4 [(gogoproto.nullable) = false]; } // A PingResponse contains the echoed ping request string. diff --git a/rpc/heartbeat_test.go b/rpc/heartbeat_test.go index 2ad757723838..5c9c06407636 100644 --- a/rpc/heartbeat_test.go +++ b/rpc/heartbeat_test.go @@ -17,6 +17,8 @@ package rpc import ( + "fmt" + "regexp" "testing" "time" @@ -103,3 +105,31 @@ func TestManualHeartbeat(t *testing.T) { manualResponse.ServerTime, regularResponse.ServerTime) } } + +func TestClockOffsetMismatch(t *testing.T) { + defer leaktest.AfterTest(t)() + defer func() { + if r := recover(); r != nil { + fmt.Println(r) + if match, _ := regexp.MatchString("clock max offset mismatch", r.(string)); !match { + t.Errorf("expected clock mismatch error") + } + } + }() + + clock := hlc.NewClock(hlc.UnixNano) + clock.SetMaxOffset(250 * time.Millisecond) + hs := &HeartbeatService{ + clock: clock, + remoteClockMonitor: newRemoteClockMonitor(context.TODO(), clock, time.Hour), + } + + request := &PingRequest{ + Ping: "testManual", + Addr: "test", + MaxOffsetNanos: (500 * time.Millisecond).Nanoseconds(), + } + ctx := context.Background() + _, _ = hs.Ping(ctx, request) + t.Fatal("should not reach") +}