From 1f4f34e657c31586c682cd14a6bf5186977a3eff Mon Sep 17 00:00:00 2001 From: Manish R Jain Date: Fri, 15 Feb 2019 18:36:08 -0800 Subject: [PATCH 1/7] Code to delete predicates which don't belong. --- dgraph/cmd/zero/tablet.go | 13 ++++++++++ dgraph/cmd/zero/zero.go | 54 +++++++++++++++++++++++++++++++++++++++ worker/predicate_move.go | 6 +++++ 3 files changed, 73 insertions(+) diff --git a/dgraph/cmd/zero/tablet.go b/dgraph/cmd/zero/tablet.go index dfeca558604..4b84fe6da9f 100644 --- a/dgraph/cmd/zero/tablet.go +++ b/dgraph/cmd/zero/tablet.go @@ -78,6 +78,11 @@ func (s *Server) rebalanceTablets() { // for the entire duration of predicate move. If this Zero stops being the leader, the final // proposal of reassigning the tablet to the destination would fail automatically. func (s *Server) movePredicate(predicate string, srcGroup, dstGroup uint32) error { + s.moveOngoing <- struct{}{} + defer func() { + <-s.moveOngoing + }() + ctx, cancel := context.WithTimeout(context.Background(), predicateMoveTimeout) defer cancel() @@ -146,6 +151,14 @@ func (s *Server) movePredicate(predicate string, srcGroup, dstGroup uint32) erro predicate, srcGroup, dstGroup) glog.Info(msg) span.Annotate(nil, msg) + + in.DestGid = 0 // Indicates deletion for the predicate. + if _, err := wc.MovePredicate(ctx, in); err != nil { + msg = fmt.Sprintf("While deleting predicate [%v] in group %d. Error: %v", + in.Predicate, in.SourceGid, err) + span.Annotate(nil, msg) + glog.Warningf(msg) + } return nil } diff --git a/dgraph/cmd/zero/zero.go b/dgraph/cmd/zero/zero.go index 57b29a02e33..70dac4020e2 100644 --- a/dgraph/cmd/zero/zero.go +++ b/dgraph/cmd/zero/zero.go @@ -60,6 +60,7 @@ type Server struct { closer *y.Closer // Used to tell stream to close. connectLock sync.Mutex // Used to serialize connect requests from servers. + moveOngoing chan struct{} blockCommitsOn *sync.Map } @@ -79,6 +80,7 @@ func (s *Server) Init() { s.leaderChangeCh = make(chan struct{}, 1) s.closer = y.NewCloser(2) // grpc and http s.blockCommitsOn = new(sync.Map) + s.moveOngoing = make(chan struct{}, 1) go s.rebalanceTablets() } @@ -596,9 +598,61 @@ func (s *Server) UpdateMembership(ctx context.Context, group *pb.Group) (*api.Pa return nil, err } } + + if len(group.Members) == 0 { + return &api.Payload{Data: []byte("OK")}, nil + } + select { + case s.moveOngoing <- struct{}{}: + default: + // If a move is going on, don't do the next steps of deleting predicates. + return &api.Payload{Data: []byte("OK")}, nil + } + defer func() { + <-s.moveOngoing + }() + + if err := s.deletePredicates(ctx, group); err != nil { + glog.Warningf("While deleting predicates: %v", err) + } return &api.Payload{Data: []byte("OK")}, nil } +func (s *Server) deletePredicates(ctx context.Context, group *pb.Group) error { + gid := group.Members[0].GroupId + state, err := s.latestMembershipState(ctx) + if err != nil { + return err + } + sg, ok := state.Groups[gid] + if !ok { + return x.Errorf("Unable to find group: %d", gid) + } + + pl := s.Leader(gid) + if pl == nil { + return x.Errorf("Unable to reach leader of group: %d.", gid) + } + wc := pb.NewWorkerClient(pl.Get()) + + for pred := range group.GetTablets() { + if _, found := sg.Tablets[pred]; found { + continue + } + glog.Infof("Tablet: %v does not belong to group: %d. Sending delete instruction.", + pred, gid) + in := &pb.MovePredicatePayload{ + Predicate: pred, + SourceGid: gid, + DestGid: 0, + } + if _, err := wc.MovePredicate(ctx, in); err != nil { + return err + } + } + return nil +} + func (s *Server) StreamMembership(_ *api.Payload, stream pb.Zero_StreamMembershipServer) error { // Send MembershipState right away. So, the connection is correctly established. ctx := stream.Context() diff --git a/worker/predicate_move.go b/worker/predicate_move.go index eb826b4f3f0..4077648c831 100644 --- a/worker/predicate_move.go +++ b/worker/predicate_move.go @@ -199,6 +199,12 @@ func (w *grpcWorker) MovePredicate(ctx context.Context, func movePredicateHelper(ctx context.Context, in *pb.MovePredicatePayload) error { span := otrace.FromContext(ctx) + if in.DestGid == 0 { + // This means we need to issue a delete predicate proposal. + p := &pb.Proposal{CleanPredicate: in.Predicate} + return groups().Node.proposeAndWait(ctx, p) + } + pl := groups().Leader(in.DestGid) if pl == nil { return x.Errorf("Unable to find a connection for group: %d\n", in.DestGid) From 01803e06ab1e397ddca964a86ecab82ed7a5f8c1 Mon Sep 17 00:00:00 2001 From: Manish R Jain Date: Wed, 20 Feb 2019 15:51:34 -0800 Subject: [PATCH 2/7] Test and fix the predicate delete logic. --- dgraph/cmd/zero/zero.go | 8 +++++++- worker/groups.go | 10 ++-------- worker/predicate_move.go | 25 ++++++++++++------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/dgraph/cmd/zero/zero.go b/dgraph/cmd/zero/zero.go index 70dac4020e2..c13baaa5571 100644 --- a/dgraph/cmd/zero/zero.go +++ b/dgraph/cmd/zero/zero.go @@ -619,7 +619,13 @@ func (s *Server) UpdateMembership(ctx context.Context, group *pb.Group) (*api.Pa } func (s *Server) deletePredicates(ctx context.Context, group *pb.Group) error { - gid := group.Members[0].GroupId + var gid uint32 + for _, member := range group.Members { + gid = member.GroupId + } + if gid == 0 { + return x.Errorf("Unable to find group") + } state, err := s.latestMembershipState(ctx) if err != nil { return err diff --git a/worker/groups.go b/worker/groups.go index 01e6d412f4d..74ff5cbbe6e 100644 --- a/worker/groups.go +++ b/worker/groups.go @@ -252,14 +252,8 @@ func (g *groupi) calculateTabletSizes() map[string]*pb.Tablet { // removed from the membership state. tablet, has := tablets[pk.Attr] if !has { - if !g.ServesTablet(pk.Attr) { - if pk.IsSchema() { - itr.Next() - } else { - // data key for predicate we don't serve, skip it. - itr.Seek(pk.SkipPredicate()) - } - continue + if pk.IsSchema() { + itr.Next() } tablet = &pb.Tablet{GroupId: gid, Predicate: pk.Attr} tablets[pk.Attr] = tablet diff --git a/worker/predicate_move.go b/worker/predicate_move.go index 4077648c831..77e7b79df40 100644 --- a/worker/predicate_move.go +++ b/worker/predicate_move.go @@ -166,6 +166,10 @@ func (w *grpcWorker) MovePredicate(ctx context.Context, ctx, span := otrace.StartSpan(ctx, "worker.MovePredicate") defer span.End() + n := groups().Node + if !n.AmLeader() { + return &emptyPayload, errNotLeader + } if groups().gid != in.SourceGid { return &emptyPayload, x.Errorf("Group id doesn't match, received request for %d, my gid: %d", @@ -174,16 +178,17 @@ func (w *grpcWorker) MovePredicate(ctx context.Context, if len(in.Predicate) == 0 { return &emptyPayload, errEmptyPredicate } - if !groups().ServesTablet(in.Predicate) { - return &emptyPayload, errUnservedTablet - } - n := groups().Node - if !n.AmLeader() { - return &emptyPayload, errNotLeader + if in.DestGid == 0 { + glog.Infof("Got instructed to delete tablet: %v", in.Predicate) + p := &pb.Proposal{CleanPredicate: in.Predicate} + return &emptyPayload, groups().Node.proposeAndWait(ctx, p) } if err := posting.Oracle().WaitForTs(ctx, in.TxnTs); err != nil { return &emptyPayload, x.Errorf("While waiting for txn ts: %d. Error: %v", in.TxnTs, err) } + if !groups().ServesTablet(in.Predicate) { + return &emptyPayload, errUnservedTablet + } msg := fmt.Sprintf("Move predicate request: %+v", in) glog.Info(msg) @@ -199,12 +204,6 @@ func (w *grpcWorker) MovePredicate(ctx context.Context, func movePredicateHelper(ctx context.Context, in *pb.MovePredicatePayload) error { span := otrace.FromContext(ctx) - if in.DestGid == 0 { - // This means we need to issue a delete predicate proposal. - p := &pb.Proposal{CleanPredicate: in.Predicate} - return groups().Node.proposeAndWait(ctx, p) - } - pl := groups().Leader(in.DestGid) if pl == nil { return x.Errorf("Unable to find a connection for group: %d\n", in.DestGid) @@ -260,7 +259,7 @@ func movePredicateHelper(ctx context.Context, in *pb.MovePredicatePayload) error } kv, err := l.MarshalToKv() if kv != nil { - // HACK: Let's set all of them at this timestamp, to see what happens. + // Let's set all of them at this move timestamp. kv.Version = in.TxnTs } return listWrap(kv), err From 2ccd76955528a7eada1e1ba3431bf2097b3f47e1 Mon Sep 17 00:00:00 2001 From: Manish R Jain Date: Wed, 20 Feb 2019 15:54:53 -0800 Subject: [PATCH 3/7] Remove cleanup tablets. This is now coordinated by Zero. --- worker/groups.go | 73 +----------------------------------------------- 1 file changed, 1 insertion(+), 72 deletions(-) diff --git a/worker/groups.go b/worker/groups.go index 74ff5cbbe6e..ceb16e32685 100644 --- a/worker/groups.go +++ b/worker/groups.go @@ -138,11 +138,9 @@ func StartRaftNodes(walStore *badger.DB, bindall bool) { x.UpdateHealthStatus(true) glog.Infof("Server is ready") - gr.closer = y.NewCloser(4) // Match CLOSER:1 in this file. + gr.closer = y.NewCloser(3) // Match CLOSER:1 in this file. go gr.sendMembershipUpdates() go gr.receiveMembershipUpdates() - - go gr.cleanupTablets() go gr.processOracleDeltaStream() gr.proposeInitialSchema() @@ -771,75 +769,6 @@ OUTER: goto START } -func (g *groupi) cleanupTablets() { - defer g.closer.Done() // CLOSER:1 - defer func() { - glog.Infof("EXITING cleanupTablets.") - }() - - // TODO: Do not clean tablets for now. This causes race conditions where we end up deleting - // predicate which is being streamed over by another group. - return - - cleanup := func() { - g.blockDeletes.Lock() - defer g.blockDeletes.Unlock() - if !g.Node.AmLeader() { - return - } - glog.Infof("Running cleaning at Node: %#x Group: %d", g.Node.Id, g.groupId()) - defer glog.Info("Cleanup Done") - - opt := badger.DefaultIteratorOptions - opt.PrefetchValues = false - txn := pstore.NewTransactionAt(math.MaxUint64, false) - defer txn.Discard() - itr := txn.NewIterator(opt) - defer itr.Close() - - for itr.Rewind(); itr.Valid(); { - item := itr.Item() - pk := x.Parse(item.Key()) - if pk == nil { - itr.Next() - continue - } - - // Delete at most one predicate at a time. - // Tablet is not being served by me and is not read only. - // Don't use servesTablet function because it can return false even if - // request made to group zero fails. We might end up deleting a predicate - // on failure of network request even though no one else is serving this - // tablet. - if tablet := g.Tablet(pk.Attr); tablet != nil && tablet.GroupId != g.groupId() { - glog.Warningf("Node: %d Group: %d. Proposing predicate delete: %v. Tablet: %+v", - g.Node.Id, g.groupId(), pk.Attr, tablet) - // Predicate moves are disabled during deletion, deletePredicate purges everything. - p := &pb.Proposal{CleanPredicate: pk.Attr} - err := groups().Node.proposeAndWait(context.Background(), p) - glog.Errorf("Cleaning up predicate: %+v. Error: %v", p, err) - return - } - if pk.IsSchema() { - itr.Seek(pk.SkipSchema()) - continue - } - itr.Seek(pk.SkipPredicate()) - } - } - - ticker := time.NewTicker(5 * time.Minute) - defer ticker.Stop() - for { - select { - case <-g.closer.HasBeenClosed(): - return - case <-ticker.C: - cleanup() - } - } -} - // processOracleDeltaStream is used to process oracle delta stream from Zero. // Zero sends information about aborted/committed transactions and maxPending. func (g *groupi) processOracleDeltaStream() { From 0db57c1b1e9821f7b81785a15f7bde21f8a16dfc Mon Sep 17 00:00:00 2001 From: Manish R Jain Date: Wed, 20 Feb 2019 17:09:19 -0800 Subject: [PATCH 4/7] Use rollup to calculate the tablet sizes. That way, we don't iterate over the data periodically. --- dgraph/cmd/zero/zero.go | 10 ++++--- worker/draft.go | 45 +++++++++++++++++++++++++++++++ worker/groups.go | 60 +++-------------------------------------- 3 files changed, 55 insertions(+), 60 deletions(-) diff --git a/dgraph/cmd/zero/zero.go b/dgraph/cmd/zero/zero.go index c13baaa5571..3f37f0116d0 100644 --- a/dgraph/cmd/zero/zero.go +++ b/dgraph/cmd/zero/zero.go @@ -619,9 +619,13 @@ func (s *Server) UpdateMembership(ctx context.Context, group *pb.Group) (*api.Pa } func (s *Server) deletePredicates(ctx context.Context, group *pb.Group) error { + if group == nil || group.Tablets == nil { + return nil + } var gid uint32 - for _, member := range group.Members { - gid = member.GroupId + for _, tablet := range group.Tablets { + gid = tablet.GroupId + break } if gid == 0 { return x.Errorf("Unable to find group") @@ -641,7 +645,7 @@ func (s *Server) deletePredicates(ctx context.Context, group *pb.Group) error { } wc := pb.NewWorkerClient(pl.Get()) - for pred := range group.GetTablets() { + for pred := range group.Tablets { if _, found := sg.Tablets[pred]; found { continue } diff --git a/worker/draft.go b/worker/draft.go index 7b057d963c1..f62831eb5cc 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -23,6 +23,7 @@ import ( "fmt" "math" "sort" + "sync" "sync/atomic" "time" @@ -834,11 +835,34 @@ func listWrap(kv *bpb.KV) *bpb.KVList { func (n *node) rollupLists(readTs uint64) error { writer := posting.NewTxnWriter(pstore) + // While we're doing rollups, we should also use this opportunity to calculate the tablet sizes. + amLeader := n.AmLeader() + m := new(sync.Map) + + addTo := func(key []byte, delta int64) { + if !amLeader { + // Only leader needs to calculate the tablet sizes. + return + } + pk := x.Parse(key) + if pk == nil { + return + } + val, ok := m.Load(pk.Attr) + if !ok { + sz := new(int64) + val, _ = m.LoadOrStore(pk.Attr, sz) + } + sz := val.(*int64) + atomic.AddInt64(sz, delta) + } + stream := pstore.NewStreamAt(readTs) stream.LogPrefix = "Rolling up" stream.ChooseKey = func(item *badger.Item) bool { switch item.UserMeta() { case posting.BitSchemaPosting, posting.BitCompletePosting, posting.BitEmptyPosting: + addTo(item.Key(), item.EstimatedSize()) return false default: return true @@ -852,6 +876,7 @@ func (n *node) rollupLists(readTs uint64) error { } atomic.AddUint64(&numKeys, 1) kv, err := l.MarshalToKv() + addTo(key, int64(kv.Size())) return listWrap(kv), err } stream.Send = func(list *bpb.KVList) error { @@ -868,6 +893,26 @@ func (n *node) rollupLists(readTs uint64) error { // We can now discard all invalid versions of keys below this ts. pstore.SetDiscardTs(readTs) + + if amLeader { + go func() { + tablets := make(map[string]*pb.Tablet) + m.Range(func(key, val interface{}) bool { + pred := key.(string) + space := val.(*int64) + sz := atomic.LoadInt64(space) + tablets[pred] = &pb.Tablet{ + GroupId: n.gid, + Predicate: pred, + Space: sz, + } + return true + }) + if err := groups().doSendMembership(tablets); err != nil { + glog.Warningf("While sending membership to Zero. Error: %v", err) + } + }() + } return nil } diff --git a/worker/groups.go b/worker/groups.go index ceb16e32685..29f05fa8c44 100644 --- a/worker/groups.go +++ b/worker/groups.go @@ -19,7 +19,6 @@ package worker import ( "fmt" "io" - "math" "sort" "sync" "sync/atomic" @@ -223,45 +222,6 @@ func (g *groupi) groupId() uint32 { return atomic.LoadUint32(&g.gid) } -// calculateTabletSizes iterates through badger and gets a size of the space occupied by each -// predicate (including data and indexes). All data for a predicate forms a Tablet. -func (g *groupi) calculateTabletSizes() map[string]*pb.Tablet { - opt := badger.DefaultIteratorOptions - opt.PrefetchValues = false - txn := pstore.NewTransactionAt(math.MaxUint64, false) - defer txn.Discard() - itr := txn.NewIterator(opt) - defer itr.Close() - - gid := g.groupId() - tablets := make(map[string]*pb.Tablet) - - for itr.Rewind(); itr.Valid(); { - item := itr.Item() - - pk := x.Parse(item.Key()) - if pk == nil { - itr.Next() - continue - } - - // We should not be skipping schema keys here, otherwise if there is no data for them, they - // won't be added to the tablets map returned by this function and would ultimately be - // removed from the membership state. - tablet, has := tablets[pk.Attr] - if !has { - if pk.IsSchema() { - itr.Next() - } - tablet = &pb.Tablet{GroupId: gid, Predicate: pk.Attr} - tablets[pk.Attr] = tablet - } - tablet.Space += item.EstimatedSize() - itr.Next() - } - return tablets -} - func MaxLeaseId() uint64 { g := groups() g.RLock() @@ -628,12 +588,8 @@ func (g *groupi) doSendMembership(tablets map[string]*pb.Tablet) error { func (g *groupi) sendMembershipUpdates() { defer g.closer.Done() // CLOSER:1 - // Calculating tablet sizes is expensive, hence we do it only every 5 mins. - slowTicker := time.NewTicker(time.Minute * 5) - defer slowTicker.Stop() - - fastTicker := time.NewTicker(time.Second) - defer fastTicker.Stop() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() consumeTriggers := func() { for { @@ -651,7 +607,7 @@ func (g *groupi) sendMembershipUpdates() { select { case <-g.closer.HasBeenClosed(): return - case <-fastTicker.C: + case <-ticker.C: if time.Since(lastSent) > 10*time.Second { // On start of node if it becomes a leader, we would send tablets size for sure. g.triggerMembershipSync() @@ -666,16 +622,6 @@ func (g *groupi) sendMembershipUpdates() { } else { lastSent = time.Now() } - case <-slowTicker.C: - if !g.Node.AmLeader() { - break // breaks select case, not for loop. - } - tablets := g.calculateTabletSizes() - if err := g.doSendMembership(tablets); err != nil { - glog.Errorf("While sending membership update with tablet: %v", err) - } else { - lastSent = time.Now() - } } } } From 5e02f5036f55f0db37904db568870392df46d212 Mon Sep 17 00:00:00 2001 From: Manish R Jain Date: Wed, 20 Feb 2019 17:52:11 -0800 Subject: [PATCH 5/7] Fix up a comment --- worker/draft.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/draft.go b/worker/draft.go index f62831eb5cc..122f29c9711 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -835,7 +835,7 @@ func listWrap(kv *bpb.KV) *bpb.KVList { func (n *node) rollupLists(readTs uint64) error { writer := posting.NewTxnWriter(pstore) - // While we're doing rollups, we should also use this opportunity to calculate the tablet sizes. + // We're doing rollups. We should use this opportunity to calculate the tablet sizes. amLeader := n.AmLeader() m := new(sync.Map) From dab56842f5f54e3480d9e431b45c71f582866515 Mon Sep 17 00:00:00 2001 From: Manish R Jain Date: Thu, 21 Feb 2019 14:25:51 -0800 Subject: [PATCH 6/7] Martin's comments --- dgraph/cmd/zero/tablet.go | 7 ++++++- dgraph/cmd/zero/zero.go | 2 +- worker/draft.go | 25 ++++++++++++++++++++----- worker/predicate_move.go | 2 +- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/dgraph/cmd/zero/tablet.go b/dgraph/cmd/zero/tablet.go index 4b84fe6da9f..5f5a21fcf7e 100644 --- a/dgraph/cmd/zero/tablet.go +++ b/dgraph/cmd/zero/tablet.go @@ -152,12 +152,17 @@ func (s *Server) movePredicate(predicate string, srcGroup, dstGroup uint32) erro glog.Info(msg) span.Annotate(nil, msg) - in.DestGid = 0 // Indicates deletion for the predicate. + // Now that the move has happened, we can delete the predicate from the source group. + in.DestGid = 0 // Indicates deletion of predicate in the source group. if _, err := wc.MovePredicate(ctx, in); err != nil { msg = fmt.Sprintf("While deleting predicate [%v] in group %d. Error: %v", in.Predicate, in.SourceGid, err) span.Annotate(nil, msg) glog.Warningf(msg) + } else { + msg = fmt.Sprintf("Deleted predicate %v in group %d", in.Predicate, in.SourceGid) + span.Annotate(nil, msg) + glog.V(1).Infof(msg) } return nil } diff --git a/dgraph/cmd/zero/zero.go b/dgraph/cmd/zero/zero.go index 3f37f0116d0..fd472243ec1 100644 --- a/dgraph/cmd/zero/zero.go +++ b/dgraph/cmd/zero/zero.go @@ -641,7 +641,7 @@ func (s *Server) deletePredicates(ctx context.Context, group *pb.Group) error { pl := s.Leader(gid) if pl == nil { - return x.Errorf("Unable to reach leader of group: %d.", gid) + return x.Errorf("Unable to reach leader of group: %d", gid) } wc := pb.NewWorkerClient(pl.Get()) diff --git a/worker/draft.go b/worker/draft.go index 122f29c9711..c9dad9a0286 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -29,6 +29,7 @@ import ( "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" + humanize "github.com/dustin/go-humanize" ostats "go.opencensus.io/stats" "go.opencensus.io/tag" @@ -853,8 +854,8 @@ func (n *node) rollupLists(readTs uint64) error { sz := new(int64) val, _ = m.LoadOrStore(pk.Attr, sz) } - sz := val.(*int64) - atomic.AddInt64(sz, delta) + size := val.(*int64) + atomic.AddInt64(size, delta) } stream := pstore.NewStreamAt(readTs) @@ -895,21 +896,35 @@ func (n *node) rollupLists(readTs uint64) error { pstore.SetDiscardTs(readTs) if amLeader { + // Only leader sends the tablet size updates to Zero. No one else does. + // doSendMembership is also being concurrently called from another goroutine. go func() { tablets := make(map[string]*pb.Tablet) + var total int64 m.Range(func(key, val interface{}) bool { pred := key.(string) - space := val.(*int64) - sz := atomic.LoadInt64(space) + size := atomic.LoadInt64(val.(*int64)) tablets[pred] = &pb.Tablet{ GroupId: n.gid, Predicate: pred, - Space: sz, + Space: size, } + total += size return true }) + // Update Zero with the tablet sizes. If Zero sees a tablet which does not belong to + // this group, it would send instruction to delete that tablet. There's an edge case + // here if the followers are still running Rollup, and happen to read a key before and + // write after the tablet deletion, causing that tablet key to resurface. Then, only the + // follower would have that key, not the leader. + // However, if the follower then becomes the leader, we'd be able to get rid of that + // key then. Alternatively, we could look into cancelling the Rollup if we see a + // predicate deletion. if err := groups().doSendMembership(tablets); err != nil { glog.Warningf("While sending membership to Zero. Error: %v", err) + } else { + glog.V(2).Infof("Sent tablet size update to Zero. Total size: %d", + humanize.Bytes(uint64(total))) } }() } diff --git a/worker/predicate_move.go b/worker/predicate_move.go index 77e7b79df40..2dec7326900 100644 --- a/worker/predicate_move.go +++ b/worker/predicate_move.go @@ -179,7 +179,7 @@ func (w *grpcWorker) MovePredicate(ctx context.Context, return &emptyPayload, errEmptyPredicate } if in.DestGid == 0 { - glog.Infof("Got instructed to delete tablet: %v", in.Predicate) + glog.Infof("Was instructed to delete tablet: %v", in.Predicate) p := &pb.Proposal{CleanPredicate: in.Predicate} return &emptyPayload, groups().Node.proposeAndWait(ctx, p) } From 6e6191bce0d3343f44a112562e6d1b0b2ec4222f Mon Sep 17 00:00:00 2001 From: Manish R Jain Date: Thu, 21 Feb 2019 14:38:48 -0800 Subject: [PATCH 7/7] Use the right format --- worker/draft.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/draft.go b/worker/draft.go index c9dad9a0286..426a8ece087 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -923,7 +923,7 @@ func (n *node) rollupLists(readTs uint64) error { if err := groups().doSendMembership(tablets); err != nil { glog.Warningf("While sending membership to Zero. Error: %v", err) } else { - glog.V(2).Infof("Sent tablet size update to Zero. Total size: %d", + glog.V(2).Infof("Sent tablet size update to Zero. Total size: %s", humanize.Bytes(uint64(total))) } }()