-
Notifications
You must be signed in to change notification settings - Fork 3.8k
/
store.go
4494 lines (4073 loc) · 159 KB
/
store.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package storage
import (
"bytes"
"context"
"fmt"
"io"
"math"
"runtime"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/coreos/etcd/raft"
"github.com/coreos/etcd/raft/raftpb"
"github.com/google/btree"
opentracing "github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"golang.org/x/time/rate"
"github.com/cockroachdb/cockroach/pkg/base"
"github.com/cockroachdb/cockroach/pkg/config"
"github.com/cockroachdb/cockroach/pkg/gossip"
"github.com/cockroachdb/cockroach/pkg/internal/client"
"github.com/cockroachdb/cockroach/pkg/keys"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/rpc"
"github.com/cockroachdb/cockroach/pkg/settings"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/sql/sqlutil"
"github.com/cockroachdb/cockroach/pkg/storage/batcheval"
"github.com/cockroachdb/cockroach/pkg/storage/compactor"
"github.com/cockroachdb/cockroach/pkg/storage/engine"
"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
"github.com/cockroachdb/cockroach/pkg/storage/idalloc"
"github.com/cockroachdb/cockroach/pkg/storage/spanset"
"github.com/cockroachdb/cockroach/pkg/storage/stateloader"
"github.com/cockroachdb/cockroach/pkg/storage/storagebase"
"github.com/cockroachdb/cockroach/pkg/storage/tscache"
"github.com/cockroachdb/cockroach/pkg/storage/txnwait"
"github.com/cockroachdb/cockroach/pkg/util/envutil"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/humanizeutil"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/metric"
"github.com/cockroachdb/cockroach/pkg/util/protoutil"
"github.com/cockroachdb/cockroach/pkg/util/retry"
"github.com/cockroachdb/cockroach/pkg/util/shuffle"
"github.com/cockroachdb/cockroach/pkg/util/stop"
"github.com/cockroachdb/cockroach/pkg/util/syncutil"
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/cockroach/pkg/util/tracing"
"github.com/cockroachdb/cockroach/pkg/util/uuid"
)
const (
// rangeIDAllocCount is the number of Range IDs to allocate per allocation.
rangeIDAllocCount = 10
defaultHeartbeatIntervalTicks = 5
// ttlStoreGossip is time-to-live for store-related info.
ttlStoreGossip = 2 * time.Minute
// preemptiveSnapshotRaftGroupID is a bogus ID for which a Raft group is
// temporarily created during the application of a preemptive snapshot.
preemptiveSnapshotRaftGroupID = math.MaxUint64
// defaultRaftEntryCacheSize is the default size in bytes for a
// store's Raft log entry cache.
defaultRaftEntryCacheSize = 1 << 24 // 16M
// replicaRequestQueueSize specifies the maximum number of requests to queue
// for a replica.
replicaRequestQueueSize = 100
defaultGossipWhenCapacityDeltaExceedsFraction = 0.01
// systemDataGossipInterval is the interval at which range lease
// holders verify that the most recent system data is gossiped.
// This ensures that system data is always eventually gossiped, even
// if a range lease holder experiences a failure causing a missed
// gossip update.
systemDataGossipInterval = 1 * time.Minute
// Messages that provide detail about why a preemptive snapshot was rejected.
snapshotStoreTooFullMsg = "store almost out of disk space"
snapshotApplySemBusyMsg = "store busy applying snapshots and/or removing replicas"
storeDrainingMsg = "store is draining"
// IntersectingSnapshotMsg is part of the error message returned from
// canApplySnapshotLocked and is exposed here so testing can rely on it.
IntersectingSnapshotMsg = "snapshot intersects existing range"
)
var changeTypeInternalToRaft = map[roachpb.ReplicaChangeType]raftpb.ConfChangeType{
roachpb.ADD_REPLICA: raftpb.ConfChangeAddNode,
roachpb.REMOVE_REPLICA: raftpb.ConfChangeRemoveNode,
}
var storeSchedulerConcurrency = envutil.EnvOrDefaultInt(
"COCKROACH_SCHEDULER_CONCURRENCY", 8*runtime.NumCPU())
var enablePreVote = envutil.EnvOrDefaultBool(
"COCKROACH_ENABLE_PREVOTE", true)
var enableTickQuiesced = envutil.EnvOrDefaultBool(
"COCKROACH_ENABLE_TICK_QUIESCED", true)
// TestStoreConfig has some fields initialized with values relevant in tests.
func TestStoreConfig(clock *hlc.Clock) StoreConfig {
if clock == nil {
clock = hlc.NewClock(hlc.UnixNano, time.Nanosecond)
}
st := cluster.MakeTestingClusterSettings()
sc := StoreConfig{
Settings: st,
AmbientCtx: log.AmbientContext{Tracer: st.Tracer},
Clock: clock,
CoalescedHeartbeatsInterval: 50 * time.Millisecond,
RaftHeartbeatIntervalTicks: 1,
ScanInterval: 10 * time.Minute,
TimestampCachePageSize: tscache.TestSklPageSize,
HistogramWindowInterval: metric.TestSampleInterval,
EnableEpochRangeLeases: true,
}
// Use shorter Raft tick settings in order to minimize start up and failover
// time in tests.
sc.RaftElectionTimeoutTicks = 3
sc.RaftTickInterval = 100 * time.Millisecond
sc.SetDefaults()
return sc
}
var (
raftMaxSizePerMsg = envutil.EnvOrDefaultInt("COCKROACH_RAFT_MAX_SIZE_PER_MSG", 16*1024)
raftMaxInflightMsgs = envutil.EnvOrDefaultInt("COCKROACH_RAFT_MAX_INFLIGHT_MSGS", 64)
)
func newRaftConfig(
strg raft.Storage, id uint64, appliedIndex uint64, storeCfg StoreConfig, logger raft.Logger,
) *raft.Config {
return &raft.Config{
ID: id,
Applied: appliedIndex,
ElectionTick: storeCfg.RaftElectionTimeoutTicks,
HeartbeatTick: storeCfg.RaftHeartbeatIntervalTicks,
Storage: strg,
Logger: logger,
// TODO(bdarnell): PreVote and CheckQuorum are two ways of
// achieving the same thing. PreVote is more compatible with
// quiesced ranges, so we want to switch to it once we've worked
// out the bugs.
PreVote: enablePreVote,
CheckQuorum: !enablePreVote,
// MaxSizePerMsg controls how many Raft log entries the leader will send to
// followers in a single MsgApp.
MaxSizePerMsg: uint64(raftMaxSizePerMsg),
// MaxInflightMsgs controls how many "inflight" messages Raft will send to
// a follower without hearing a response. The total number of Raft log
// entries is a combination of this setting and MaxSizePerMsg. The current
// settings provide for up to 1 MB of raft log to be sent without
// acknowledgement. With an average entry size of 1 KB that translates to
// ~1024 commands that might be executed in the handling of a single
// raft.Ready operation.
MaxInflightMsgs: raftMaxInflightMsgs,
}
}
// verifyKeys verifies keys. If checkEndKey is true, then the end key
// is verified to be non-nil and greater than start key. If
// checkEndKey is false, end key is verified to be nil. Additionally,
// verifies that start key is less than KeyMax and end key is less
// than or equal to KeyMax. It also verifies that a key range that
// contains range-local keys is completely range-local.
func verifyKeys(start, end roachpb.Key, checkEndKey bool) error {
if bytes.Compare(start, roachpb.KeyMax) >= 0 {
return errors.Errorf("start key %q must be less than KeyMax", start)
}
if !checkEndKey {
if len(end) != 0 {
return errors.Errorf("end key %q should not be specified for this operation", end)
}
return nil
}
if end == nil {
return errors.Errorf("end key must be specified")
}
if bytes.Compare(roachpb.KeyMax, end) < 0 {
return errors.Errorf("end key %q must be less than or equal to KeyMax", end)
}
{
sAddr, err := keys.Addr(start)
if err != nil {
return err
}
eAddr, err := keys.Addr(end)
if err != nil {
return err
}
if !sAddr.Less(eAddr) {
return errors.Errorf("end key %q must be greater than start %q", end, start)
}
if !bytes.Equal(sAddr, start) {
if bytes.Equal(eAddr, end) {
return errors.Errorf("start key is range-local, but end key is not")
}
} else if bytes.Compare(start, keys.LocalMax) < 0 {
// It's a range op, not local but somehow plows through local data -
// not cool.
return errors.Errorf("start key in [%q,%q) must be greater than LocalMax", start, end)
}
}
return nil
}
// rangeKeyItem is a common interface for roachpb.Key and Range.
type rangeKeyItem interface {
endKey() roachpb.RKey
}
// rangeBTreeKey is a type alias of roachpb.RKey that implements the
// rangeKeyItem interface and the btree.Item interface.
type rangeBTreeKey roachpb.RKey
var _ rangeKeyItem = rangeBTreeKey{}
func (k rangeBTreeKey) endKey() roachpb.RKey {
return (roachpb.RKey)(k)
}
var _ btree.Item = rangeBTreeKey{}
func (k rangeBTreeKey) Less(i btree.Item) bool {
return k.endKey().Less(i.(rangeKeyItem).endKey())
}
// A NotBootstrappedError indicates that an engine has not yet been
// bootstrapped due to a store identifier not being present.
type NotBootstrappedError struct{}
// Error formats error.
func (e *NotBootstrappedError) Error() string {
return "store has not been bootstrapped"
}
// A storeReplicaVisitor calls a visitor function for each of a store's
// initialized Replicas (in unspecified order).
type storeReplicaVisitor struct {
store *Store
repls []*Replica // Replicas to be visited.
visited int // Number of visited ranges, -1 before first call to Visit()
}
// Len implements shuffle.Interface.
func (rs storeReplicaVisitor) Len() int { return len(rs.repls) }
// Swap implements shuffle.Interface.
func (rs storeReplicaVisitor) Swap(i, j int) { rs.repls[i], rs.repls[j] = rs.repls[j], rs.repls[i] }
// newStoreReplicaVisitor constructs a storeReplicaVisitor.
func newStoreReplicaVisitor(store *Store) *storeReplicaVisitor {
return &storeReplicaVisitor{
store: store,
visited: -1,
}
}
// Visit calls the visitor with each Replica until false is returned.
func (rs *storeReplicaVisitor) Visit(visitor func(*Replica) bool) {
// Copy the range IDs to a slice so that we iterate over some (possibly
// stale) view of all Replicas without holding the Store lock. In particular,
// no locks are acquired during the copy process.
rs.repls = nil
rs.store.mu.replicas.Range(func(k int64, v unsafe.Pointer) bool {
rs.repls = append(rs.repls, (*Replica)(v))
return true
})
// The Replicas are already in "unspecified order" due to map iteration,
// but we want to make sure it's completely random to prevent issues in
// tests where stores are scanning replicas in lock-step and one store is
// winning the race and getting a first crack at processing the replicas on
// its queues.
//
// TODO(peter): Re-evaluate whether this is necessary after we allow
// rebalancing away from the leaseholder. See TestRebalance_3To5Small.
shuffle.Shuffle(rs)
rs.visited = 0
for _, repl := range rs.repls {
// TODO(tschottdorf): let the visitor figure out if something's been
// destroyed once we return errors from mutexes (#9190). After all, it
// can still happen with this code.
rs.visited++
repl.mu.RLock()
destroyed := repl.mu.destroyStatus
initialized := repl.isInitializedRLocked()
repl.mu.RUnlock()
if initialized && (destroyed.IsAlive() || destroyed.reason == destroyReasonRemovalPending) && !visitor(repl) {
break
}
}
rs.visited = 0
}
// EstimatedCount returns an estimated count of the underlying store's
// replicas.
//
// TODO(tschottdorf): this method has highly doubtful semantics.
func (rs *storeReplicaVisitor) EstimatedCount() int {
if rs.visited <= 0 {
return rs.store.ReplicaCount()
}
return len(rs.repls) - rs.visited
}
type raftRequestInfo struct {
req *RaftMessageRequest
respStream RaftMessageResponseStream
}
type raftRequestQueue struct {
syncutil.Mutex
infos []raftRequestInfo
}
// A Store maintains a map of ranges by start key. A Store corresponds
// to one physical device.
type Store struct {
Ident roachpb.StoreIdent
cfg StoreConfig
db *client.DB
engine engine.Engine // The underlying key-value store
compactor *compactor.Compactor // Schedules compaction of the engine
tsCache tscache.Cache // Most recent timestamps for keys / key ranges
allocator Allocator // Makes allocation decisions
rangeIDAlloc *idalloc.Allocator // Range ID allocator
gcQueue *gcQueue // Garbage collection queue
splitQueue *splitQueue // Range splitting queue
replicateQueue *replicateQueue // Replication queue
replicaGCQueue *replicaGCQueue // Replica GC queue
raftLogQueue *raftLogQueue // Raft log truncation queue
raftSnapshotQueue *raftSnapshotQueue // Raft repair queue
tsMaintenanceQueue *timeSeriesMaintenanceQueue // Time series maintenance queue
scanner *replicaScanner // Replica scanner
consistencyQueue *consistencyQueue // Replica consistency check queue
metrics *StoreMetrics
intentResolver *intentResolver
raftEntryCache *raftEntryCache
// gossipRangeCountdown and leaseRangeCountdown are countdowns of
// changes to range and leaseholder counts, after which the store
// descriptor will be re-gossiped earlier than the normal periodic
// gossip interval. Updated atomically.
gossipRangeCountdown int32
gossipLeaseCountdown int32
// gossipWritesPerSecondVal serves a similar purpose, but simply records
// the most recently gossiped value so that we can tell if a newly measured
// value differs by enough to justify re-gossiping the store.
gossipWritesPerSecondVal syncutil.AtomicFloat64
coalescedMu struct {
syncutil.Mutex
heartbeats map[roachpb.StoreIdent][]RaftHeartbeat
heartbeatResponses map[roachpb.StoreIdent][]RaftHeartbeat
}
// 1 if the store was started, 0 if it wasn't. To be accessed using atomic
// ops.
started int32
stopper *stop.Stopper
// The time when the store was Start()ed, in nanos.
startedAt int64
nodeDesc *roachpb.NodeDescriptor
initComplete sync.WaitGroup // Signaled by async init tasks
idleReplicaElectionTime struct {
syncutil.Mutex
at time.Time
}
// Semaphore to limit concurrent non-empty snapshot application and replica
// data destruction.
snapshotApplySem chan struct{}
// draining holds a bool which indicates whether this store is draining. See
// SetDraining() for a more detailed explanation of behavior changes.
//
// TODO(bdarnell,tschottdorf): Would look better inside of `mu`, which at
// the time of its creation was riddled with deadlock (but that situation
// has likely improved).
draining atomic.Value
// Locking notes: To avoid deadlocks, the following lock order must be
// obeyed: Replica.raftMu < Replica.readOnlyCmdMu < Store.mu < Replica.mu
// < Replica.unreachablesMu < Store.coalescedMu < Store.scheduler.mu.
// (It is not required to acquire every lock in sequence, but when multiple
// locks are held at the same time, it is incorrect to acquire a lock with
// "lesser" value in this sequence after one with "greater" value).
//
// Methods of Store with a "Locked" suffix require that
// Store.mu.Mutex be held. Other locking requirements are indicated
// in comments.
//
// The locking structure here is complex because A) Store is a
// container of Replicas, so it must generally be consulted before
// doing anything with any Replica, B) some Replica operations
// (including splits) modify the Store. Therefore we generally lock
// Store.mu to find a Replica, release it, then call a method on the
// Replica. These short-lived locks of Store.mu and Replica.mu are
// often surrounded by a long-lived lock of Replica.raftMu as
// described below.
//
// There are two major entry points to this stack of locks:
// Store.Send (which handles incoming RPCs) and raft-related message
// processing (including handleRaftReady on the processRaft
// goroutine and HandleRaftRequest on GRPC goroutines). Reads are
// processed solely through Store.Send; writes start out on
// Store.Send until they propose their raft command and then they
// finish on the raft goroutines.
//
// TODO(bdarnell): a Replica could be destroyed immediately after
// Store.Send finds the Replica and releases the lock. We need
// another RWMutex to be held by anything using a Replica to ensure
// that everything is finished before releasing it. #7169
//
// Detailed description of the locks:
//
// * Replica.raftMu: Held while any raft messages are being processed
// (including handleRaftReady and HandleRaftRequest) or while the set of
// Replicas in the Store is being changed (which may happen outside of raft
// via the replica GC queue).
//
// * Replica.readOnlyCmdMu (RWMutex): Held in read mode while any
// read-only command is in progress on the replica; held in write
// mode while executing a commit trigger. This is necessary
// because read-only commands mutate the Replica's timestamp cache
// (while holding Replica.mu in addition to readOnlyCmdMu). The
// RWMutex ensures that no reads are being executed during a split
// (which copies the timestamp cache) while still allowing
// multiple reads in parallel (#3148). TODO(bdarnell): this lock
// only needs to be held during splitTrigger, not all triggers.
//
// * Store.mu: Protects the Store's map of its Replicas. Acquired and
// released briefly at the start of each request; metadata operations like
// splits acquire it again to update the map. Even though these lock
// acquisitions do not make up a single critical section, it is safe thanks
// to Replica.raftMu which prevents any concurrent modifications.
//
// * Replica.mu: Protects the Replica's in-memory state. Acquired
// and released briefly as needed (note that while the lock is
// held "briefly" in that it is not held for an entire request, we
// do sometimes do I/O while holding the lock, as in
// Replica.Entries). This lock should be held when calling any
// methods on the raft group. Raft may call back into the Replica
// via the methods of the raft.Storage interface, which assume the
// lock is held even though they do not follow our convention of
// the "Locked" suffix.
//
// * Store.scheduler.mu: Protects the Raft scheduler internal
// state. Callbacks from the scheduler are performed while not holding this
// mutex in order to observe the above ordering constraints.
//
// Splits (and merges, but they're not finished and so will not be discussed
// here) deserve special consideration: they operate on two ranges. Naively,
// this is fine because the right-hand range is brand new, but an
// uninitialized version may have been created by a raft message before we
// process the split (see commentary on Replica.splitTrigger). We make this
// safe by locking the right-hand range for the duration of the Raft command
// containing the split/merge trigger.
//
// Note that because we acquire and release Store.mu and Replica.mu
// repeatedly rather than holding a lock for an entire request, we are
// actually relying on higher-level locks to ensure that things don't change
// out from under us. In particular, handleRaftReady accesses the replicaID
// more than once, and we rely on Replica.raftMu to ensure that this is not
// modified by a concurrent HandleRaftRequest. (#4476)
mu struct {
syncutil.RWMutex
// Map of replicas by Range ID (map[roachpb.RangeID]*Replica). This
// includes `uninitReplicas`. May be read without holding Store.mu.
replicas syncutil.IntMap
// A btree key containing objects of type *Replica or
// *ReplicaPlaceholder (both of which have an associated key range, on
// the EndKey of which the btree is keyed)
replicasByKey *btree.BTree
uninitReplicas map[roachpb.RangeID]*Replica // Map of uninitialized replicas by Range ID
// replicaPlaceholders is a map to access all placeholders, so they can
// be directly accessed and cleared after stepping all raft groups.
replicaPlaceholders map[roachpb.RangeID]*ReplicaPlaceholder
}
// replicaQueues is a map of per-Replica incoming request queues. These
// queues might more naturally belong in Replica, but are kept separate to
// avoid reworking the locking in getOrCreateReplica which requires
// Replica.raftMu to be held while a replica is being inserted into
// Store.mu.replicas.
replicaQueues syncutil.IntMap // map[roachpb.RangeID]*raftRequestQueue
scheduler *raftScheduler
counts struct {
// Number of placeholders removed due to error.
removedPlaceholders int32
// Number of placeholders successfully filled by a snapshot.
filledPlaceholders int32
// Number of placeholders removed due to a snapshot that was dropped by
// raft.
droppedPlaceholders int32
}
}
var _ client.Sender = &Store{}
// A StoreConfig encompasses the auxiliary objects and configuration
// required to create a store.
// All fields holding a pointer or an interface are required to create
// a store; the rest will have sane defaults set if omitted.
type StoreConfig struct {
AmbientCtx log.AmbientContext
base.RaftConfig
Settings *cluster.Settings
Clock *hlc.Clock
DB *client.DB
Gossip *gossip.Gossip
NodeLiveness *NodeLiveness
StorePool *StorePool
Transport *RaftTransport
RPCContext *rpc.Context
// SQLExecutor is used by the store to execute SQL statements in a way that
// is more direct than using a sql.Executor.
SQLExecutor sqlutil.InternalExecutor
// TimeSeriesDataStore is an interface used by the store's time series
// maintenance queue to dispatch individual maintenance tasks.
TimeSeriesDataStore TimeSeriesDataStore
// DontRetryPushTxnFailures will propagate a push txn failure immediately
// instead of utilizing the txn wait queue to wait for the transaction to
// finish or be pushed by a higher priority contender.
DontRetryPushTxnFailures bool
// CoalescedHeartbeatsInterval is the interval for which heartbeat messages
// are queued and then sent as a single coalesced heartbeat; it is a
// fraction of the RaftTickInterval so that heartbeats don't get delayed by
// an entire tick. Delaying coalescing heartbeat responses has a bad
// interaction with quiescence because the coalesced (delayed) heartbeat
// response can unquiesce the leader. Consider:
//
// T+0: leader queues MsgHeartbeat
// T+1: leader sends MsgHeartbeat
// follower receives MsgHeartbeat
// follower queues MsgHeartbeatResp
// T+2: leader queues quiesce message
// follower sends MsgHeartbeatResp
// leader receives MsgHeartbeatResp
// T+3: leader sends quiesce message
//
// Thus we want to make sure that heartbeats are responded to faster than
// the quiesce cadence.
CoalescedHeartbeatsInterval time.Duration
// RaftHeartbeatIntervalTicks is the number of ticks that pass between heartbeats.
RaftHeartbeatIntervalTicks int
// ScanInterval is the default value for the scan interval
ScanInterval time.Duration
// ScanMaxIdleTime is the maximum time the scanner will be idle between ranges.
// If enabled (> 0), the scanner may complete in less than ScanInterval for small
// stores.
ScanMaxIdleTime time.Duration
// If LogRangeEvents is true, major changes to ranges will be logged into
// the range event log.
LogRangeEvents bool
// RaftEntryCacheSize is the size in bytes of the Raft log entry cache
// shared by all Raft groups managed by the store.
RaftEntryCacheSize uint64
// IntentResolverTaskLimit is the maximum number of asynchronous tasks that
// may be started by the intent resolver. -1 indicates no asynchronous tasks
// are allowed. 0 uses the default value (defaultIntentResolverTaskLimit)
// which is non-zero.
IntentResolverTaskLimit int
TestingKnobs StoreTestingKnobs
// concurrentSnapshotApplyLimit specifies the maximum number of empty
// snapshots and the maximum number of non-empty snapshots that are permitted
// to be applied concurrently.
concurrentSnapshotApplyLimit int
// TimestampCachePageSize is (server.Config).TimestampCachePageSize
TimestampCachePageSize uint32
// HistogramWindowInterval is (server.Config).HistogramWindowInterval
HistogramWindowInterval time.Duration
// EnableEpochRangeLeases controls whether epoch-based range leases are used.
EnableEpochRangeLeases bool
// GossipWhenCapacityDeltaExceedsFraction specifies the fraction from the last
// gossiped store capacity values which need be exceeded before the store will
// gossip immediately without waiting for the periodic gossip interval.
GossipWhenCapacityDeltaExceedsFraction float64
}
// StoreTestingKnobs is a part of the context used to control parts of
// the system. The Testing*Filter functions are called at various
// points in the request pipeline if they are non-nil. These can be
// used either for synchronization (e.g. to write to a channel when a
// particular point is reached) or to change the behavior by returning
// an error (which aborts all further processing for the command).
type StoreTestingKnobs struct {
EvalKnobs batcheval.TestingKnobs
// TestingProposalFilter is called before proposing each command.
TestingProposalFilter storagebase.ReplicaProposalFilter
// TestingApplyFilter is called before applying the results of a
// command on each replica. If it returns an error, the command will
// not be applied. If it returns an error on some replicas but not
// others, the behavior is poorly defined unless that error is a
// ReplicaCorruptionError.
TestingApplyFilter storagebase.ReplicaApplyFilter
// TestingPostApplyFilter is called after a command is applied to
// rocksdb but before in-memory side effects have been processed.
TestingPostApplyFilter storagebase.ReplicaApplyFilter
// TestingResponseFilter is called after the replica processes a
// command in order for unittests to modify the batch response,
// error returned to the client, or to simulate network failures.
TestingResponseFilter storagebase.ReplicaResponseFilter
// If non-nil, BadChecksumPanic is called by CheckConsistency() instead of
// panicking on a checksum mismatch.
BadChecksumPanic func(roachpb.StoreIdent)
// If non-nil, BadChecksumReportDiff is called by CheckConsistency() on a
// checksum mismatch to report the diff between snapshots.
BadChecksumReportDiff func(roachpb.StoreIdent, []ReplicaSnapshotDiff)
// Disables the use of optional one phase commits. Even when enabled, requests
// that set the Require1PC flag are permitted to use one phase commits. This
// prevents wedging node liveness, which requires one phase commits during
// liveness updates.
DisableOptional1PC bool
// A hack to manipulate the clock before sending a batch request to a replica.
// TODO(kaneda): This hook is not encouraged to use. Get rid of it once
// we make TestServer take a ManualClock.
ClockBeforeSend func(*hlc.Clock, roachpb.BatchRequest)
// OnCampaign is called if the replica campaigns for Raft leadership
// when initializing the Raft group. Note that this method is invoked
// with both Replica.raftMu and Replica.mu locked.
OnCampaign func(*Replica)
// OnCommandQueueAction is called when the BatchRequest performs an action
// on the CommandQueue.
OnCommandQueueAction func(*roachpb.BatchRequest, storagebase.CommandQueueAction)
// MaxOffset, if set, overrides the server clock's MaxOffset at server
// creation time.
// See also DisableMaxOffsetCheck.
MaxOffset time.Duration
// DisableMaxOffsetCheck disables the rejection (in Store.Send) of requests
// with the timestamp too much in the future. Normally, this rejection is a
// good sanity check, but certain tests unfortunately insert a "message from
// the future" into the system to advance the clock of a TestServer. We
// should get rid of such practices once we make TestServer take a
// ManualClock.
DisableMaxOffsetCheck bool
// DontPreventUseOfOldLeaseOnStart disables the initialization of
// replica.mu.minLeaseProposedTS on replica.Init(). This has the effect of
// allowing the replica to use the lease that it had in a previous life (in
// case the tests persisted the engine used in said previous life).
DontPreventUseOfOldLeaseOnStart bool
// LeaseRequestEvent, if set, is called when replica.requestLeaseLocked() is
// called to acquire a new lease. This can be used to assert that a request
// triggers a lease acquisition.
LeaseRequestEvent func(ts hlc.Timestamp)
// LeaseTransferBlockedOnExtensionEvent, if set, is called when
// replica.TransferLease() encounters an in-progress lease extension.
// nextLeader is the replica that we're trying to transfer the lease to.
LeaseTransferBlockedOnExtensionEvent func(nextLeader roachpb.ReplicaDescriptor)
// DisableReplicaGCQueue disables the replica GC queue.
DisableReplicaGCQueue bool
// DisableReplicateQueue disables the replication queue.
DisableReplicateQueue bool
// DisableReplicaRebalancing disables rebalancing of replicas but otherwise
// leaves the replicate queue operational.
DisableReplicaRebalancing bool
// DisableSplitQueue disables the split queue.
DisableSplitQueue bool
// DisableTimeSeriesMaintenanceQueue disables the time series maintenance
// queue.
DisableTimeSeriesMaintenanceQueue bool
// DisableRaftSnapshotQueue disables the raft snapshot queue.
DisableRaftSnapshotQueue bool
// DisableScanner disables the replica scanner.
DisableScanner bool
// DisablePeriodicGossips disables periodic gossiping.
DisablePeriodicGossips bool
// DisableRefreshReasonTicks disables refreshing pending commands when a new
// leader is discovered.
DisableRefreshReasonNewLeader bool
// DisableRefreshReasonTicks disables refreshing pending commands when a
// snapshot is applied.
DisableRefreshReasonSnapshotApplied bool
// DisableRefreshReasonTicks disables refreshing pending commands
// periodically.
DisableRefreshReasonTicks bool
// DisableProcessRaft disables the process raft loop.
DisableProcessRaft bool
// DisableLastProcessedCheck disables checking on replica queue last processed times.
DisableLastProcessedCheck bool
// ReplicateQueueAcceptsUnsplit allows the replication queue to
// process ranges that need to be split, for use in tests that use
// the replication queue but disable the split queue.
ReplicateQueueAcceptsUnsplit bool
// SkipMinSizeCheck, if set, makes the store creation process skip the check
// for a minimum size.
SkipMinSizeCheck bool
// DisableAsyncIntentResolution disables the async intent resolution
// path (but leaves synchronous resolution). This can avoid some
// edge cases in tests that start and stop servers.
DisableAsyncIntentResolution bool
// DisableLeaseCapacityGossip disables the ability of a changing number of
// leases to trigger the store to gossip its capacity. With this enabled,
// only changes in the number of replicas can cause the store to gossip its
// capacity.
DisableLeaseCapacityGossip bool
// BootstrapVersion overrides the version the stores will be bootstrapped with.
BootstrapVersion *cluster.ClusterVersion
}
var _ base.ModuleTestingKnobs = &StoreTestingKnobs{}
// ModuleTestingKnobs is part of the base.ModuleTestingKnobs interface.
func (*StoreTestingKnobs) ModuleTestingKnobs() {}
// Valid returns true if the StoreConfig is populated correctly.
// We don't check for Gossip and DB since some of our tests pass
// that as nil.
func (sc *StoreConfig) Valid() bool {
return sc.Clock != nil && sc.Transport != nil &&
sc.RaftTickInterval != 0 && sc.RaftHeartbeatIntervalTicks > 0 &&
sc.RaftElectionTimeoutTicks > 0 && sc.ScanInterval >= 0 &&
sc.AmbientCtx.Tracer != nil
}
// SetDefaults initializes unset fields in StoreConfig to values
// suitable for use on a local network.
// TODO(tschottdorf): see if this ought to be configurable via flags.
func (sc *StoreConfig) SetDefaults() {
sc.RaftConfig.SetDefaults()
if sc.CoalescedHeartbeatsInterval == 0 {
sc.CoalescedHeartbeatsInterval = sc.RaftTickInterval / 2
}
if sc.RaftHeartbeatIntervalTicks == 0 {
sc.RaftHeartbeatIntervalTicks = defaultHeartbeatIntervalTicks
}
if sc.RaftEntryCacheSize == 0 {
sc.RaftEntryCacheSize = defaultRaftEntryCacheSize
}
if sc.IntentResolverTaskLimit == 0 {
sc.IntentResolverTaskLimit = defaultIntentResolverTaskLimit
} else if sc.IntentResolverTaskLimit == -1 {
sc.IntentResolverTaskLimit = 0
}
if sc.concurrentSnapshotApplyLimit == 0 {
// NB: setting this value higher than 1 is likely to degrade client
// throughput.
sc.concurrentSnapshotApplyLimit =
envutil.EnvOrDefaultInt("COCKROACH_CONCURRENT_SNAPSHOT_APPLY_LIMIT", 1)
}
if sc.GossipWhenCapacityDeltaExceedsFraction == 0 {
sc.GossipWhenCapacityDeltaExceedsFraction = defaultGossipWhenCapacityDeltaExceedsFraction
}
}
// LeaseExpiration returns an int64 to increment a manual clock with to
// make sure that all active range leases expire.
func (sc *StoreConfig) LeaseExpiration() int64 {
// Due to lease extensions, the remaining interval can be longer than just
// the sum of the offset (=length of stasis period) and the active
// duration, but definitely not by 2x.
maxOffset := sc.Clock.MaxOffset()
if maxOffset == timeutil.ClocklessMaxOffset {
// Don't do shady math on clockless reads.
maxOffset = 0
}
return 2 * (sc.RangeLeaseActiveDuration() + maxOffset).Nanoseconds()
}
// NewStore returns a new instance of a store.
func NewStore(cfg StoreConfig, eng engine.Engine, nodeDesc *roachpb.NodeDescriptor) *Store {
// TODO(tschottdorf): find better place to set these defaults.
cfg.SetDefaults()
if !cfg.Valid() {
log.Fatalf(context.Background(), "invalid store configuration: %+v", &cfg)
}
s := &Store{
cfg: cfg,
db: cfg.DB, // TODO(tschottdorf): remove redundancy.
engine: eng,
nodeDesc: nodeDesc,
metrics: newStoreMetrics(cfg.HistogramWindowInterval),
}
if cfg.RPCContext != nil {
s.allocator = MakeAllocator(cfg.StorePool, cfg.RPCContext.RemoteClocks.Latency)
} else {
s.allocator = MakeAllocator(cfg.StorePool, func(string) (time.Duration, bool) {
return 0, false
})
}
s.intentResolver = newIntentResolver(s, cfg.IntentResolverTaskLimit)
s.raftEntryCache = newRaftEntryCache(cfg.RaftEntryCacheSize)
s.draining.Store(false)
s.scheduler = newRaftScheduler(s.cfg.AmbientCtx, s.metrics, s, storeSchedulerConcurrency)
s.coalescedMu.Lock()
s.coalescedMu.heartbeats = map[roachpb.StoreIdent][]RaftHeartbeat{}
s.coalescedMu.heartbeatResponses = map[roachpb.StoreIdent][]RaftHeartbeat{}
s.coalescedMu.Unlock()
s.mu.Lock()
s.mu.replicaPlaceholders = map[roachpb.RangeID]*ReplicaPlaceholder{}
s.mu.replicasByKey = btree.New(64 /* degree */)
s.mu.uninitReplicas = map[roachpb.RangeID]*Replica{}
s.mu.Unlock()
tsCacheMetrics := tscache.MakeMetrics()
s.tsCache = tscache.New(cfg.Clock, cfg.TimestampCachePageSize, tsCacheMetrics)
s.metrics.registry.AddMetricStruct(tsCacheMetrics)
s.compactor = compactor.NewCompactor(
s.engine.(engine.WithSSTables),
s.Capacity,
func(ctx context.Context) { s.asyncGossipStore(ctx, "compactor-initiated rocksdb compaction") },
)
s.metrics.registry.AddMetricStruct(s.compactor.Metrics)
s.snapshotApplySem = make(chan struct{}, cfg.concurrentSnapshotApplyLimit)
if s.cfg.Gossip != nil {
// Add range scanner and configure with queues.
s.scanner = newReplicaScanner(
s.cfg.AmbientCtx, s.cfg.Clock, cfg.ScanInterval,
cfg.ScanMaxIdleTime, newStoreReplicaVisitor(s),
)
s.gcQueue = newGCQueue(s, s.cfg.Gossip)
s.splitQueue = newSplitQueue(s, s.db, s.cfg.Gossip)
s.replicateQueue = newReplicateQueue(s, s.cfg.Gossip, s.allocator)
s.replicaGCQueue = newReplicaGCQueue(s, s.db, s.cfg.Gossip)
s.raftLogQueue = newRaftLogQueue(s, s.db, s.cfg.Gossip)
s.raftSnapshotQueue = newRaftSnapshotQueue(s, s.cfg.Gossip)
s.consistencyQueue = newConsistencyQueue(s, s.cfg.Gossip)
s.scanner.AddQueues(
s.gcQueue, s.splitQueue, s.replicateQueue, s.replicaGCQueue,
s.raftLogQueue, s.raftSnapshotQueue, s.consistencyQueue)
if s.cfg.TimeSeriesDataStore != nil {
s.tsMaintenanceQueue = newTimeSeriesMaintenanceQueue(
s, s.db, s.cfg.Gossip, s.cfg.TimeSeriesDataStore,
)
s.scanner.AddQueues(s.tsMaintenanceQueue)
}
}
if cfg.TestingKnobs.DisableReplicaGCQueue {
s.setReplicaGCQueueActive(false)
}
if cfg.TestingKnobs.DisableReplicateQueue {
s.setReplicateQueueActive(false)
}
if cfg.TestingKnobs.DisableSplitQueue {
s.setSplitQueueActive(false)
}
if cfg.TestingKnobs.DisableTimeSeriesMaintenanceQueue {
s.setTimeSeriesMaintenanceQueueActive(false)
}
if cfg.TestingKnobs.DisableRaftSnapshotQueue {
s.setRaftSnapshotQueueActive(false)
}
if cfg.TestingKnobs.DisableScanner {
s.setScannerActive(false)
}
return s
}
// String formats a store for debug output.
func (s *Store) String() string {
return fmt.Sprintf("[n%d,s%d]", s.Ident.NodeID, s.Ident.StoreID)
}
// ClusterSettings returns the node's ClusterSettings.
func (s *Store) ClusterSettings() *cluster.Settings {
return s.cfg.Settings
}
// AnnotateCtx is a convenience wrapper; see AmbientContext.
func (s *Store) AnnotateCtx(ctx context.Context) context.Context {
return s.cfg.AmbientCtx.AnnotateCtx(ctx)
}
// SetDraining (when called with 'true') causes incoming lease transfers to be
// rejected, prevents all of the Store's Replicas from acquiring or extending
// range leases, and attempts to transfer away any leases owned.
// When called with 'false', returns to the normal mode of operation.
func (s *Store) SetDraining(drain bool) {
s.draining.Store(drain)
if !drain {
newStoreReplicaVisitor(s).Visit(func(r *Replica) bool {
r.mu.Lock()
r.mu.draining = drain
r.mu.Unlock()
return true
})
return
}
var wg sync.WaitGroup
ctx := log.WithLogTag(context.Background(), "drain", nil)
// Limit the number of concurrent lease transfers.
sem := make(chan struct{}, 100)
sysCfg, sysCfgSet := s.cfg.Gossip.GetSystemConfig()
newStoreReplicaVisitor(s).Visit(func(r *Replica) bool {
wg.Add(1)
if err := s.stopper.RunLimitedAsyncTask(
r.AnnotateCtx(ctx), "storage.Store: draining replica", sem, true, /* wait */
func(ctx context.Context) {
defer wg.Done()
r.mu.Lock()
r.mu.draining = true
r.mu.Unlock()
var drainingLease roachpb.Lease
for {
var llHandle *leaseRequestHandle
r.mu.Lock()
lease, nextLease := r.getLeaseRLocked()
if nextLease != nil && nextLease.OwnedBy(s.StoreID()) {
llHandle = r.mu.pendingLeaseRequest.JoinRequest()
}
r.mu.Unlock()
if llHandle != nil {
<-llHandle.C()
continue
}
drainingLease = lease
break
}
if drainingLease.OwnedBy(s.StoreID()) && r.IsLeaseValid(drainingLease, s.Clock().Now()) {
desc := r.Desc()
zone := config.DefaultZoneConfig()
if sysCfgSet {
var err error
zone, err = sysCfg.GetZoneConfigForKey(desc.StartKey)
if log.V(1) && err != nil {
log.Errorf(ctx, "could not get zone config for key %s when draining: %s", desc.StartKey, err)
}
}
transferred, err := s.replicateQueue.transferLease(
ctx,
r,
desc,
zone,
transferLeaseOptions{},
)
if log.V(1) {
if transferred {
log.Infof(ctx, "transferred lease %s for replica %s", drainingLease, desc)
} else {