-
Notifications
You must be signed in to change notification settings - Fork 3.8k
/
store.go
3783 lines (3396 loc) · 146 KB
/
store.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2014 The Cockroach Authors.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
package kvserver
import (
"bytes"
"context"
"fmt"
"math"
"path/filepath"
"runtime"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"unsafe"
"github.com/cockroachdb/cockroach/pkg/base"
"github.com/cockroachdb/cockroach/pkg/cloud"
"github.com/cockroachdb/cockroach/pkg/clusterversion"
"github.com/cockroachdb/cockroach/pkg/config"
"github.com/cockroachdb/cockroach/pkg/config/zonepb"
"github.com/cockroachdb/cockroach/pkg/gossip"
"github.com/cockroachdb/cockroach/pkg/keys"
"github.com/cockroachdb/cockroach/pkg/kv"
"github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/raftentry"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/tenantrate"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/tscache"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/rpc"
"github.com/cockroachdb/cockroach/pkg/rpc/nodedialer"
"github.com/cockroachdb/cockroach/pkg/settings"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/spanconfig"
"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigstore"
"github.com/cockroachdb/cockroach/pkg/sql/sqlutil"
"github.com/cockroachdb/cockroach/pkg/storage"
"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
"github.com/cockroachdb/cockroach/pkg/util/admission"
"github.com/cockroachdb/cockroach/pkg/util/contextutil"
"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
"github.com/cockroachdb/cockroach/pkg/util/envutil"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/iterutil"
"github.com/cockroachdb/cockroach/pkg/util/limit"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/metric"
"github.com/cockroachdb/cockroach/pkg/util/mon"
"github.com/cockroachdb/cockroach/pkg/util/protoutil"
"github.com/cockroachdb/cockroach/pkg/util/quotapool"
"github.com/cockroachdb/cockroach/pkg/util/retry"
"github.com/cockroachdb/cockroach/pkg/util/shuffle"
"github.com/cockroachdb/cockroach/pkg/util/stop"
"github.com/cockroachdb/cockroach/pkg/util/syncutil"
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/cockroach/pkg/util/tracing"
"github.com/cockroachdb/cockroach/pkg/util/uuid"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/logtags"
"github.com/cockroachdb/redact"
raft "go.etcd.io/etcd/raft/v3"
"golang.org/x/time/rate"
)
const (
// rangeIDAllocCount is the number of Range IDs to allocate per allocation.
rangeIDAllocCount = 10
// defaultRaftEntryCacheSize is the default size in bytes for a
// store's Raft log entry cache.
defaultRaftEntryCacheSize = 1 << 24 // 16M
// replicaRequestQueueSize specifies the maximum number of requests to queue
// for a replica.
replicaRequestQueueSize = 100
defaultGossipWhenCapacityDeltaExceedsFraction = 0.01
// systemDataGossipInterval is the interval at which range lease
// holders verify that the most recent system data is gossiped.
// This ensures that system data is always eventually gossiped, even
// if a range lease holder experiences a failure causing a missed
// gossip update.
systemDataGossipInterval = 1 * time.Minute
)
var storeSchedulerConcurrency = envutil.EnvOrDefaultInt(
// For small machines, we scale the scheduler concurrency by the number of
// CPUs. 8*NumCPU was determined in 9a68241 (April 2017) as the optimal
// concurrency level on 8 CPU machines. For larger machines, we've seen
// (#56851) that this scaling curve can be too aggressive and lead to too much
// contention in the Raft scheduler, so we cap the concurrency level at 96.
//
// As of November 2020, this default value could be re-tuned.
"COCKROACH_SCHEDULER_CONCURRENCY", min(8*runtime.GOMAXPROCS(0), 96))
var logSSTInfoTicks = envutil.EnvOrDefaultInt(
"COCKROACH_LOG_SST_INFO_TICKS_INTERVAL", 60)
// bulkIOWriteLimit is defined here because it is used by BulkIOWriteLimiter.
var bulkIOWriteLimit = settings.RegisterByteSizeSetting(
settings.TenantWritable,
"kv.bulk_io_write.max_rate",
"the rate limit (bytes/sec) to use for writes to disk on behalf of bulk io ops",
1<<40,
).WithPublic()
// addSSTableRequestLimit limits concurrent AddSSTable requests.
var addSSTableRequestLimit = settings.RegisterIntSetting(
settings.TenantWritable,
"kv.bulk_io_write.concurrent_addsstable_requests",
"number of concurrent AddSSTable requests per store before queueing",
1,
settings.PositiveInt,
)
// addSSTableAsWritesRequestLimit limits concurrent AddSSTable requests with
// IngestAsWrites set. These are smaller (kv.bulk_io_write.small_write_size),
// and will end up in the Pebble memtable (default 64 MB) before flushing to
// disk, so we can allow a greater amount of concurrency than regular AddSSTable
// requests. Applied independently of concurrent_addsstable_requests.
var addSSTableAsWritesRequestLimit = settings.RegisterIntSetting(
settings.TenantWritable,
"kv.bulk_io_write.concurrent_addsstable_as_writes_requests",
"number of concurrent AddSSTable requests ingested as writes per store before queueing",
10,
settings.PositiveInt,
)
// concurrentRangefeedItersLimit limits concurrent rangefeed catchup iterators.
var concurrentRangefeedItersLimit = settings.RegisterIntSetting(
settings.TenantWritable,
"kv.rangefeed.concurrent_catchup_iterators",
"number of rangefeeds catchup iterators a store will allow concurrently before queueing",
16,
settings.PositiveInt,
)
// concurrentscanInterleavedIntentsLimit is the number of concurrent
// ScanInterleavedIntents requests that will be run on a store. Used as part
// of pre-evaluation throttling.
var concurrentscanInterleavedIntentsLimit = settings.RegisterIntSetting(
settings.TenantWritable,
"kv.migration.concurrent_scan_interleaved_intents",
"number of scan interleaved intents requests a store will handle concurrently before queueing",
1,
settings.PositiveInt,
)
// Minimum time interval between system config updates which will lead to
// enqueuing replicas.
var queueAdditionOnSystemConfigUpdateRate = settings.RegisterFloatSetting(
settings.TenantWritable,
"kv.store.system_config_update.queue_add_rate",
"the rate (per second) at which the store will add, all replicas to the split and merge queue due to system config gossip",
.5,
settings.NonNegativeFloat,
)
// Minimum time interval between system config updates which will lead to
// enqueuing replicas. The default is relatively high to deal with startup
// scenarios.
var queueAdditionOnSystemConfigUpdateBurst = settings.RegisterIntSetting(
settings.TenantWritable,
"kv.store.system_config_update.queue_add_burst",
"the burst rate at which the store will add all replicas to the split and merge queue due to system config gossip",
32,
settings.NonNegativeInt,
)
// leaseTransferWait is the timeout for a single iteration of draining range leases.
var leaseTransferWait = func() *settings.DurationSetting {
s := settings.RegisterDurationSetting(
settings.TenantWritable,
leaseTransferWaitSettingName,
"the timeout for a single iteration of the range lease transfer phase of draining "+
"(note that the --drain-wait parameter for cockroach node drain may need adjustment "+
"after changing this setting)",
5*time.Second,
func(v time.Duration) error {
if v < 0 {
return errors.Errorf("cannot set %s to a negative duration: %s",
leaseTransferWaitSettingName, v)
}
return nil
},
)
s.SetVisibility(settings.Public)
return s
}()
const leaseTransferWaitSettingName = "server.shutdown.lease_transfer_wait"
// ExportRequestsLimit is the number of Export requests that can run at once.
// Each extracts data from RocksDB to a temp file and then uploads it to cloud
// storage. In order to not exhaust the disk or memory, or saturate the network,
// limit the number of these that can be run in parallel. This number was chosen
// by a guessing - it could be improved by more measured heuristics. Exported
// here since we check it in in the caller to limit generated requests as well
// to prevent excessive queuing.
var ExportRequestsLimit = settings.RegisterIntSetting(
settings.TenantWritable,
"kv.bulk_io_write.concurrent_export_requests",
"number of export requests a store will handle concurrently before queuing",
3,
settings.PositiveInt,
)
// TestStoreConfig has some fields initialized with values relevant in tests.
func TestStoreConfig(clock *hlc.Clock) StoreConfig {
return testStoreConfig(clock, clusterversion.TestingBinaryVersion)
}
func testStoreConfig(clock *hlc.Clock, version roachpb.Version) StoreConfig {
if clock == nil {
clock = hlc.NewClock(hlc.UnixNano, time.Nanosecond)
}
st := cluster.MakeTestingClusterSettingsWithVersions(version, version, true)
tracer := tracing.NewTracerWithOpt(context.TODO(), tracing.WithClusterSettings(&st.SV))
sc := StoreConfig{
DefaultSpanConfig: zonepb.DefaultZoneConfigRef().AsSpanConfig(),
Settings: st,
AmbientCtx: log.MakeTestingAmbientContext(tracer),
Clock: clock,
CoalescedHeartbeatsInterval: 50 * time.Millisecond,
ScanInterval: 10 * time.Minute,
HistogramWindowInterval: metric.TestSampleInterval,
ProtectedTimestampReader: spanconfig.EmptyProtectedTSReader(clock),
// Use a constant empty system config, which mirrors the previously
// existing logic to install an empty system config in gossip.
SystemConfigProvider: config.NewConstantSystemConfigProvider(
config.NewSystemConfig(zonepb.DefaultZoneConfigRef()),
),
}
// Use shorter Raft tick settings in order to minimize start up and failover
// time in tests.
sc.RaftHeartbeatIntervalTicks = 1
sc.RaftElectionTimeoutTicks = 3
sc.RaftTickInterval = 100 * time.Millisecond
sc.SetDefaults()
return sc
}
func newRaftConfig(
strg raft.Storage, id uint64, appliedIndex uint64, storeCfg StoreConfig, logger raft.Logger,
) *raft.Config {
return &raft.Config{
ID: id,
Applied: appliedIndex,
ElectionTick: storeCfg.RaftElectionTimeoutTicks,
HeartbeatTick: storeCfg.RaftHeartbeatIntervalTicks,
MaxUncommittedEntriesSize: storeCfg.RaftMaxUncommittedEntriesSize,
MaxCommittedSizePerReady: storeCfg.RaftMaxCommittedSizePerReady,
MaxSizePerMsg: storeCfg.RaftMaxSizePerMsg,
MaxInflightMsgs: storeCfg.RaftMaxInflightMsgs,
Storage: strg,
Logger: logger,
PreVote: true,
}
}
// verifyKeys verifies keys. If checkEndKey is true, then the end key
// is verified to be non-nil and greater than start key. If
// checkEndKey is false, end key is verified to be nil. Additionally,
// verifies that start key is less than KeyMax and end key is less
// than or equal to KeyMax. It also verifies that a key range that
// contains range-local keys is completely range-local.
func verifyKeys(start, end roachpb.Key, checkEndKey bool) error {
if bytes.Compare(start, roachpb.KeyMax) >= 0 {
return errors.Errorf("start key %q must be less than KeyMax", start)
}
if !checkEndKey {
if len(end) != 0 {
return errors.Errorf("end key %q should not be specified for this operation", end)
}
return nil
}
if end == nil {
return errors.Errorf("end key must be specified")
}
if bytes.Compare(roachpb.KeyMax, end) < 0 {
return errors.Errorf("end key %q must be less than or equal to KeyMax", end)
}
{
sAddr, err := keys.Addr(start)
if err != nil {
return err
}
eAddr, err := keys.Addr(end)
if err != nil {
return err
}
if !sAddr.Less(eAddr) {
return errors.Errorf("end key %q must be greater than start %q", end, start)
}
if !bytes.Equal(sAddr, start) {
if bytes.Equal(eAddr, end) {
return errors.Errorf("start key is range-local, but end key is not")
}
} else if bytes.Compare(start, keys.LocalMax) < 0 {
// It's a range op, not local but somehow plows through local data -
// not cool.
return errors.Errorf("start key in [%q,%q) must be greater than LocalMax", start, end)
}
}
return nil
}
// A NotBootstrappedError indicates that an engine has not yet been
// bootstrapped due to a store identifier not being present.
type NotBootstrappedError struct{}
// Error formats error.
func (e *NotBootstrappedError) Error() string {
return "store has not been bootstrapped"
}
// A storeReplicaVisitor calls a visitor function for each of a store's
// initialized Replicas (in unspecified order). It provides an option
// to visit replicas in increasing RangeID order.
type storeReplicaVisitor struct {
store *Store
repls []*Replica // Replicas to be visited
ordered bool // Option to visit replicas in sorted order
visited int // Number of visited ranges, -1 before first call to Visit()
}
// Len implements sort.Interface.
func (rs storeReplicaVisitor) Len() int { return len(rs.repls) }
// Less implements sort.Interface.
func (rs storeReplicaVisitor) Less(i, j int) bool { return rs.repls[i].RangeID < rs.repls[j].RangeID }
// Swap implements sort.Interface.
func (rs storeReplicaVisitor) Swap(i, j int) { rs.repls[i], rs.repls[j] = rs.repls[j], rs.repls[i] }
// newStoreReplicaVisitor constructs a storeReplicaVisitor.
func newStoreReplicaVisitor(store *Store) *storeReplicaVisitor {
return &storeReplicaVisitor{
store: store,
visited: -1,
}
}
// InOrder tells the visitor to visit replicas in increasing RangeID order.
func (rs *storeReplicaVisitor) InOrder() *storeReplicaVisitor {
rs.ordered = true
return rs
}
// Visit calls the visitor with each Replica until false is returned.
func (rs *storeReplicaVisitor) Visit(visitor func(*Replica) bool) {
// Copy the range IDs to a slice so that we iterate over some (possibly
// stale) view of all Replicas without holding the Store lock. In particular,
// no locks are acquired during the copy process.
rs.repls = nil
rs.store.mu.replicasByRangeID.Range(func(repl *Replica) {
rs.repls = append(rs.repls, repl)
})
if rs.ordered {
// If the replicas were requested in sorted order, perform the sort.
sort.Sort(rs)
} else {
// The Replicas are already in "unspecified order" due to map iteration,
// but we want to make sure it's completely random to prevent issues in
// tests where stores are scanning replicas in lock-step and one store is
// winning the race and getting a first crack at processing the replicas on
// its queues.
//
// TODO(peter): Re-evaluate whether this is necessary after we allow
// rebalancing away from the leaseholder. See TestRebalance_3To5Small.
shuffle.Shuffle(rs)
}
rs.visited = 0
for _, repl := range rs.repls {
// TODO(tschottdorf): let the visitor figure out if something's been
// destroyed once we return errors from mutexes (#9190). After all, it
// can still happen with this code.
rs.visited++
repl.mu.RLock()
destroyed := repl.mu.destroyStatus
initialized := repl.isInitializedRLocked()
repl.mu.RUnlock()
if initialized && destroyed.IsAlive() && !visitor(repl) {
break
}
}
rs.visited = 0
}
// EstimatedCount returns an estimated count of the underlying store's
// replicas.
//
// TODO(tschottdorf): this method has highly doubtful semantics.
func (rs *storeReplicaVisitor) EstimatedCount() int {
if rs.visited <= 0 {
return rs.store.ReplicaCount()
}
return len(rs.repls) - rs.visited
}
/*
A Store maintains a set of Replicas whose data is stored on a storage.Engine
usually corresponding to a dedicated storage medium. It also houses a collection
of subsystems that, in broad terms, perform maintenance of each Replica on the
Store when required. In particular, this includes various queues such as the
split, merge, rebalance, GC, and replicaGC queues (to name just a few).
INVARIANT: the set of all Ranges (as determined by, e.g. a transactionally
consistent scan of the meta index ranges) always exactly covers the addressable
keyspace roachpb.KeyMin (inclusive) to roachpb.KeyMax (exclusive).
Ranges
Each Replica is part of a Range, i.e. corresponds to what other systems would
call a shard. A Range is a consensus group backed by Raft, i.e. each Replica is
a state machine backed by a replicated log of commands. In CockroachDB, Ranges
own a contiguous chunk of addressable keyspace, where the word "addressable" is
a fine-print that interested readers can learn about in keys.Addr; in short the
Replica data is logically contiguous, but not contiguous when viewed as Engine
kv pairs.
Considerable complexity is incurred by the features of dynamic re-sharding and
relocation, i.e. range splits, range merges, and range relocation (also called
replication changes, or, in the case of lateral movement, rebalancing). Each of
these interact heavily with the Range as a consensus group (of which each
Replica is a member). All of these intricacies are described at a high level in
this comment.
RangeDescriptor
A roachpb.RangeDescriptor is the configuration of a Range. It is an
MVCC-backed key-value pair (where the key is derived from the StartKey via
keys.RangeDescriptorKey, which in particular resides on the Range itself)
that is accessible to the transactional KV API much like any other, but is
for internal use only (and in particular plays no role at the SQL layer).
Splits, Merges, and Rebalances are all carried out as distributed
transactions. They are all complex in their own right but they share the
basic approach of transactionally acting on the RangeDescriptor. Each of
these operations at some point will
- start a transaction
- update the RangeDescriptor (for example, to reflect a split, or a change
to the Replicas comprising the members of the Range)
- update the meta ranges (which form a search index used for request routing, see
kv.RangeLookup and updateRangeAddressing for details)
- commit with a roachpb.InternalCommitTrigger.
In particular, note that the RangeDescriptor is the first write issued in the
transaction, and so the transaction record will be created on the affected
range. This allows us to establish a helpful invariant:
INVARIANT: an intent on keys.RangeDescriptorKey is resolved atomically with
the (application of the) roachpb.EndTxnRequest committing the transaction.
A Replica's active configuration is dictated by its visible version of the
RangeDescriptor, and the above invariant simplifies this. Without the invariant,
the new RangeDescriptor would come into effect when the intent is resolved,
which is a) later (so requests might get routed to this Replica without it being
ready to handle them appropriately) and b) requires special-casing around
ResolveIntent acting on the RangeDescriptor.
INVARIANT: A Store never contains two Replicas from the same Range, nor do the
key ranges for any two of its Replicas overlap. (Note that there is no
requirement that these Replicas come from a consistent set of Ranges; the
RangeDescriptor.Generation orders overlapping descriptors).
To illustrate this last invariant, consider a split of a Replica [a-z) into two
Replicas, [a-c) and [c,z). The Store will never contain both; it has to swap
directly from [a-z) to [a,c)+[c,z). For a similar example, consider accepting a
new Replica [b,d) when a Replica [a,c) is already present, which is similarly
not allowed. To understand how such situations could arise, consider that a
series of changes to the RangeDescriptor is observed asynchronously by
followers. This means that a follower may have Replica [a-z) while any number of
splits and replication changes are already known to the leaseholder, and the new
Ranges created in the process may be attempting to add a Replica to the slow
follower.
With the invariant, we effectively allow looking up a unique (if it exists)
Replica for any given key, and ensure that no two Replicas on a Store operate on
shared keyspace (as seen by the storage.Engine). Refer to the Replica Lifecycle
diagram below for details on how this invariant is upheld.
Replica Lifecycle
A Replica should be thought of primarily as a State Machine applying commands
from a replicated log (the log being replicated across the members of the
Range). The Store's RaftTransport receives Raft messages from Replicas residing
on other Stores and routes them to the appropriate Replicas via
Store.HandleRaftRequest (which is part of the RaftMessageHandler interface),
ultimately resulting in a call to Replica.handleRaftReadyRaftMuLocked, which
houses the integration with the etcd/raft library (raft.RawNode). This may
generate Raft messages to be sent to other Stores; these are handed to
Replica.sendRaftMessagesRaftMuLocked which ultimately hands them to the Store's
RaftTransport.SendAsync method. Raft uses message passing (not
request-response), and outgoing messages will use a gRPC stream that differs
from that used for incoming messages (which makes asymmetric partitions more
likely in case of stream-specific problems). The steady state is relatively
straightforward but when Ranges are being reconfigured, an understanding the
Replica Lifecycle becomes important and upholding the Store's invariants becomes
more complex.
A first phenomenon to understand is that of uninitialized Replicas, which is the
State Machine at applied index zero, i.e. has an empty state. In CockroachDB, an
uninitialized Replica can only advance to a nonzero log position ("become
initialized") via a Raft snapshot (this is because we initialize all Ranges in
the system at log index raftInitialLogIndex which allows us to write arbitrary
amounts of data into the initial state without having to worry about the size
of individual log entries; see WriteInitialReplicaState).
An uninitialized Replica has no notion of its active RangeDescriptor yet, has no
data, and should be thought of as a pure raft peer that can react to incoming
messages (it can't send messages, as it is unaware of who the other members of
the group are; its configuration is zero, consistent with the configuration
represented by "no RangeDescriptor"); in particular it may be asked to cast a
vote to determine Raft leadership. (This should not be required in CockroachDB
today since we always add an uninitialized Replica as a roachpb.LEARNER first
and only promote it after it has successfully received a Raft snapshot; we don't
rely on this fact today)
Uninitialized Replicas should be viewed as an implementation detail that is (or
should be!) invisible to most access to a Store in the context of processing
requests coming from the KV API, as uninitialized Replicas cannot serve such
requests. At the time of writing, uninitialized Replicas need to be handled
in many code paths that should never encounter them in the first place. We
will be addressing this with #72374.
Raft snapshots can also be directed at initialized Replicas. For practical
reasons, we cannot preserve the entire committed replicated log forever and
periodically purge a prefix that is known to be durably applied on a quorum of
peers. Under certain circumstances (see newTruncateDecision) this may cut a
follower off from the log, and this follower will require a Raft snapshot before
being able to resume replication. Another (rare) source of snapshots occurs
around Range splits. During a split, the involved Replicas shrink and
simultaneously instantiate the right-hand side Replica, but since Replicas carry
out this operation at different times, a "faster" initialized right-hand side
might contact a "slow" store on which the right-hand side has not yet been
instantiated, leading to the creation of an uninitialized Replica that may
request a snapshot. See maybeDelaySplitToAvoidSnapshot.
The diagram is a lot to take in. The various transitions are discussed in
prose below, and the source .dot file is in store_doc_replica_lifecycle.dot.
+---------------------+
+------------------ | Absent | ---------------------------------------------------------------------------------------------------+
| +---------------------+ |
| | Subsume Crash applySnapshot |
| | Store.Start +---------------+ +---------+ +---------------+ |
| v v | v | v | |
| +-----------------------------------------------------------------------------------------------------------------------+ |
+---------+------------------ | | |
| | | Initialized | |
| | | | |
| +----+------------------ | | -+----+
| | | +-----------------------------------------------------------------------------------------------------------------------+ | |
| | | | ^ ^ | | | | |
| | | Raft msg | Crash | applySnapshot | post-split | | | | |
| | | v | | | | | | |
| | | +---------------------------------------------------------+ pre-split | | | | |
| | +-----------------> | | <---------------------+--------------+--------------------+----+ |
| | | | | | | |
| | | Uninitialized | Raft msg | | | |
| | | | -----------------+ | | | |
| | | | | | | | |
| | | | <----------------+ | | | |
| | +---------------------------------------------------------+ | | apply removal | |
| | | | | | | |
| | | ReplicaTooOldError | higher ReplicaID | Replica GC | | |
| | v v v | | |
| | Merged (snapshot) +---------------------------------------------------------------------------------------------+ | | |
| +----------------------> | | <+ | |
| | | | |
| apply Merge | | ReplicaTooOld | |
+---------------------------> | Removed | <---------------------+ |
| | |
| | higher ReplicaID |
| | <-------------------------------+
+---------------------------------------------------------------------------------------------+
When a Store starts, it iterates through all RangeDescriptors it can find on its
Engine. Finding a RangeDescriptor by definition implies that the Replica is
initialized. Raft state (a raftpb.HardState) for uninitialized Replicas may
exist, however it would be ignored until a message arrives addressing that
Replica, or a split trigger applies that instantiates and then initializes it.
Uninitialized Replicas principally arise when a replication change occurs and a
new member is added to a Range. This new member will be contacted by the Raft
leader, creating an uninitialized Replica on the recipient which will then
negotiate a snapshot to become initialized and to receive the replicated log. A
split can be understood as a special case of that, except that all members of
the Range are created at around the same time (though in practice with arbitrary
delays due to the usual distributed systems reasons), and the uninitialized
Replica creation is triggered by the split trigger executing on the left-hand
side of the split, or a Replica of the right-hand side (already initialized by
the split trigger on another Store) reaching out, whichever occurs first.
An uninitialized Replica requires a snapshot to become initialized. The case in
which the Replica is the right-hand side of a split can be understood as the
application of a snapshot as well (though this is not reflected in code at the
time of writing) where the left-hand side applies the split by (logically)
moving any data past the split point to the right-hand side Replica, thus
initializing it. In principle, since writes to the state machine do not need to
be made durable, it is conceivable that a split or snapshot could "unapply" due
to an ill-timed crash (though snapshots currently use SST ingestion, which the
storage engine performs durably). Similarly, entry application (which only
occurs on initialized Replicas) is not synced and so a suffix of applied entries
may need to be re-applied following a crash.
If an uninitialized Replica receives a Raft message from a peer informing it
that it is no longer part of a more up-to-date Range configuration (via a
ReplicaTooOldError) or that it has been removed and re-added under a higher
ReplicaID, the uninitialized Replica is removed. There is currently no
general-purpose mechanism to determine whether an uninitialized Replica is
outdated; an uninitialized Replica could in principle leak "forever" if the
Range quickly changes its members such that the triggers mentioned here don't
apply. A full scan of meta2 would be required as there is no RangeID-keyed index
on the RangeDescriptors (the meta ranges are keyed on the EndKey, which allows
routing requests based on the key ranges they touch). The fact that
uninitialized Replicas can be removed has to be taken into account by splits as
well; the split trigger may find that the right-hand side uninitialized Replica
has already been removed, in which case the right half of the split has to be
discarded (see acquireSplitLock and splitPostApply).
Initialized Replicas represent the common case. They can apply snapshots
(required if they get cut off from the raft log via log truncation) which will
always move the applied log position forward, however this is rare. A Replica
typically spends most of its life applying log entries and/or serving reads. The
mechanisms that can lead to removal of uninitialized Replicas apply to
initialized Replicas in the exact same way, but there are additional ways for an
initialized Replica to be removed. The most general such mechanism is the
replica GC queue, which periodically checks the meta2 copy of the Range's
descriptor (using a consistent read, i.e. reading the latest version). If this
indicates that the Replica on the local Store should no longer exist, the
Replica is removed. Additionally, the Replica may directly witness a change that
indicates a need for a removal. For example, it may apply a change to the range
descriptor that removes it, or it may be destroyed by the application of a merge
on its left neighboring Replica, which may also occur through a snapshot. Merges
are the single most complex reconfiguration operation and can only be touched
upon here. At their core, they will at some point "freeze" the right-hand side
Replicas (via roachpb.SubsumeRequest) to prevent additional read or write
activity, and also ensure that the two sets of Ranges to be merged are
co-located on the same Stores as well as are all initialized.
INVARIANT: An initialized Replica's RangeDescriptor always includes it as a member.
INVARIANT: RangeDescriptor.StartKey is immutable (splits and merges mutate the
EndKey only). In particular, an initialized Replica's StartKey is immutable.
INVARIANT: A Replica's ReplicaID is constant.
NOTE: the way to read this is that a Replica object will not be re-used for
multiple ReplicaIDs. Instead, the existing Replica will be destroyed and a new
Replica instantiated.
These invariants significantly reduce complexity since we do not need to handle
the excluded situations. Particularly a changing replicaID is bug-inducing since
at the Replication layer a change in replica ID is a complete change of
identity, and re-use of in-memory structures poses the threat of erroneously
re-using cached information.
INVARIANT: for each key `k` in the replicated key space, the Generation of the
RangeDescriptor containing is strictly increasing over time (see
RangeDescriptor.Generation).
NOTE: we rely on this invariant for cache coherency in `kvcoord`; it currently
plays no role in `kvserver` though we could use it to improve the handling of
snapshot overlaps; see Store.checkSnapshotOverlapLocked.
NOTE: a key may be owned by different RangeIDs at different points in time due
to Range splits and merges.
INVARIANT: on each Store and for each RangeID, the ReplicaID is strictly
increasing over time (see Replica.setTombstoneKey).
NOTE: to the best of our knowledge, we don't rely on this invariant.
*/
type Store struct {
Ident *roachpb.StoreIdent // pointer to catch access before Start() is called
cfg StoreConfig
db *kv.DB
engine storage.Engine // The underlying key-value store
tsCache tscache.Cache // Most recent timestamps for keys / key ranges
allocator Allocator // Makes allocation decisions
replRankings *replicaRankings
storeRebalancer *StoreRebalancer
rangeIDAlloc *idalloc.Allocator // Range ID allocator
mvccGCQueue *mvccGCQueue // MVCC GC queue
mergeQueue *mergeQueue // Range merging queue
splitQueue *splitQueue // Range splitting queue
replicateQueue *replicateQueue // Replication queue
replicaGCQueue *replicaGCQueue // Replica GC queue
raftLogQueue *raftLogQueue // Raft log truncation queue
// Carries out truncations proposed by the raft log queue, and "replicated"
// via raft, when they are safe. Created in Store.Start.
raftTruncator *raftLogTruncator
raftSnapshotQueue *raftSnapshotQueue // Raft repair queue
tsMaintenanceQueue *timeSeriesMaintenanceQueue // Time series maintenance queue
scanner *replicaScanner // Replica scanner
consistencyQueue *consistencyQueue // Replica consistency check queue
consistencyLimiter *quotapool.RateLimiter // Rate limits consistency checks
consistencySem *quotapool.IntPool // Limit concurrent consistency checks
metrics *StoreMetrics
intentResolver *intentresolver.IntentResolver
recoveryMgr txnrecovery.Manager
raftEntryCache *raftentry.Cache
limiters batcheval.Limiters
txnWaitMetrics *txnwait.Metrics
sstSnapshotStorage SSTSnapshotStorage
protectedtsReader spanconfig.ProtectedTSReader
ctSender *sidetransport.Sender
// gossipRangeCountdown and leaseRangeCountdown are countdowns of
// changes to range and leaseholder counts, after which the store
// descriptor will be re-gossiped earlier than the normal periodic
// gossip interval. Updated atomically.
gossipRangeCountdown int32
gossipLeaseCountdown int32
// gossipQueriesPerSecondVal and gossipWritesPerSecond serve similar
// purposes, but simply record the most recently gossiped value so that we
// can tell if a newly measured value differs by enough to justify
// re-gossiping the store.
gossipQueriesPerSecondVal syncutil.AtomicFloat64
gossipWritesPerSecondVal syncutil.AtomicFloat64
coalescedMu struct {
syncutil.Mutex
heartbeats map[roachpb.StoreIdent][]kvserverpb.RaftHeartbeat
heartbeatResponses map[roachpb.StoreIdent][]kvserverpb.RaftHeartbeat
}
// 1 if the store was started, 0 if it wasn't. To be accessed using atomic
// ops.
started int32
stopper *stop.Stopper
// The time when the store was Start()ed, in nanos.
startedAt int64
nodeDesc *roachpb.NodeDescriptor
initComplete sync.WaitGroup // Signaled by async init tasks
// Semaphore to limit concurrent non-empty snapshot application.
snapshotApplySem chan struct{}
// Track newly-acquired expiration-based leases that we want to proactively
// renew. An object is sent on the signal whenever a new entry is added to
// the map.
renewableLeases syncutil.IntMap // map[roachpb.RangeID]*Replica
renewableLeasesSignal chan struct{} // 1-buffered
// draining holds a bool which indicates whether this store is draining. See
// SetDraining() for a more detailed explanation of behavior changes.
//
// TODO(bdarnell,tschottdorf): Would look better inside of `mu`, which at
// the time of its creation was riddled with deadlock (but that situation
// has likely improved).
draining atomic.Value
// Locking notes: To avoid deadlocks, the following lock order must be
// obeyed: baseQueue.mu < Replica.raftMu < Replica.readOnlyCmdMu < Store.mu
// < Replica.mu < Replica.unreachablesMu < Store.coalescedMu < Store.scheduler.mu.
// (It is not required to acquire every lock in sequence, but when multiple
// locks are held at the same time, it is incorrect to acquire a lock with
// "lesser" value in this sequence after one with "greater" value).
//
// Methods of Store with a "Locked" suffix require that
// Store.mu.Mutex be held. Other locking requirements are indicated
// in comments.
//
// The locking structure here is complex because A) Store is a
// container of Replicas, so it must generally be consulted before
// doing anything with any Replica, B) some Replica operations
// (including splits) modify the Store. Therefore we generally lock
// Store.mu to find a Replica, release it, then call a method on the
// Replica. These short-lived locks of Store.mu and Replica.mu are
// often surrounded by a long-lived lock of Replica.raftMu as
// described below.
//
// There are two major entry points to this stack of locks:
// Store.Send (which handles incoming RPCs) and raft-related message
// processing (including handleRaftReady on the processRaft
// goroutine and HandleRaftRequest on GRPC goroutines). Reads are
// processed solely through Store.Send; writes start out on
// Store.Send until they propose their raft command and then they
// finish on the raft goroutines.
//
// TODO(bdarnell): a Replica could be destroyed immediately after
// Store.Send finds the Replica and releases the lock. We need
// another RWMutex to be held by anything using a Replica to ensure
// that everything is finished before releasing it. #7169
//
// Detailed description of the locks:
//
// * Replica.raftMu: Held while any raft messages are being processed
// (including handleRaftReady and HandleRaftRequest) or while the set of
// Replicas in the Store is being changed (which may happen outside of raft
// via the replica GC queue).
//
// If holding raftMus for multiple different replicas simultaneously,
// acquire the locks in the order that the replicas appear in replicasByKey.
//
// * Replica.readOnlyCmdMu (RWMutex): Held in read mode while any
// read-only command is in progress on the replica; held in write
// mode while executing a commit trigger. This is necessary
// because read-only commands mutate the Replica's timestamp cache
// (while holding Replica.mu in addition to readOnlyCmdMu). The
// RWMutex ensures that no reads are being executed during a split
// (which copies the timestamp cache) while still allowing
// multiple reads in parallel (#3148). TODO(bdarnell): this lock
// only needs to be held during splitTrigger, not all triggers.
//
// * baseQueue.mu: The mutex contained in each of the store's queues (such
// as the replicate queue, replica GC queue, MVCC GC queue, ...). The mutex is
// typically acquired when deciding whether to add a replica to the respective
// queue.
//
// * Store.mu: Protects the Store's map of its Replicas. Acquired and
// released briefly at the start of each request; metadata operations like
// splits acquire it again to update the map. Even though these lock
// acquisitions do not make up a single critical section, it is safe thanks
// to Replica.raftMu which prevents any concurrent modifications.
//
// * Replica.mu: Protects the Replica's in-memory state. Acquired
// and released briefly as needed (note that while the lock is
// held "briefly" in that it is not held for an entire request, we
// do sometimes do I/O while holding the lock, as in
// Replica.Entries). This lock should be held when calling any
// methods on the raft group. Raft may call back into the Replica
// via the methods of the raft.Storage interface, which assume the
// lock is held even though they do not follow our convention of
// the "Locked" suffix.
//
// * Store.scheduler.mu: Protects the Raft scheduler internal
// state. Callbacks from the scheduler are performed while not holding this
// mutex in order to observe the above ordering constraints.
//
// Splits and merges deserve special consideration: they operate on two
// ranges. For splits, this might seem fine because the right-hand range is
// brand new, but an uninitialized version may have been created by a raft
// message before we process the split (see commentary on
// Replica.splitTrigger). We make this safe, for both splits and merges, by
// locking the right-hand range for the duration of the Raft command
// containing the split/merge trigger.
//
// Note that because we acquire and release Store.mu and Replica.mu
// repeatedly rather than holding a lock for an entire request, we are
// actually relying on higher-level locks to ensure that things don't change
// out from under us. In particular, handleRaftReady accesses the replicaID
// more than once, and we rely on Replica.raftMu to ensure that this is not
// modified by a concurrent HandleRaftRequest. (#4476)
mu struct {
syncutil.RWMutex
// Map of replicas by Range ID (map[roachpb.RangeID]*Replica).
// May be read without holding Store.mu.
replicasByRangeID rangeIDReplicaMap
// A btree key containing objects of type *Replica or *ReplicaPlaceholder.
// Both types have an associated key range; the btree is keyed on their
// start keys.
//
// INVARIANT: Any ReplicaPlaceholder in this map is also in replicaPlaceholders.
// INVARIANT: Any Replica with Replica.IsInitialized()==true is also in replicasByRangeID.
replicasByKey *storeReplicaBTree
// All *Replica objects for which Replica.IsInitialized is false.
//
// INVARIANT: any entry in this map is also in replicasByRangeID.
uninitReplicas map[roachpb.RangeID]*Replica // Map of uninitialized replicas by Range ID
// replicaPlaceholders is a map to access all placeholders, so they can
// be directly accessed and cleared after stepping all raft groups.
//
// INVARIANT: any entry in this map is also in replicasByKey.
replicaPlaceholders map[roachpb.RangeID]*ReplicaPlaceholder
}
// The unquiesced subset of replicas.
unquiescedReplicas struct {
syncutil.Mutex
m map[roachpb.RangeID]struct{}
}
// The subset of replicas with active rangefeeds.
rangefeedReplicas struct {
syncutil.Mutex
m map[roachpb.RangeID]struct{}
}
// replicaQueues is a map of per-Replica incoming request queues. These
// queues might more naturally belong in Replica, but are kept separate to
// avoid reworking the locking in getOrCreateReplica which requires
// Replica.raftMu to be held while a replica is being inserted into
// Store.mu.replicas.
replicaQueues syncutil.IntMap // map[roachpb.RangeID]*raftRequestQueue
scheduler *raftScheduler
// livenessMap is a map from nodeID to a bool indicating
// liveness. It is updated periodically in raftTickLoop()
// and reactively in nodeIsLiveCallback() on liveness updates.
livenessMap atomic.Value
// cachedCapacity caches information on store capacity to prevent
// expensive recomputations in case leases or replicas are rapidly
// rebalancing.
cachedCapacity struct {
syncutil.Mutex
roachpb.StoreCapacity
}
counts struct {
// Number of placeholders removed due to error. Not a good fit for meaningful
// metrics, as snapshots to initialized ranges don't get a placeholder.
failedPlaceholders int32
// Number of placeholders successfully filled by a snapshot. Not a good fit
// for meaningful metrics, as snapshots to initialized ranges don't get a
// placeholder.
filledPlaceholders int32
// Number of placeholders removed due to a snapshot that was dropped by
// raft. Not a good fit for meaningful metrics, as snapshots to initialized
// ranges don't get a placeholder.
droppedPlaceholders int32
}
// tenantRateLimiters manages tenantrate.Limiters
tenantRateLimiters *tenantrate.LimiterFactory
computeInitialMetrics sync.Once
systemConfigUpdateQueueRateLimiter *quotapool.RateLimiter
spanConfigUpdateQueueRateLimiter *quotapool.RateLimiter
}
var _ kv.Sender = &Store{}
// A StoreConfig encompasses the auxiliary objects and configuration
// required to create a store.
// All fields holding a pointer or an interface are required to create
// a store; the rest will have sane defaults set if omitted.
type StoreConfig struct {
AmbientCtx log.AmbientContext
base.RaftConfig
DefaultSpanConfig roachpb.SpanConfig
Settings *cluster.Settings
Clock *hlc.Clock
DB *kv.DB
Gossip *gossip.Gossip
NodeLiveness *liveness.NodeLiveness
StorePool *StorePool
Transport *RaftTransport
NodeDialer *nodedialer.Dialer
RPCContext *rpc.Context
RangeDescriptorCache *rangecache.RangeCache
ClosedTimestampSender *sidetransport.Sender
ClosedTimestampReceiver sidetransportReceiver
// SQLExecutor is used by the store to execute SQL statements.
SQLExecutor sqlutil.InternalExecutor
// TimeSeriesDataStore is an interface used by the store's time series
// maintenance queue to dispatch individual maintenance tasks.
TimeSeriesDataStore TimeSeriesDataStore
// CoalescedHeartbeatsInterval is the interval for which heartbeat messages
// are queued and then sent as a single coalesced heartbeat; it is a
// fraction of the RaftTickInterval so that heartbeats don't get delayed by
// an entire tick. Delaying coalescing heartbeat responses has a bad
// interaction with quiescence because the coalesced (delayed) heartbeat
// response can unquiesce the leader. Consider:
//
// T+0: leader queues MsgHeartbeat
// T+1: leader sends MsgHeartbeat
// follower receives MsgHeartbeat
// follower queues MsgHeartbeatResp
// T+2: leader queues quiesce message
// follower sends MsgHeartbeatResp
// leader receives MsgHeartbeatResp
// T+3: leader sends quiesce message